Skip to contentSkip to conversation

oe.license: Add functions to split and canonicalise license strings.

PATCH In review Meh Mbeh Ida Delphine 4 participants 12 messages Mar 9, 2021 – Mar 12, 2021

lore
Share & tools
lore
#0 lore raw

PATCH 1/4 oe.license: Add functions to split and canonicalise license strings.

These functions ensures recipe LICENSE strings are split into their individual licenses and then canonicalised to easily enable their matching with licenses of sources during the packaging process.

---
 meta/lib/oe/license.py | 23 +++++++++++++++++++++++
 1 file changed, 23 insertions(+)
meta/lib/oe/license.py +23 -0
index 665d32ecbb..9e9957be1d 100644
@@ -236,3 +236,26 @@ def list_licenses(licensestr):
     except SyntaxError as exc:
         raise LicenseSyntaxError(licensestr, exc)
     return visitor.licenses
+
+def canonical_license(license, d):
+    """
+    Return the canonical (SPDX) form of the license if available (so GPLv3
+    becomes GPL-3.0), for the license named 'X+', return canonical form of
+    'X' if available and the tailing '+' (so GPLv3+ becomes GPL-3.0+),
+    or the passed license if there is no canonical form.
+    """
+    lic = d.getVarFlag('SPDXLICENSEMAP', license) or ""
+    if not lic and license.endswith('+'):
+        lic = d.getVarFlag('SPDXLICENSEMAP', license.rstrip('+'))
+        if lic:
+            lic += '+'
+    return lic or license
+
+def split_spdx_lic(licensestr,d):
+    """
+    Split the license strings and returns a set of the
+    canonicalised licenses.
+    """
+    split_lic = list_licenses(licensestr)
+    spdx_lic = set([canonical_license(l, d) for l in split_lic])
+    return spdx_lic
\ No newline at end of file
Signature · 1 line
2.25.1
#1 lore raw

PATCH 2/4 package.bbclass: Displays warnings for just licences in recipes.

After the comparism between computedpkglics and recipe lics, the warnings are displayed if and only if there are licenses in the recipes not found in the sources.

---
 meta/classes/package.bbclass | 17 ++++++++++++-----
 1 file changed, 12 insertions(+), 5 deletions(-)
meta/classes/package.bbclass +12 -5
index c127178f95..4e1be661b3 100644
@@ -1551,6 +1551,7 @@ PKGDESTWORK = "${WORKDIR}/pkgdata"
 PKGDATA_VARS = "PN PE PV PR PKGE PKGV PKGR LICENSE DESCRIPTION SUMMARY RDEPENDS RPROVIDES RRECOMMENDS RSUGGESTS RREPLACES RCONFLICTS SECTION PKG ALLOW_EMPTY FILES CONFFILES FILES_INFO PACKAGE_ADD_METADATA pkg_postinst pkg_postrm pkg_preinst pkg_prerm"
 
 python emit_pkgdata() {
+    import oe.license
     from glob import glob
     import json
     import subprocess
@@ -1748,8 +1749,8 @@ fi
                     if r[0] not in computedlics:
                         computedlics[r[0]] = set()
                     computedlics[r[0]].add(filelics[subf])
-        #if computedlics:
-        #    bb.warn(str(computedlics))
+        
+        
         dvar = d.getVar('PKGD')
         for f in computedlics:
             shortf = f.replace(dvar, "")
@@ -1762,13 +1763,19 @@ fi
                     computedpkglics[pkg].update(computedlics[f])
             if not found:
                 bb.warn("%s not in %s" % (f, str(filemap)))
-        #if computedpkglics:
-        #    bb.warn(str(computedpkglics))
+        
         for pkg in computedpkglics:
             lic = d.getVar('LICENSE_%s' % (pkg))
             if not lic:
                 lic = d.getVar('LICENSE')
-            bb.warn("License for package %s is %s vs %s" % (pkg, computedpkglics[pkg], lic))
+
+            # Splits the LICENSE values and canonicalise each license
+            # in the set of split license(s)
+            spdx_lic = oe.license.split_spdx_lic(lic, d)    
+            
+            # Displays warnings for licenses found in the recipes and not sources
+            if spdx_lic - computedpkglics[pkg]:
+                bb.warn("License for package %s is %s vs %s" % (pkg, computedpkglics[pkg], spdx_lic))
 }
 emit_pkgdata[dirs] = "${PKGDESTWORK}/runtime ${PKGDESTWORK}/runtime-reverse ${PKGDESTWORK}/runtime-rprovides"
 
Signature · 1 line
2.25.1
#2 lore raw

Re: [OE-core] [poky-contrib][PATCH 2/4] package.bbclass: Displays warnings for just licences in recipes.

Quoted 30 lines

-----Original Message-----
From: [email protected] <openembedded- [email protected]> On Behalf Of Meh Mbeh Ida Delphine
Sent: den 10 mars 2021 00:31
To: [email protected]
Subject: [OE-core] [poky-contrib][PATCH 2/4] package.bbclass: Displays warnings for just licences in recipes.

After the comparism between computedpkglics and recipe lics, the warnings are displayed if and only if there are licenses in the recipes not found in the sources.

---
 meta/classes/package.bbclass | 17 ++++++++++++-----
 1 file changed, 12 insertions(+), 5 deletions(-)
meta/classes/package.bbclass +1 -0
index c127178f95..4e1be661b3 100644
@@ -1551,6 +1551,7 @@ PKGDESTWORK = "${WORKDIR}/pkgdata"
 PKGDATA_VARS = "PN PE PV PR PKGE PKGV PKGR LICENSE DESCRIPTION SUMMARY RDEPENDS RPROVIDES RRECOMMENDS RSUGGESTS RREPLACES RCONFLICTS SECTION PKG ALLOW_EMPTY FILES CONFFILES FILES_INFO PACKAGE_ADD_METADATA pkg_postinst pkg_postrm pkg_preinst pkg_prerm"
 
 python emit_pkgdata() {
+    import oe.license
     from glob import glob
     import json
     import subprocess

meta/classes/package.bbclass fi · line 1753

Quoted 7 lines
meta/classes/package.bbclass +2 -2
@@ -1748,8 +1749,8 @@ fi
                     if r[0] not in computedlics:
                         computedlics[r[0]] = set()
                     computedlics[r[0]].add(filelics[subf])
-        #if computedlics:
-        #    bb.warn(str(computedlics))
+
+

One empty line is enough.

Quoted 15 lines
meta/classes/package.bbclass +1 -3
         dvar = d.getVar('PKGD')
         for f in computedlics:
             shortf = f.replace(dvar, "")
@@ -1762,13 +1763,19 @@ fi
                     computedpkglics[pkg].update(computedlics[f])
             if not found:
                 bb.warn("%s not in %s" % (f, str(filemap)))
-        #if computedpkglics:
-        #    bb.warn(str(computedpkglics))
+
         for pkg in computedpkglics:
             lic = d.getVar('LICENSE_%s' % (pkg))
             if not lic:
                 lic = d.getVar('LICENSE')
-            bb.warn("License for package %s is %s vs %s" % (pkg, computedpkglics[pkg], lic))

meta/classes/package.bbclass fi · line 1780

Quoted 10 lines
meta/classes/package.bbclass +8 -0
+
+            # Splits the LICENSE values and canonicalise each license
+            # in the set of split license(s)
+            spdx_lic = oe.license.split_spdx_lic(lic, d)
+
+            # Displays warnings for licenses found in the recipes and not sources
+            if spdx_lic - computedpkglics[pkg]:
+                bb.warn("License for package %s is %s vs %s" % (pkg, computedpkglics[pkg], spdx_lic))
 }
 emit_pkgdata[dirs] = "${PKGDESTWORK}/runtime ${PKGDESTWORK}/runtime-reverse ${PKGDESTWORK}/runtime-rprovides"

Since you are now using a function from oe.license that references Bitbake variables, I believe you should add emit_pkgdata[vardeps] for those variables as bitbake will not pick them up automatically.

Quoted 3 lines
meta/classes/package.bbclass +0 -1
--

2.25.1

//Peter

#3 lore raw

PATCH 3/4 package.bbclass: Add handle_qa_error check

In reply to Meh Mbeh Ida Delphine

package_qa_handle_error() from insane.bbclass handles display of warnings by allowing the user decide whether they want the issues treated as warnings, errors or hidden completely. This is done by setting a variable "license_source_spdx" in local.conf.

---
 meta/classes/package.bbclass | 128 ++++++++++++++++++-----------------
 1 file changed, 65 insertions(+), 63 deletions(-)
meta/classes/package.bbclass +65 -63
index 4e1be661b3..8c0a49ad76 100644
@@ -1552,6 +1552,7 @@ PKGDATA_VARS = "PN PE PV PR PKGE PKGV PKGR LICENSE DESCRIPTION SUMMARY RDEPENDS
 
 python emit_pkgdata() {
     import oe.license
+    import bb.utils
     from glob import glob
     import json
     import subprocess
@@ -1712,70 +1713,71 @@ fi
         write_extra_runtime_pkgs(global_variants, packages, pkgdatadir)
 
     sourceresult = d.getVar('TEMPDBGSRCMAPPING', False)
-    sources = {}
-    if sourceresult:
-        for r in sourceresult:
-            sources[r[0]] = r[1]
-        with open(data_file + ".srclist", 'w') as f:
-            f.write(json.dumps(sources, sort_keys=True))
-
-        filelics = {}
-        for dirent in [d.getVar('PKGD'), d.getVar('STAGING_DIR_TARGET')]:
-            p = subprocess.Popen(["grep", 'SPDX-License-Identifier:', '-R', '-I'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=dirent)
-            out, err = p.communicate()
-            if p.returncode == 0:
-                for l in out.decode("utf-8").split("\n"):
-                    l = l.strip()
-                    if not l:
-                        continue
-                    l = l.split(":")
-                    if len(l) < 3:
-                        bb.warn(str(l))
-                        continue
-                    fn = "/" + l[0]
-                    lic = l[2].strip()
-                    if lic.endswith("*/"):
-                        lic = lic[:-2]
-                    lic = lic.strip()
-                    filelics[fn] = lic
-        with open(data_file + ".filelics", 'w') as f:
-            f.write(json.dumps(filelics, sort_keys=True))
-
-        computedlics = {}
-        computedpkglics = {}
-        for r in sourceresult:
-            for subf in r[1]:
-                if subf in filelics:
-                    if r[0] not in computedlics:
-                        computedlics[r[0]] = set()
-                    computedlics[r[0]].add(filelics[subf])
-        
-        
-        dvar = d.getVar('PKGD')
-        for f in computedlics:
-            shortf = f.replace(dvar, "")
-            found = False
-            for pkg in filemap:
-                if shortf in filemap[pkg]:
-                    found = True
-                    if pkg not in computedpkglics:
-                        computedpkglics[pkg] = set()
-                    computedpkglics[pkg].update(computedlics[f])
-            if not found:
-                bb.warn("%s not in %s" % (f, str(filemap)))
-        
-        for pkg in computedpkglics:
-            lic = d.getVar('LICENSE_%s' % (pkg))
-            if not lic:
-                lic = d.getVar('LICENSE')
-
-            # Splits the LICENSE values and canonicalise each license
-            # in the set of split license(s)
-            spdx_lic = oe.license.split_spdx_lic(lic, d)    
+    if bb.utils.contains("license_source_spdx", "1", True, False, d):
+        sources = {}
+        if sourceresult:
+            for r in sourceresult:
+                sources[r[0]] = r[1]
+            with open(data_file + ".srclist", 'w') as f:
+                f.write(json.dumps(sources, sort_keys=True))
+
+            filelics = {}
+            for dirent in [d.getVar('PKGD'), d.getVar('STAGING_DIR_TARGET')]:
+                p = subprocess.Popen(["grep", 'SPDX-License-Identifier:', '-R', '-I'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=dirent)
+                out, err = p.communicate()
+                if p.returncode == 0:
+                    for l in out.decode("utf-8").split("\n"):
+                        l = l.strip()
+                        if not l:
+                            continue
+                        l = l.split(":")
+                        if len(l) < 3:
+                            bb.warn(str(l))
+                            continue
+                        fn = "/" + l[0]
+                        lic = l[2].strip()
+                        if lic.endswith("*/"):
+                            lic = lic[:-2]
+                        lic = lic.strip()
+                        filelics[fn] = lic
+            with open(data_file + ".filelics", 'w') as f:
+                f.write(json.dumps(filelics, sort_keys=True))
+
+            computedlics = {}
+            computedpkglics = {}
+            for r in sourceresult:
+                for subf in r[1]:
+                    if subf in filelics:
+                        if r[0] not in computedlics:
+                            computedlics[r[0]] = set()
+                        computedlics[r[0]].add(filelics[subf])
+            
+            
+            dvar = d.getVar('PKGD')
+            for f in computedlics:
+                shortf = f.replace(dvar, "")
+                found = False
+                for pkg in filemap:
+                    if shortf in filemap[pkg]:
+                        found = True
+                        if pkg not in computedpkglics:
+                            computedpkglics[pkg] = set()
+                        computedpkglics[pkg].update(computedlics[f])
+                if not found:
+                    bb.warn("%s not in %s" % (f, str(filemap)))
             
-            # Displays warnings for licenses found in the recipes and not sources
-            if spdx_lic - computedpkglics[pkg]:
-                bb.warn("License for package %s is %s vs %s" % (pkg, computedpkglics[pkg], spdx_lic))
+            for pkg in computedpkglics:
+                lic = d.getVar('LICENSE_%s' % (pkg))
+                if not lic:
+                    lic = d.getVar('LICENSE')
+
+                # Splits the LICENSE values and canonicalise each license
+                # in the set of split license(s)
+                spdx_lic = oe.license.split_spdx_lic(lic, d)    
+                
+                # Displays warnings for licenses found in the recipes and not sources
+                if spdx_lic - computedpkglics[pkg]:
+                    package_qa_handle_error("license_source_spdx", f'License for package {pkg} is {computedpkglics[pkg]}(source SPDX headers) vs {spdx_lic} (LICENSE)', d)
 }
 emit_pkgdata[dirs] = "${PKGDESTWORK}/runtime ${PKGDESTWORK}/runtime-reverse ${PKGDESTWORK}/runtime-rprovides"
 
Signature · 1 line
2.25.1
#4 lore raw

Re: [OE-core] [poky-contrib][PATCH 3/4] package.bbclass: Add handle_qa_error check

Quoted 86 lines

-----Original Message-----
From: [email protected] <openembedded- [email protected]> On Behalf Of Meh Mbeh Ida Delphine
Sent: den 10 mars 2021 00:31
To: [email protected]
Subject: [OE-core] [poky-contrib][PATCH 3/4] package.bbclass: Add handle_qa_error check

package_qa_handle_error() from insane.bbclass handles display of warnings by allowing the user decide whether they want the issues treated as warnings, errors or hidden completely. This is done by setting a variable "license_source_spdx" in local.conf.

---
 meta/classes/package.bbclass | 128 ++++++++++++++++++-----------------
 1 file changed, 65 insertions(+), 63 deletions(-)
meta/classes/package.bbclass +0 -0
index 4e1be661b3..8c0a49ad76 100644
@@ -1552,6 +1552,7 @@ PKGDATA_VARS = "PN PE PV PR PKGE PKGV PKGR LICENSE

DESCRIPTION SUMMARY RDEPENDS

python emit_pkgdata() {
import oe.license

meta/classes/package.bbclass +1 -51
+    import bb.utils
     from glob import glob
     import json
     import subprocess
@@ -1712,70 +1713,71 @@ fi
         write_extra_runtime_pkgs(global_variants, packages, pkgdatadir)
 
     sourceresult = d.getVar('TEMPDBGSRCMAPPING', False)
-    sources = {}
-    if sourceresult:
-        for r in sourceresult:
-            sources[r[0]] = r[1]
-        with open(data_file + ".srclist", 'w') as f:
-            f.write(json.dumps(sources, sort_keys=True))
-
-        filelics = {}
-        for dirent in [d.getVar('PKGD'), d.getVar('STAGING_DIR_TARGET')]:
-            p = subprocess.Popen(["grep", 'SPDX-License-Identifier:', '-R', '-I'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=dirent)
-            out, err = p.communicate()
-            if p.returncode == 0:
-                for l in out.decode("utf-8").split("\n"):
-                    l = l.strip()
-                    if not l:
-                        continue
-                    l = l.split(":")
-                    if len(l) < 3:
-                        bb.warn(str(l))
-                        continue
-                    fn = "/" + l[0]
-                    lic = l[2].strip()
-                    if lic.endswith("*/"):
-                        lic = lic[:-2]
-                    lic = lic.strip()
-                    filelics[fn] = lic
-        with open(data_file + ".filelics", 'w') as f:
-            f.write(json.dumps(filelics, sort_keys=True))
-
-        computedlics = {}
-        computedpkglics = {}
-        for r in sourceresult:
-            for subf in r[1]:
-                if subf in filelics:
-                    if r[0] not in computedlics:
-                        computedlics[r[0]] = set()
-                    computedlics[r[0]].add(filelics[subf])
-
-
-        dvar = d.getVar('PKGD')
-        for f in computedlics:
-            shortf = f.replace(dvar, "")
-            found = False
-            for pkg in filemap:
-                if shortf in filemap[pkg]:
-                    found = True
-                    if pkg not in computedpkglics:
-                        computedpkglics[pkg] = set()
-                    computedpkglics[pkg].update(computedlics[f])
-            if not found:
-                bb.warn("%s not in %s" % (f, str(filemap)))

meta/classes/package.bbclass fi · line 1716

Quoted 10 lines
meta/classes/package.bbclass +1 -9
-
-        for pkg in computedpkglics:
-            lic = d.getVar('LICENSE_%s' % (pkg))
-            if not lic:
-                lic = d.getVar('LICENSE')
-
-            # Splits the LICENSE values and canonicalise each license
-            # in the set of split license(s)
-            spdx_lic = oe.license.split_spdx_lic(lic, d)
+    if bb.utils.contains("license_source_spdx", "1", True, False, d):

This kind of bitbake variable that controls functionality is typically in upper case. It also needs to be documented in the manual, and it should have a default value (??=). It would also make more sense to use bb.utils.to_boolean() than bb.utils.contains(). You can also do:

if not bb.utils.to_boolean(d.getVar("license_source_spdx")):
return

thereby avoiding the reindentation, making code review easier.

Quoted 57 lines
meta/classes/package.bbclass +53 -3
+        sources = {}
+        if sourceresult:
+            for r in sourceresult:
+                sources[r[0]] = r[1]
+            with open(data_file + ".srclist", 'w') as f:
+                f.write(json.dumps(sources, sort_keys=True))
+
+            filelics = {}
+            for dirent in [d.getVar('PKGD'), d.getVar('STAGING_DIR_TARGET')]:
+                p = subprocess.Popen(["grep", 'SPDX-License-Identifier:', '-R', '-I'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=dirent)
+                out, err = p.communicate()
+                if p.returncode == 0:
+                    for l in out.decode("utf-8").split("\n"):
+                        l = l.strip()
+                        if not l:
+                            continue
+                        l = l.split(":")
+                        if len(l) < 3:
+                            bb.warn(str(l))
+                            continue
+                        fn = "/" + l[0]
+                        lic = l[2].strip()
+                        if lic.endswith("*/"):
+                            lic = lic[:-2]
+                        lic = lic.strip()
+                        filelics[fn] = lic
+            with open(data_file + ".filelics", 'w') as f:
+                f.write(json.dumps(filelics, sort_keys=True))
+
+            computedlics = {}
+            computedpkglics = {}
+            for r in sourceresult:
+                for subf in r[1]:
+                    if subf in filelics:
+                        if r[0] not in computedlics:
+                            computedlics[r[0]] = set()
+                        computedlics[r[0]].add(filelics[subf])
+
+
+            dvar = d.getVar('PKGD')
+            for f in computedlics:
+                shortf = f.replace(dvar, "")
+                found = False
+                for pkg in filemap:
+                    if shortf in filemap[pkg]:
+                        found = True
+                        if pkg not in computedpkglics:
+                            computedpkglics[pkg] = set()
+                        computedpkglics[pkg].update(computedlics[f])
+                if not found:
+                    bb.warn("%s not in %s" % (f, str(filemap)))
 
-            # Displays warnings for licenses found in the recipes and not sources
-            if spdx_lic - computedpkglics[pkg]:
-                bb.warn("License for package %s is %s vs %s" % (pkg, computedpkglics[pkg], spdx_lic))
+            for pkg in computedpkglics:
+                lic = d.getVar('LICENSE_%s' % (pkg))

meta/classes/package.bbclass fi · line 1780

Quoted 10 lines
meta/classes/package.bbclass +10 -0
+                if not lic:
+                    lic = d.getVar('LICENSE')
+
+                # Splits the LICENSE values and canonicalise each license
+                # in the set of split license(s)
+                spdx_lic = oe.license.split_spdx_lic(lic, d)
+
+                # Displays warnings for licenses found in the recipes and not sources
+                if spdx_lic - computedpkglics[pkg]:
+                    package_qa_handle_error("license_source_spdx", f'License for package {pkg} is {computedpkglics[pkg]}(source SPDX headers) vs {spdx_lic} (LICENSE)', d)

I'm all for using f'' strings, as I think it makes for a lot more readable code. However, I am not sure it has been approved for use in the OE code yet.

Quoted 5 lines
meta/classes/package.bbclass +0 -1
 }
 emit_pkgdata[dirs] = "${PKGDESTWORK}/runtime ${PKGDESTWORK}/runtime-reverse ${PKGDESTWORK}/runtime-rprovides"
 
--

2.25.1

#5 lore raw

Re: [OE-core] [poky-contrib][PATCH 3/4] package.bbclass: Add handle_qa_error check

Indeed, bitbake only requires python 3.5.0 right now, we can't use 3.6 or 3.7 features in it or oe-core at this time.

Quoted 202 lines · Peter Kjellerstedt
Quoted 34 lines

-----Original Message-----
From: [email protected] <openembedded- [email protected]> On Behalf Of Meh Mbeh Ida Delphine
Sent: den 10 mars 2021 00:31
To: [email protected]
Subject: [OE-core] [poky-contrib][PATCH 3/4] package.bbclass: Add handle_qa_error check

package_qa_handle_error() from insane.bbclass handles display of warnings by allowing the user decide whether they want the issues treated as warnings, errors or hidden completely. This is done by setting a variable "license_source_spdx" in local.conf.

---
 meta/classes/package.bbclass | 128 ++++++++++++++++++-----------------
 1 file changed, 65 insertions(+), 63 deletions(-)
meta/classes/package.bbclass +0 -0
index 4e1be661b3..8c0a49ad76 100644
@@ -1552,6 +1552,7 @@ PKGDATA_VARS = "PN PE PV PR PKGE PKGV PKGR LICENSE

DESCRIPTION SUMMARY RDEPENDS

python emit_pkgdata() {
import oe.license

meta/classes/package.bbclass +1 -0
+    import bb.utils
     from glob import glob
     import json
     import subprocess
@@ -1712,70 +1713,71 @@ fi
         write_extra_runtime_pkgs(global_variants, packages, pkgdatadir)
 

meta/classes/package.bbclass fi · line 1723

Quoted 10 lines
meta/classes/package.bbclass +0 -9
     sourceresult = d.getVar('TEMPDBGSRCMAPPING', False)
-    sources = {}
-    if sourceresult:
-        for r in sourceresult:
-            sources[r[0]] = r[1]
-        with open(data_file + ".srclist", 'w') as f:
-            f.write(json.dumps(sources, sort_keys=True))
-
-        filelics = {}
-        for dirent in [d.getVar('PKGD'),

d.getVar('STAGING_DIR_TARGET')]:

meta/classes/package.bbclass fi · line 1724

Quoted 1 line
meta/classes/package.bbclass +0 -1
-            p = subprocess.Popen(["grep", 'SPDX-License-Identifier:',

'-R', '-I'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=dirent)

Quoted 41 lines
meta/classes/package.bbclass +0 -41
-            out, err = p.communicate()
-            if p.returncode == 0:
-                for l in out.decode("utf-8").split("\n"):
-                    l = l.strip()
-                    if not l:
-                        continue
-                    l = l.split(":")
-                    if len(l) < 3:
-                        bb.warn(str(l))
-                        continue
-                    fn = "/" + l[0]
-                    lic = l[2].strip()
-                    if lic.endswith("*/"):
-                        lic = lic[:-2]
-                    lic = lic.strip()
-                    filelics[fn] = lic
-        with open(data_file + ".filelics", 'w') as f:
-            f.write(json.dumps(filelics, sort_keys=True))
-
-        computedlics = {}
-        computedpkglics = {}
-        for r in sourceresult:
-            for subf in r[1]:
-                if subf in filelics:
-                    if r[0] not in computedlics:
-                        computedlics[r[0]] = set()
-                    computedlics[r[0]].add(filelics[subf])
-
-
-        dvar = d.getVar('PKGD')
-        for f in computedlics:
-            shortf = f.replace(dvar, "")
-            found = False
-            for pkg in filemap:
-                if shortf in filemap[pkg]:
-                    found = True
-                    if pkg not in computedpkglics:
-                        computedpkglics[pkg] = set()
-                    computedpkglics[pkg].update(computedlics[f])
-            if not found:
-                bb.warn("%s not in %s" % (f, str(filemap)))

meta/classes/package.bbclass fi · line 1716

Quoted 10 lines
meta/classes/package.bbclass +1 -9
-
-        for pkg in computedpkglics:
-            lic = d.getVar('LICENSE_%s' % (pkg))
-            if not lic:
-                lic = d.getVar('LICENSE')
-
-            # Splits the LICENSE values and canonicalise each license
-            # in the set of split license(s)
-            spdx_lic = oe.license.split_spdx_lic(lic, d)
+    if bb.utils.contains("license_source_spdx", "1", True, False, d):

This kind of bitbake variable that controls functionality is typically in upper case. It also needs to be documented in the manual, and it should have
a default value (??=). It would also make more sense to use bb.utils.to_boolean() than bb.utils.contains(). You can also do:

if not bb.utils.to_boolean(d.getVar("license_source_spdx")):
return

thereby avoiding the reindentation, making code review easier.

meta/classes/package.bbclass fi

Quoted 9 lines
meta/classes/package.bbclass +9 -0
+        sources = {}
+        if sourceresult:
+            for r in sourceresult:
+                sources[r[0]] = r[1]
+            with open(data_file + ".srclist", 'w') as f:
+                f.write(json.dumps(sources, sort_keys=True))
+
+            filelics = {}
+            for dirent in [d.getVar('PKGD'),

d.getVar('STAGING_DIR_TARGET')]:

meta/classes/package.bbclass fi · line 1726

Quoted 1 line
meta/classes/package.bbclass +1 -0
+                p = subprocess.Popen(["grep",

'SPDX-License-Identifier:', '-R', '-I'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=dirent)

Quoted 33 lines
meta/classes/package.bbclass +33 -0
+                out, err = p.communicate()
+                if p.returncode == 0:
+                    for l in out.decode("utf-8").split("\n"):
+                        l = l.strip()
+                        if not l:
+                            continue
+                        l = l.split(":")
+                        if len(l) < 3:
+                            bb.warn(str(l))
+                            continue
+                        fn = "/" + l[0]
+                        lic = l[2].strip()
+                        if lic.endswith("*/"):
+                            lic = lic[:-2]
+                        lic = lic.strip()
+                        filelics[fn] = lic
+            with open(data_file + ".filelics", 'w') as f:
+                f.write(json.dumps(filelics, sort_keys=True))
+
+            computedlics = {}
+            computedpkglics = {}
+            for r in sourceresult:
+                for subf in r[1]:
+                    if subf in filelics:
+                        if r[0] not in computedlics:
+                            computedlics[r[0]] = set()
+                        computedlics[r[0]].add(filelics[subf])
+
+
+            dvar = d.getVar('PKGD')
+            for f in computedlics:
+                shortf = f.replace(dvar, "")
+                found = False

meta/classes/package.bbclass fi

Quoted 10 lines
meta/classes/package.bbclass +8 -1
+                for pkg in filemap:
+                    if shortf in filemap[pkg]:
+                        found = True
+                        if pkg not in computedpkglics:
+                            computedpkglics[pkg] = set()
+                        computedpkglics[pkg].update(computedlics[f])
+                if not found:
+                    bb.warn("%s not in %s" % (f, str(filemap)))
 
-            # Displays warnings for licenses found in the recipes and

not sources

meta/classes/package.bbclass fi · line 1778

Quoted 2 lines
meta/classes/package.bbclass +0 -2
-            if spdx_lic - computedpkglics[pkg]:
-                bb.warn("License for package %s is %s vs %s" % (pkg,

computedpkglics[pkg], spdx_lic))

meta/classes/package.bbclass fi

Quoted 6 lines
meta/classes/package.bbclass +6 -0
+            for pkg in computedpkglics:
+                lic = d.getVar('LICENSE_%s' % (pkg))
+                if not lic:
+                    lic = d.getVar('LICENSE')
+
+                # Splits the LICENSE values and canonicalise each

license

meta/classes/package.bbclass fi

Quoted 4 lines
meta/classes/package.bbclass +4 -0
+                # in the set of split license(s)
+                spdx_lic = oe.license.split_spdx_lic(lic, d)
+
+                # Displays warnings for licenses found in the recipes

and not sources

meta/classes/package.bbclass fi · line 1780

Quoted 2 lines
meta/classes/package.bbclass +2 -0
+                if spdx_lic - computedpkglics[pkg]:
+                    package_qa_handle_error("license_source_spdx",

f'License for package {pkg} is {computedpkglics[pkg]}(source SPDX headers) vs {spdx_lic} (LICENSE)', d)

I'm all for using f'' strings, as I think it makes for a lot more readable code. However, I am not sure it has been approved for use in the OE code yet.

meta/classes/package.bbclass fi · line 1782

Quoted 2 lines
meta/classes/package.bbclass +0 -0
 }
 emit_pkgdata[dirs] = "${PKGDESTWORK}/runtime

${PKGDESTWORK}/runtime-reverse ${PKGDESTWORK}/runtime-rprovides"

Quoted 3 lines
meta/classes/package.bbclass +0 -1
--

2.25.1

Signature · 4 lines
Christopher Larson
kergoth at gmail dot com
Founder - BitBake, OpenEmbedded, OpenZaurus
Senior Software Engineer, Mentor Graphics
#7 lore raw

Re: [OE-core] [poky-contrib][PATCH 3/4] package.bbclass: Add handle_qa_error check

Thanks for the feedback. I will make the changes accordingly. I was hoping to document a in the manual after getting feedback in case there are corrections that need to be made.

Cheers,
Ida.

Quoted 16 lines · Richard Purdie,
Quoted 2 lines · Christopher Larson

Indeed, bitbake only requires python 3.5.0 right now, we can't use
3.6 or 3.7 features in it or oe-core at this time.

We did increase the version requirement to 3.6:

http://git.yoctoproject.org/cgit.cgi/poky/commit/?id=c08279f9c3a0260f80fb0948520a24bf2fcdb422

at least in OE-Core.

Cheers,

Richard

#8 lore raw

PATCH 4/4 Moved logic to get filelics from package.bbclass to license.py

In reply to Meh Mbeh Ida Delphine

The logic to obtain filelics is moved from package.bbclass to license.py as a standalone function.
Also, the check in package.bbclass after obtaining filelics ensures licenses with * WITH Linux-syscall-note" are ignored, as long as they're only in header files.

---
 meta/classes/package.bbclass | 24 ++++++------------------
 meta/lib/oe/license.py       | 29 ++++++++++++++++++++++++++++-
 2 files changed, 34 insertions(+), 19 deletions(-)
meta/classes/package.bbclass +6 -18
index 8c0a49ad76..a2e0eab848 100644
@@ -1721,25 +1721,13 @@ fi
             with open(data_file + ".srclist", 'w') as f:
                 f.write(json.dumps(sources, sort_keys=True))
 
+            flics = oe.license.get_filelics([d.getVar('PKGD'), d.getVar('STAGING_DIR_TARGET')])
             filelics = {}
-            for dirent in [d.getVar('PKGD'), d.getVar('STAGING_DIR_TARGET')]:
-                p = subprocess.Popen(["grep", 'SPDX-License-Identifier:', '-R', '-I'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=dirent)
-                out, err = p.communicate()
-                if p.returncode == 0:
-                    for l in out.decode("utf-8").split("\n"):
-                        l = l.strip()
-                        if not l:
-                            continue
-                        l = l.split(":")
-                        if len(l) < 3:
-                            bb.warn(str(l))
-                            continue
-                        fn = "/" + l[0]
-                        lic = l[2].strip()
-                        if lic.endswith("*/"):
-                            lic = lic[:-2]
-                        lic = lic.strip()
-                        filelics[fn] = lic
+            for k, v in flics.items():
+                if k.endswith(".h") and v.endswith("WITH Linux-syscall-note"):
+                    continue
+                else:
+                    filelics[k] = v
             with open(data_file + ".filelics", 'w') as f:
                 f.write(json.dumps(filelics, sort_keys=True))
 
meta/lib/oe/license.py +28 -1
index 9e9957be1d..3c40fa73eb 100644
@@ -3,6 +3,8 @@
 #
 """Code for parsing OpenEmbedded license strings"""
 
+import subprocess
+import bb
 import ast
 import re
 from fnmatch import fnmatchcase as fnmatch
@@ -258,4 +260,29 @@ def split_spdx_lic(licensestr,d):
     """
     split_lic = list_licenses(licensestr)
     spdx_lic = set([canonical_license(l, d) for l in split_lic])
-    return spdx_lic
\ No newline at end of file
+    return spdx_lic
+
+def get_filelics(dirs):
+    """"
+    This function returns a dictionary of file paths (keys) and their corresponding SPDX header identifiers (values).
+    """
+    filelics = {}
+    for dirent in dirs:
+        p = subprocess.Popen(["grep", 'SPDX-License-Identifier:', '-R','-I'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=dirent)
+        out, err = p.communicate()
+        if p.returncode == 0:
+            for l in out.decode("utf-8").split("\n"):
+                l = l.strip()
+                if not l:
+                    continue
+                l = l.split(":")
+                if len(l) < 3:
+                    bb.warn(str(l))
+                    continue
+                fn = "/" + l[0]
+                lic = l[2].strip()
+                if lic.endswith("*/"):
+                    lic = lic[:-2]
+                lic = lic.strip()
+                filelics[fn] = lic
+    return filelics
\ No newline at end of file
Signature · 1 line
2.25.1
#9 lore raw

Re: [OE-core] [poky-contrib][PATCH 4/4] Moved logic to get filelics from package.bbclass to license.py

Quoted 43 lines

-----Original Message-----
From: [email protected] <openembedded- [email protected]> On Behalf Of Meh Mbeh Ida Delphine
Sent: den 10 mars 2021 00:32
To: [email protected]
Subject: [OE-core] [poky-contrib][PATCH 4/4] Moved logic to get filelics from package.bbclass to license.py

The logic to obtain filelics is moved from package.bbclass to license.py as a standalone function.
Also, the check in package.bbclass after obtaining filelics ensures licenses with * WITH Linux-syscall-note" are ignored, as long as they're only in header files.

---
 meta/classes/package.bbclass | 24 ++++++------------------
 meta/lib/oe/license.py       | 29 ++++++++++++++++++++++++++++-
 2 files changed, 34 insertions(+), 19 deletions(-)
meta/classes/package.bbclass +1 -1
index 8c0a49ad76..a2e0eab848 100644
@@ -1721,25 +1721,13 @@ fi
             with open(data_file + ".srclist", 'w') as f:
                 f.write(json.dumps(sources, sort_keys=True))
 
+            flics = oe.license.get_filelics([d.getVar('PKGD'),
 d.getVar('STAGING_DIR_TARGET')])
             filelics = {}
-            for dirent in [d.getVar('PKGD'),

d.getVar('STAGING_DIR_TARGET')]:
- p = subprocess.Popen(["grep", 'SPDX-License-Identifier:',
'-R', '-I'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=dirent)

meta/classes/package.bbclass +0 -8
-                out, err = p.communicate()
-                if p.returncode == 0:
-                    for l in out.decode("utf-8").split("\n"):
-                        l = l.strip()
-                        if not l:
-                            continue
-                        l = l.split(":")
-                        if len(l) < 3:

meta/classes/package.bbclass fi · line 1727

Quoted 10 lines
meta/classes/package.bbclass +2 -8
-                            bb.warn(str(l))
-                            continue
-                        fn = "/" + l[0]
-                        lic = l[2].strip()
-                        if lic.endswith("*/"):
-                            lic = lic[:-2]
-                        lic = lic.strip()
-                        filelics[fn] = lic
+            for k, v in flics.items():
+                if k.endswith(".h") and v.endswith("WITH Linux-syscall-note"):

What about .hh files, or .hpp files, or header files from some other language than C/C++? At the least, this if statement should have a comment explaining what is going on.

Quoted 55 lines
meta/classes/package.bbclass +3 -0
+                    continue
+                else:
+                    filelics[k] = v
             with open(data_file + ".filelics", 'w') as f:
                 f.write(json.dumps(filelics, sort_keys=True))
meta/lib/oe/license.py +7 -1
index 9e9957be1d..3c40fa73eb 100644
@@ -3,6 +3,8 @@
 #
 """Code for parsing OpenEmbedded license strings"""
 
+import subprocess
+import bb
 import ast
 import re
 from fnmatch import fnmatchcase as fnmatch
@@ -258,4 +260,29 @@ def split_spdx_lic(licensestr,d):
     """
     split_lic = list_licenses(licensestr)
     spdx_lic = set([canonical_license(l, d) for l in split_lic])
-    return spdx_lic
\ No newline at end of file
+    return spdx_lic
+
+def get_filelics(dirs):
+    """"
+    This function returns a dictionary of file paths (keys) and their

corresponding SPDX header identifiers (values).

meta/lib/oe/license.py +4 -0
+    """
+    filelics = {}
+    for dirent in dirs:
+        p = subprocess.Popen(["grep", 'SPDX-License-Identifier:', '-R','-

I'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=dirent)

meta/lib/oe/license.py +17 -1
+        out, err = p.communicate()
+        if p.returncode == 0:
+            for l in out.decode("utf-8").split("\n"):
+                l = l.strip()
+                if not l:
+                    continue
+                l = l.split(":")
+                if len(l) < 3:
+                    bb.warn(str(l))
+                    continue
+                fn = "/" + l[0]
+                lic = l[2].strip()
+                if lic.endswith("*/"):
+                    lic = lic[:-2]
+                lic = lic.strip()
+                filelics[fn] = lic
+    return filelics
\ No newline at end of file
--
Quoted 1 line

2.25.1

//Peter

#10 lore raw

Re: [OE-core] [poky-contrib][PATCH 4/4] Moved logic to get filelics from package.bbclass to license.py

Okay. I will fix that.

Quoted 121 lines · Peter Kjellerstedt,
Quoted 32 lines

-----Original Message-----
From: [email protected] <openembedded- [email protected]> On Behalf Of Meh Mbeh Ida Delphine
Sent: den 10 mars 2021 00:32
To: [email protected]
Subject: [OE-core] [poky-contrib][PATCH 4/4] Moved logic to get filelics from package.bbclass to license.py

The logic to obtain filelics is moved from package.bbclass to license.py as a standalone function.
Also, the check in package.bbclass after obtaining filelics ensures licenses with * WITH Linux-syscall-note" are ignored, as long as they're only in header files.

---
 meta/classes/package.bbclass | 24 ++++++------------------
 meta/lib/oe/license.py       | 29 ++++++++++++++++++++++++++++-
 2 files changed, 34 insertions(+), 19 deletions(-)
meta/classes/package.bbclass +1 -1
index 8c0a49ad76..a2e0eab848 100644
@@ -1721,25 +1721,13 @@ fi
             with open(data_file + ".srclist", 'w') as f:
                 f.write(json.dumps(sources, sort_keys=True))
 
+            flics = oe.license.get_filelics([d.getVar('PKGD'),
 d.getVar('STAGING_DIR_TARGET')])
             filelics = {}
-            for dirent in [d.getVar('PKGD'),
Quoted 2 lines

d.getVar('STAGING_DIR_TARGET')]:
- p = subprocess.Popen(["grep",

'SPDX-License-Identifier:',

Quoted 9 lines

'-R', '-I'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=dirent)

meta/classes/package.bbclass +0 -8
-                out, err = p.communicate()
-                if p.returncode == 0:
-                    for l in out.decode("utf-8").split("\n"):
-                        l = l.strip()
-                        if not l:
-                            continue
-                        l = l.split(":")
-                        if len(l) < 3:

meta/classes/package.bbclass fi

Quoted 10 lines
meta/classes/package.bbclass +2 -8
-                            bb.warn(str(l))
-                            continue
-                        fn = "/" + l[0]
-                        lic = l[2].strip()
-                        if lic.endswith("*/"):
-                            lic = lic[:-2]
-                        lic = lic.strip()
-                        filelics[fn] = lic
+            for k, v in flics.items():
+                if k.endswith(".h") and v.endswith("WITH

Linux-syscall-note"):

What about .hh files, or .hpp files, or header files from some other language than C/C++? At the least, this if statement should have a comment explaining what is going on.

Quoted 31 lines
meta/classes/package.bbclass +3 -0
+                    continue
+                else:
+                    filelics[k] = v
             with open(data_file + ".filelics", 'w') as f:
                 f.write(json.dumps(filelics, sort_keys=True))
meta/lib/oe/license.py +7 -1
index 9e9957be1d..3c40fa73eb 100644
@@ -3,6 +3,8 @@
 #
 """Code for parsing OpenEmbedded license strings"""
 
+import subprocess
+import bb
 import ast
 import re
 from fnmatch import fnmatchcase as fnmatch
@@ -258,4 +260,29 @@ def split_spdx_lic(licensestr,d):
     """
     split_lic = list_licenses(licensestr)
     spdx_lic = set([canonical_license(l, d) for l in split_lic])
-    return spdx_lic
\ No newline at end of file
+    return spdx_lic
+
+def get_filelics(dirs):
+    """"
+    This function returns a dictionary of file paths (keys) and their

corresponding SPDX header identifiers (values).

meta/lib/oe/license.py def split_spdx_lic(licensestr,d):

Quoted 4 lines
meta/lib/oe/license.py +4 -0
+    """
+    filelics = {}
+    for dirent in dirs:
+        p = subprocess.Popen(["grep", 'SPDX-License-Identifier:',

'-R','-

Quoted 20 lines

I'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=dirent)

meta/lib/oe/license.py +17 -1
+        out, err = p.communicate()
+        if p.returncode == 0:
+            for l in out.decode("utf-8").split("\n"):
+                l = l.strip()
+                if not l:
+                    continue
+                l = l.split(":")
+                if len(l) < 3:
+                    bb.warn(str(l))
+                    continue
+                fn = "/" + l[0]
+                lic = l[2].strip()
+                if lic.endswith("*/"):
+                    lic = lic[:-2]
+                lic = lic.strip()
+                filelics[fn] = lic
+    return filelics
\ No newline at end of file
--
Quoted 1 line

2.25.1

//Peter

#11 lore raw
In reply to Meh Mbeh Ida Delphine
Quoted 23 lines

-----Original Message-----
From: [email protected] <[email protected]> On Behalf Of Meh Mbeh Ida Delphine
Sent: den 10 mars 2021 00:31
To: [email protected]
Subject: [OE-core] [poky-contrib][PATCH 1/4] oe.license: Add functions to split and canonicalise license strings.

These functions ensures recipe LICENSE strings are split into their individual licenses and then canonicalised to easily enable their matching with licenses of sources during the packaging process.

---
 meta/lib/oe/license.py | 23 +++++++++++++++++++++++
 1 file changed, 23 insertions(+)
meta/lib/oe/license.py +0 -0
index 665d32ecbb..9e9957be1d 100644
@@ -236,3 +236,26 @@ def list_licenses(licensestr):
     except SyntaxError as exc:
         raise LicenseSyntaxError(licensestr, exc)
     return visitor.licenses

meta/lib/oe/license.py def list_licenses(licensestr): · line 248

Quoted 10 lines
meta/lib/oe/license.py +10 -0
+
+def canonical_license(license, d):
+    """
+    Return the canonical (SPDX) form of the license if available (so GPLv3
+    becomes GPL-3.0), for the license named 'X+', return canonical form of
+    'X' if available and the tailing '+' (so GPLv3+ becomes GPL-3.0+),
+    or the passed license if there is no canonical form.
+    """
+    lic = d.getVarFlag('SPDXLICENSEMAP', license) or ""
+    if not lic and license.endswith('+'):

Do we really need this if statement now that there are entries for all relevant licenses that use '+' in the SPDXLICENSEMAP? Isn't it better to rely on the SPDXLICENSEMAP to handle any special cases like this than to add code all over the place to support this legacy way of marking licenses as "-or-later".

Quoted 4 lines
meta/lib/oe/license.py +4 -0
+        lic = d.getVarFlag('SPDXLICENSEMAP', license.rstrip('+'))
+        if lic:
+            lic += '+'
+    return lic or license

meta/lib/oe/license.py def list_licenses(licensestr): · line 261

Quoted 10 lines
meta/lib/oe/license.py +9 -0
+
+def split_spdx_lic(licensestr,d):
+    """
+    Split the license strings and returns a set of the
+    canonicalised licenses.
+    """
+    split_lic = list_licenses(licensestr)
+    spdx_lic = set([canonical_license(l, d) for l in split_lic])
+    return spdx_lic
\ No newline at end of file

Please add a newline on the last line.

Quoted 2 lines

--
2.25.1

//Peter