create-spdx: Get SPDX-License-Identifier from source
This patch will read the begining of source files and try to find the SPDX-License-Identifier to populate the licenseInfoInFiles field for each source file. This does not populate licenseConculed at this time, nor rolls it up to package level.
--- classes/create-spdx.bbclass | 25 +++++++++++++++++++++++++ lib/oe/spdx.py | 2 +- 2 files changed, 26 insertions(+), 1 deletion(-)
classes/create-spdx.bbclass +25 -0
@@ -30,6 +30,21 @@ SPDX_LICENSES ??= "${COREBASE}/meta/files/spdx-licenses.json" do_image_complete[depends] = "virtual/kernel:do_create_spdx" +def extract_licenses(filename): + import re + lic_regex = re.compile('SPDX-License-Identifier:\s+([-A-Za-z\d. ]+)[ |\n|\r\n]*?') + + try: + with open(filename, 'r') as f: + size = min(15000, os.stat(filename).st_size) + txt = f.read(size) + licenses = re.findall(lic_regex, txt) + if licenses: + return licenses + except Exception as e: + bb.warn(f"Exception on {filename}: {e}") + return None + def get_doc_namespace(d, doc): import uuid namespace_uuid = uuid.uuid5(uuid.NAMESPACE_DNS, d.getVar("SPDX_UUID_NAMESPACE"))
@@ -232,6 +247,16 @@ def add_package_files(d, doc, spdx_pkg, topdir, get_spdxid, get_types, *, archiv checksumValue=bb.utils.sha256_file(filepath), )) + if "SOURCES" in spdx_file.fileTypes: + licenses = extract_licenses(filepath) + if licenses is not None: + for lic in licenses: + spdx_file.licenseInfoInFiles.append(lic.strip()) + else: + spdx_file.licenseInfoInFiles.append("NOASSERTATION") + else: + spdx_file.licenseInfoInFiles.append("NOASSERTATION") + doc.files.append(spdx_file) doc.add_relationship(spdx_pkg, "CONTAINS", spdx_file) spdx_pkg.hasFiles.append(spdx_file.SPDXID)
lib/oe/spdx.py +1 -1
@@ -236,7 +236,7 @@ class SPDXFile(SPDXObject): fileName = _String() licenseConcluded = _String(default="NOASSERTION") copyrightText = _String(default="NOASSERTION") - licenseInfoInFiles = _StringList(default=["NOASSERTION"]) + licenseInfoInFiles = _StringList() checksums = _ObjectList(SPDXChecksum) fileTypes = _StringList()
Signature · 1 line
2.31.1
Quoted 38 lines · Saul Wold
This patch will read the begining of source files and try to find the SPDX-License-Identifier to populate the licenseInfoInFiles field for each source file. This does not populate licenseConculed at this time, nor rolls it up to package level.
Signed-off-by Saul Wold <[email protected]>--- classes/create-spdx.bbclass | 25 +++++++++++++++++++++++++ lib/oe/spdx.py | 2 +- 2 files changed, 26 insertions(+), 1 deletion(-)classes/create-spdx.bbclass +15
-0@@ -30,6 +30,21 @@ SPDX_LICENSES ??= "${COREBASE}/meta/files/spdx-licenses.json" do_image_complete[depends] = "virtual/kernel:do_create_spdx" +def extract_licenses(filename): + import re + lic_regex = re.compile('SPDX-License-Identifier:\s+([-A-Za-z\d. ]+)[ |\n|\r\n]*?') + + try: + with open(filename, 'r') as f: + size = min(15000, os.stat(filename).st_size) + txt = f.read(size) + licenses = re.findall(lic_regex, txt) + if licenses: + return licenses + except Exception as e: + bb.warn(f"Exception on {filename}: {e}") + return None + def get_doc_namespace(d, doc): import uuid namespace_uuid = uuid.uuid5(uuid.NAMESPACE_DNS, d.getVar("SPDX_UUID_NAMESPACE"))
classes/create-spdx.bbclass def add_package_files(d, doc, spdx_pkg, topdir, get_spdxid, get_types, *, archiv · line 256
Quoted 10 lines
classes/create-spdx.bbclass +7
-0@@ -232,6 +247,16 @@ def add_package_files(d, doc, spdx_pkg, topdir, get_spdxid, get_types, *, archiv checksumValue=bb.utils.sha256_file(filepath), )) + if "SOURCES" in spdx_file.fileTypes: + licenses = extract_licenses(filepath) + if licenses is not None: + for lic in licenses: + spdx_file.licenseInfoInFiles.append(lic.strip()) + else: + spdx_file.licenseInfoInFiles.append("NOASSERTATION")
"NOASSERTION"
classes/create-spdx.bbclass def add_package_files(d, doc, spdx_pkg, topdir, get_spdxid, get_types, *, archiv · line 258
Quoted 2 lines
classes/create-spdx.bbclass +2
-0+ else: + spdx_file.licenseInfoInFiles.append("NOASSERTATION")
"NOASSERTION"
Quoted 9 lines
classes/create-spdx.bbclass +1
-0+ doc.files.append(spdx_file) doc.add_relationship(spdx_pkg, "CONTAINS", spdx_file) spdx_pkg.hasFiles.append(spdx_file.SPDXID)lib/oe/spdx.py +0
-0
lib/oe/spdx.py class SPDXFile(SPDXObject): · line 239
Quoted 5 lines
lib/oe/spdx.py +1
-1@@ -236,7 +236,7 @@ class SPDXFile(SPDXObject): fileName = _String() licenseConcluded = _String(default="NOASSERTION") copyrightText = _String(default="NOASSERTION") - licenseInfoInFiles = _StringList(default=["NOASSERTION"]) + licenseInfoInFiles = _StringList()
It's required to have "NOASSERTION" as the default if you don't do anything, so we shouldn't change the default here (by and large, this file should capture the spec over our use of it).
It's on my TODO list to make the "default" lists behave like default scalars, where appending replaces the default instead of appending to it, but I haven't gotten there yet; it hasn't come up as a problem before.
Probably need to do something like:
license_info_from_file = []
# scan files here if license_info_from_files:
spdx_file.licenseInfoInFiles = license_info_from_files
Quoted 3 lines
lib/oe/spdx.py +0
-0checksums = _ObjectList(SPDXChecksum) fileTypes = _StringList()