create-spdx: Get SPDX-License-Identifier from source
PATCH 1/1 create-spdx: Get SPDX-License-Identifier from source
This patch will read the begining of source files and try to find the SPDX-License-Identifier to populate the licenseInfoInFiles field for each source file. This does not populate licenseConculed at this time, nor rolls it up to package level.
We read as binary to since some source code seem to have some binary characters, the license is then converted to ascii strings.
---
Merge after Joshua's patch (spdx: Add set helper for list properties) merges
meta/classes/create-spdx.bbclass | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+)
meta/classes/create-spdx.bbclass +23 -0
@@ -37,6 +37,24 @@ SPDX_SUPPLIER[doc] = "The SPDX PackageSupplier field for SPDX packages created f do_image_complete[depends] = "virtual/kernel:do_create_spdx" +def extract_licenses(filename): + import re + import oe.spdx + + lic_regex = re.compile(b'SPDX-License-Identifier:\s+([-A-Za-z\d. ]+)[ |\n|\r\n]*?') + + try: + with open(filename, 'rb') as f: + size = min(15000, os.stat(filename).st_size) + txt = f.read(size) + licenses = re.findall(lic_regex, txt) + if licenses: + ascii_licenses = [lic.decode('ascii') for lic in licenses] + return ascii_licenses + except Exception as e: + bb.warn(f"Exception reading {filename}: {e}") + return None + def get_doc_namespace(d, doc): import uuid namespace_uuid = uuid.uuid5(uuid.NAMESPACE_DNS, d.getVar("SPDX_UUID_NAMESPACE"))
@@ -232,6 +250,11 @@ def add_package_files(d, doc, spdx_pkg, topdir, get_spdxid, get_types, *, archiv checksumValue=bb.utils.sha256_file(filepath), )) + if "SOURCE" in spdx_file.fileTypes: + extracted_lics = extract_licenses(filepath) + if extracted_lics: + spdx_file.licenseInfoInFiles = extracted_lics + doc.files.append(spdx_file) doc.add_relationship(spdx_pkg, "CONTAINS", spdx_file) spdx_pkg.hasFiles.append(spdx_file.SPDXID)
Signature · 1 line
2.31.1
Quoted 10 lines
-----Original Message-----
From: [email protected] <[email protected]> On Behalf Of Saul Wold
Sent: den 2 februari 2022 01:02
To: [email protected]; [email protected]
Cc: Saul Wold <[email protected]>
Subject: [OE-core] [PATCH] create-spdx: Get SPDX-License-Identifier from sourceThis patch will read the begining of source files and try to find the SPDX-License-Identifier to populate the licenseInfoInFiles field for each source file. This does not populate licenseConculed
I assume that should be "licenseConcluded".
Quoted 3 lines
at this time, nor rolls it up to package level.
We read as binary to since some source code seem to have some
to -> too
Quoted 15 lines
binary characters, the license is then converted to ascii strings.
Signed-off-by Saul Wold <[email protected]>---
Merge after Joshua's patch (spdx: Add set helper for list properties) mergesmeta/classes/create-spdx.bbclass | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+)meta/classes/create-spdx.bbclass +0
-0
meta/classes/create-spdx.bbclass SPDX_SUPPLIER[doc] = "The SPDX PackageSupplier field for SPDX packages created f · line 42
Quoted 6 lines
meta/classes/create-spdx.bbclass +3
-0@@ -37,6 +37,24 @@ SPDX_SUPPLIER[doc] = "The SPDX PackageSupplier field for SPDX packages created f do_image_complete[depends] = "virtual/kernel:do_create_spdx" +def extract_licenses(filename): + import re + import oe.spdx
You do not use oe.spdx in this function.
meta/classes/create-spdx.bbclass SPDX_SUPPLIER[doc] = "The SPDX PackageSupplier field for SPDX packages created f · line 44
Quoted 2 lines
meta/classes/create-spdx.bbclass +2
-0+ + lic_regex = re.compile(b'SPDX-License-Identifier:\s+([-A-Za-z\d. ]+)[ |\n|\r\n]*?')
I assume you meant:
lic_regex = re.compile(b'SPDX-License-Identifier:\s+([-A-Za-z\d. ]+)(?: |\n|\r\n)*?')
Not that it really matters though, as it will yield the same result as:
lic_regex = re.compile(b'SPDX-License-Identifier:\s+([-A-Za-z\d. ]+)')
However, neither of the expressions above will correctly match all the SPDX-License-Identifier examples at https://spdx.dev/ids/#how.
Use this instead:
lic_regex = re.compile(b'^\W*SPDX-License-Identifier:\s*([ \w\d.()+-]+?)(?:\s+\W*)?$', re.MULTILINE)
Quoted 29 lines
meta/classes/create-spdx.bbclass +18
-1+ + try: + with open(filename, 'rb') as f: + size = min(15000, os.stat(filename).st_size) + txt = f.read(size) + licenses = re.findall(lic_regex, txt) + if licenses: + ascii_licenses = [lic.decode('ascii') for lic in licenses] + return ascii_licenses + except Exception as e: + bb.warn(f"Exception reading {filename}: {e}") + return None + def get_doc_namespace(d, doc): import uuid namespace_uuid = uuid.uuid5(uuid.NAMESPACE_DNS, d.getVar("SPDX_UUID_NAMESPACE"))@@ -232,6 +250,11 @@ def add_package_files(d, doc, spdx_pkg, topdir, get_spdxid, get_types, *, archiv checksumValue=bb.utils.sha256_file(filepath), )) + if "SOURCE" in spdx_file.fileTypes: + extracted_lics = extract_licenses(filepath) + if extracted_lics: + spdx_file.licenseInfoInFiles = extracted_lics + doc.files.append(spdx_file) doc.add_relationship(spdx_pkg, "CONTAINS", spdx_file) spdx_pkg.hasFiles.append(spdx_file.SPDXID) --
Quoted 1 line
2.31.1
//Peter
Quoted 7 lines · Peter Kjellerstedt
Quoted 7 lines
-----Original Message-----
From: [email protected] <[email protected]> On Behalf Of Saul Wold
Sent: den 2 februari 2022 01:02
To: [email protected]; [email protected]
Cc: Saul Wold <[email protected]>
Subject: [OE-core] [PATCH] create-spdx: Get SPDX-License-Identifier from source
Quoted 6 lines
Quoted 3 lines
This patch will read the begining of source files and try to find the SPDX-License-Identifier to populate the licenseInfoInFiles field for each source file. This does not populate licenseConculed
I assume that should be "licenseConcluded".
Well that depends on if "we" want to take some "ownership" of the conclusion as the "preparer". How would we handle the case of 2 SPDX-License-Identifiers tags in a file, is it an "AND" or an "OR"? Simple example.
The description of licenseConcluded is:
"License expression for licenseConcluded. The licensing that the preparer of this SPDX document has concluded, based on the evidence, actually applies to the package."
At somepoint we might be able to fill in that field, but for now I think we leave it as NOASSERTION.
Sau!
Quoted 81 lines
Quoted 3 lines
at this time, nor rolls it up to package level.
We read as binary to since some source code seem to have some
to -> too
Quoted 15 lines
binary characters, the license is then converted to ascii strings.
Signed-off-by Saul Wold <[email protected]>---
Merge after Joshua's patch (spdx: Add set helper for list properties) mergesmeta/classes/create-spdx.bbclass | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+)meta/classes/create-spdx.bbclass +0
-0meta/classes/create-spdx.bbclass SPDX_SUPPLIER[doc] = "The SPDX PackageSupplier field for SPDX packages created f · line 42
Quoted 6 lines
meta/classes/create-spdx.bbclass +3
-0@@ -37,6 +37,24 @@ SPDX_SUPPLIER[doc] = "The SPDX PackageSupplier field for SPDX packages created f do_image_complete[depends] = "virtual/kernel:do_create_spdx" +def extract_licenses(filename): + import re + import oe.spdxYou do not use oe.spdx in this function.
meta/classes/create-spdx.bbclass SPDX_SUPPLIER[doc] = "The SPDX PackageSupplier field for SPDX packages created f · line 44
Quoted 2 lines
meta/classes/create-spdx.bbclass +2
-0+ + lic_regex = re.compile(b'SPDX-License-Identifier:\s+([-A-Za-z\d. ]+)[ |\n|\r\n]*?')I assume you meant:
lic_regex = re.compile(b'SPDX-License-Identifier:\s+([-A-Za-z\d. ]+)(?: |\n|\r\n)*?')Not that it really matters though, as it will yield the same result as:
lic_regex = re.compile(b'SPDX-License-Identifier:\s+([-A-Za-z\d. ]+)')However, neither of the expressions above will correctly match all the SPDX-License-Identifier examples at https://spdx.dev/ids/#how.
Use this instead:
lic_regex = re.compile(b'^\W*SPDX-License-Identifier:\s*([ \w\d.()+-]+?)(?:\s+\W*)?$', re.MULTILINE)Quoted 29 lines
meta/classes/create-spdx.bbclass +18
-1+ + try: + with open(filename, 'rb') as f: + size = min(15000, os.stat(filename).st_size) + txt = f.read(size) + licenses = re.findall(lic_regex, txt) + if licenses: + ascii_licenses = [lic.decode('ascii') for lic in licenses] + return ascii_licenses + except Exception as e: + bb.warn(f"Exception reading {filename}: {e}") + return None + def get_doc_namespace(d, doc): import uuid namespace_uuid = uuid.uuid5(uuid.NAMESPACE_DNS, d.getVar("SPDX_UUID_NAMESPACE"))@@ -232,6 +250,11 @@ def add_package_files(d, doc, spdx_pkg, topdir, get_spdxid, get_types, *, archiv checksumValue=bb.utils.sha256_file(filepath), )) + if "SOURCE" in spdx_file.fileTypes: + extracted_lics = extract_licenses(filepath) + if extracted_lics: + spdx_file.licenseInfoInFiles = extracted_lics + doc.files.append(spdx_file) doc.add_relationship(spdx_pkg, "CONTAINS", spdx_file) spdx_pkg.hasFiles.append(spdx_file.SPDXID) --Quoted 1 line
2.31.1
//Peter
Signature · 1 line
Sau!
Quoted 31 lines
-----Original Message-----
From: Saul Wold <[email protected]>
Sent: den 2 februari 2022 05:07
To: Peter Kjellerstedt <[email protected]>; openembedded- [email protected]; [email protected]
Subject: Re: [OE-core] [PATCH] create-spdx: Get SPDX-License-Identifier from sourceQuoted 2 lines · Peter Kjellerstedt
Quoted 2 lines
-----Original Message-----
From: [email protected] <openembedded-[email protected]> On Behalf Of Saul Wold
Quoted 4 lines
Quoted 4 lines
Sent: den 2 februari 2022 01:02
To: [email protected]; [email protected]
Cc: Saul Wold <[email protected]>
Subject: [OE-core] [PATCH] create-spdx: Get SPDX-License-Identifierfrom source
Quoted 6 lines
Quoted 4 lines
This patch will read the begining of source files and try to find the SPDX-License-Identifier to populate the licenseInfoInFiles field for each source file. This does not populate licenseConculed
I assume that should be "licenseConcluded".
Well that depends on if "we" want to take some "ownership" of the conclusion as the "preparer". How would we handle the case of 2 SPDX-License-Identifiers tags in a file, is it an "AND" or an "OR"? Simple example.
The description of licenseConcluded is:
Quoted 8 lines
"License expression for licenseConcluded. The licensing that the preparer of this SPDX document has concluded, based on the evidence, actually applies to the package."
At somepoint we might be able to fill in that field, but for now I think we leave it as NOASSERTION.
Sau!
Sorry, you misunderstood. Since I do not know the specification, I could only assume that the field you intended to refer to was actually named "licenseConcluded" rather than "licenseConculed".
//Peter
Quoted 93 lines
Quoted 13 lines
Quoted 3 lines
at this time, nor rolls it up to package level.
We read as binary to since some source code seem to have some
to -> too
Quoted 7 lines
binary characters, the license is then converted to ascii strings.
Signed-off-by Saul Wold <[email protected]>---
Merge after Joshua's patch (spdx: Add set helper for list properties) mergesQuoted 4 lines
meta/classes/create-
Quoted 4 lines
meta/classes/create-spdx.bbclass | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+)meta/classes/create- renamed from meta/classes/create-spdx.bbclass +0
-0spdx.bbclass
Quoted 4 lines
meta/classes/create-spdx.bbclass SPDX_SUPPLIER[doc] = "The SPDX PackageSupplier field
Quoted 4 lines
index
8b4203fdb5d..588489cc2b0100644meta/classes/create-spdx.bbclass +0
-0@@ -37,6 +37,24 @@ SPDX_SUPPLIER[doc] = "The SPDX PackageSupplier fieldfor SPDX packages created f
Quoted 11 lines
meta/classes/create-spdx.bbclass SPDX_SUPPLIER[doc] = "The SPDX PackageSupplier field for SPDX packages created f · line 42
Quoted 6 lines
meta/classes/create-spdx.bbclass +3
-0do_image_complete[depends] = "virtual/kernel:do_create_spdx" +def extract_licenses(filename): + import re + import oe.spdxYou do not use oe.spdx in this function.
Quoted 2 lines
meta/classes/create-spdx.bbclass +2
-0+ + lic_regex = re.compile(b'SPDX-License-Identifier:\s+([-A-Za-z\d.]+)[ |\n|\r\n]*?')
Quoted 4 lines
I assume you meant:
lic_regex = re.compile(b'SPDX-License-Identifier:\s+([-A-Za-z\d.]+)(?: |\n|\r\n)*?')
Quoted 4 lines
Not that it really matters though, as it will yield the same result as:
lic_regex = re.compile(b'SPDX-License-Identifier:\s+([-A-Za-z\d.]+)')
Quoted 7 lines
However, neither of the expressions above will correctly match all the SPDX-License-Identifier examples at https://spdx.dev/ids/#how.
Use this instead:
lic_regex = re.compile(b'^\W*SPDX-License-Identifier:\s*([\w\d.()+-]+?)(?:\s+\W*)?$', re.MULTILINE)
Quoted 9 lines
meta/classes/create-spdx.bbclass SPDX_SUPPLIER[doc] = "The SPDX PackageSupplier field for SPDX packages created f
Quoted 9 lines
meta/classes/create-spdx.bbclass +8
-0+ + try: + with open(filename, 'rb') as f: + size = min(15000, os.stat(filename).st_size) + txt = f.read(size) + licenses = re.findall(lic_regex, txt) + if licenses: + ascii_licenses = [lic.decode('ascii') for lic inlicenses]
Quoted 8 lines
meta/classes/create-spdx.bbclass SPDX_SUPPLIER[doc] = "The SPDX PackageSupplier field for SPDX packages created f
Quoted 8 lines
meta/classes/create-spdx.bbclass +5
-0+ return ascii_licenses + except Exception as e: + bb.warn(f"Exception reading {filename}: {e}") + return None + def get_doc_namespace(d, doc): import uuid namespace_uuid = uuid.uuid5(uuid.NAMESPACE_DNS,
d.getVar("SPDX_UUID_NAMESPACE"))Quoted 1 line
meta/classes/create-spdx.bbclass def add_package_files(d, doc, spdx_pkg, topdir,
Quoted 1 line
meta/classes/create-spdx.bbclass +0
-0@@ -232,6 +250,11 @@ def add_package_files(d, doc, spdx_pkg, topdir,
get_spdxid,get_types, *, archivQuoted 14 lines
Quoted 12 lines
meta/classes/create-spdx.bbclass +5
-1checksumValue=bb.utils.sha256_file(filepath), )) + if "SOURCE" in spdx_file.fileTypes: + extracted_lics = extract_licenses(filepath) + if extracted_lics: + spdx_file.licenseInfoInFiles = extracted_lics + doc.files.append(spdx_file) doc.add_relationship(spdx_pkg, "CONTAINS", spdx_file) spdx_pkg.hasFiles.append(spdx_file.SPDXID) --Quoted 1 line
2.31.1
Quoted 2 lines
//Peter
--
Sau!