Skip to contentSkip to conversation

create-spdx: Get SPDX-License-Identifier from source

PATCH Superseded Saul Wold 2 participants 4 messages Feb 2, 2022

v1 of 3 Superseded Latest is v3 →

lore
Share & tools
lore

Saul Wold

author
#0 lore raw

PATCH 1/1 create-spdx: Get SPDX-License-Identifier from source

This patch will read the begining of source files and try to find the SPDX-License-Identifier to populate the licenseInfoInFiles field for each source file. This does not populate licenseConculed at this time, nor rolls it up to package level.

We read as binary to since some source code seem to have some binary characters, the license is then converted to ascii strings.

---
Merge after Joshua's patch (spdx: Add set helper for list properties) merges

 meta/classes/create-spdx.bbclass | 23 +++++++++++++++++++++++
 1 file changed, 23 insertions(+)
meta/classes/create-spdx.bbclass +23 -0
index 8b4203fdb5d..588489cc2b0 100644
@@ -37,6 +37,24 @@ SPDX_SUPPLIER[doc] = "The SPDX PackageSupplier field for SPDX packages created f
 
 do_image_complete[depends] = "virtual/kernel:do_create_spdx"
 
+def extract_licenses(filename):
+    import re
+    import oe.spdx
+
+    lic_regex = re.compile(b'SPDX-License-Identifier:\s+([-A-Za-z\d. ]+)[ |\n|\r\n]*?')
+
+    try:
+        with open(filename, 'rb') as f:
+            size = min(15000, os.stat(filename).st_size)
+            txt = f.read(size)
+            licenses = re.findall(lic_regex, txt)
+            if licenses:
+                ascii_licenses = [lic.decode('ascii') for lic in licenses]
+                return ascii_licenses
+    except Exception as e:
+        bb.warn(f"Exception reading {filename}: {e}")
+    return None
+
 def get_doc_namespace(d, doc):
     import uuid
     namespace_uuid = uuid.uuid5(uuid.NAMESPACE_DNS, d.getVar("SPDX_UUID_NAMESPACE"))
@@ -232,6 +250,11 @@ def add_package_files(d, doc, spdx_pkg, topdir, get_spdxid, get_types, *, archiv
                         checksumValue=bb.utils.sha256_file(filepath),
                     ))
 
+                if "SOURCE" in spdx_file.fileTypes:
+                    extracted_lics = extract_licenses(filepath)
+                    if extracted_lics:
+                        spdx_file.licenseInfoInFiles = extracted_lics
+
                 doc.files.append(spdx_file)
                 doc.add_relationship(spdx_pkg, "CONTAINS", spdx_file)
                 spdx_pkg.hasFiles.append(spdx_file.SPDXID)
Signature · 1 line
2.31.1
#1 lore raw
Quoted 10 lines

-----Original Message-----
From: [email protected] <[email protected]> On Behalf Of Saul Wold
Sent: den 2 februari 2022 01:02
To: [email protected]; [email protected]
Cc: Saul Wold <[email protected]>
Subject: [OE-core] [PATCH] create-spdx: Get SPDX-License-Identifier from source

This patch will read the begining of source files and try to find the SPDX-License-Identifier to populate the licenseInfoInFiles field for each source file. This does not populate licenseConculed

I assume that should be "licenseConcluded".

Quoted 3 lines

at this time, nor rolls it up to package level.

We read as binary to since some source code seem to have some

to -> too

Quoted 15 lines

binary characters, the license is then converted to ascii strings.

---
Merge after Joshua's patch (spdx: Add set helper for list properties) merges

 meta/classes/create-spdx.bbclass | 23 +++++++++++++++++++++++
 1 file changed, 23 insertions(+)
meta/classes/create-spdx.bbclass +0 -0
index 8b4203fdb5d..588489cc2b0 100644

meta/classes/create-spdx.bbclass SPDX_SUPPLIER[doc] = "The SPDX PackageSupplier field for SPDX packages created f · line 42

Quoted 6 lines
meta/classes/create-spdx.bbclass +3 -0
@@ -37,6 +37,24 @@ SPDX_SUPPLIER[doc] = "The SPDX PackageSupplier field for SPDX packages created f
 
 do_image_complete[depends] = "virtual/kernel:do_create_spdx"
 
+def extract_licenses(filename):
+    import re
+    import oe.spdx

You do not use oe.spdx in this function.

meta/classes/create-spdx.bbclass SPDX_SUPPLIER[doc] = "The SPDX PackageSupplier field for SPDX packages created f · line 44

Quoted 2 lines
meta/classes/create-spdx.bbclass +2 -0
+
+    lic_regex = re.compile(b'SPDX-License-Identifier:\s+([-A-Za-z\d. ]+)[ |\n|\r\n]*?')

I assume you meant:

    lic_regex = re.compile(b'SPDX-License-Identifier:\s+([-A-Za-z\d. ]+)(?: |\n|\r\n)*?')

Not that it really matters though, as it will yield the same result as:

    lic_regex = re.compile(b'SPDX-License-Identifier:\s+([-A-Za-z\d. ]+)')

However, neither of the expressions above will correctly match all the SPDX-License-Identifier examples at https://spdx.dev/ids/#how.

Use this instead:

    lic_regex = re.compile(b'^\W*SPDX-License-Identifier:\s*([ \w\d.()+-]+?)(?:\s+\W*)?$', re.MULTILINE)
Quoted 29 lines
meta/classes/create-spdx.bbclass +18 -1
+
+    try:
+        with open(filename, 'rb') as f:
+            size = min(15000, os.stat(filename).st_size)
+            txt = f.read(size)
+            licenses = re.findall(lic_regex, txt)
+            if licenses:
+                ascii_licenses = [lic.decode('ascii') for lic in licenses]
+                return ascii_licenses
+    except Exception as e:
+        bb.warn(f"Exception reading {filename}: {e}")
+    return None
+
 def get_doc_namespace(d, doc):
     import uuid
     namespace_uuid = uuid.uuid5(uuid.NAMESPACE_DNS, d.getVar("SPDX_UUID_NAMESPACE"))
@@ -232,6 +250,11 @@ def add_package_files(d, doc, spdx_pkg, topdir, get_spdxid, get_types, *, archiv
                         checksumValue=bb.utils.sha256_file(filepath),
                     ))
 
+                if "SOURCE" in spdx_file.fileTypes:
+                    extracted_lics = extract_licenses(filepath)
+                    if extracted_lics:
+                        spdx_file.licenseInfoInFiles = extracted_lics
+
                 doc.files.append(spdx_file)
                 doc.add_relationship(spdx_pkg, "CONTAINS", spdx_file)
                 spdx_pkg.hasFiles.append(spdx_file.SPDXID)
--
Quoted 1 line

2.31.1

//Peter

author
#2 lore raw
Quoted 7 lines · Peter Kjellerstedt
Quoted 7 lines

-----Original Message-----
From: [email protected] <[email protected]> On Behalf Of Saul Wold
Sent: den 2 februari 2022 01:02
To: [email protected]; [email protected]
Cc: Saul Wold <[email protected]>
Subject: [OE-core] [PATCH] create-spdx: Get SPDX-License-Identifier from source

Quoted 6 lines
Quoted 3 lines

This patch will read the begining of source files and try to find the SPDX-License-Identifier to populate the licenseInfoInFiles field for each source file. This does not populate licenseConculed

I assume that should be "licenseConcluded".

Well that depends on if "we" want to take some "ownership" of the conclusion as the "preparer". How would we handle the case of 2 SPDX-License-Identifiers tags in a file, is it an "AND" or an "OR"? Simple example.

The description of licenseConcluded is:

"License expression for licenseConcluded. The licensing that the preparer of this SPDX document has concluded, based on the evidence, actually applies to the package."

At somepoint we might be able to fill in that field, but for now I think we leave it as NOASSERTION.

Sau!

Quoted 81 lines
Quoted 3 lines

at this time, nor rolls it up to package level.

We read as binary to since some source code seem to have some

to -> too

Quoted 15 lines

binary characters, the license is then converted to ascii strings.

---
Merge after Joshua's patch (spdx: Add set helper for list properties) merges

  meta/classes/create-spdx.bbclass | 23 +++++++++++++++++++++++
  1 file changed, 23 insertions(+)
meta/classes/create-spdx.bbclass +0 -0
index 8b4203fdb5d..588489cc2b0 100644

meta/classes/create-spdx.bbclass SPDX_SUPPLIER[doc] = "The SPDX PackageSupplier field for SPDX packages created f · line 42

Quoted 6 lines
meta/classes/create-spdx.bbclass +3 -0
@@ -37,6 +37,24 @@ SPDX_SUPPLIER[doc] = "The SPDX PackageSupplier field for SPDX packages created f
 
  do_image_complete[depends] = "virtual/kernel:do_create_spdx"
 
+def extract_licenses(filename):
+    import re
+    import oe.spdx

You do not use oe.spdx in this function.

meta/classes/create-spdx.bbclass SPDX_SUPPLIER[doc] = "The SPDX PackageSupplier field for SPDX packages created f · line 44

Quoted 2 lines
meta/classes/create-spdx.bbclass +2 -0
+
+    lic_regex = re.compile(b'SPDX-License-Identifier:\s+([-A-Za-z\d. ]+)[ |\n|\r\n]*?')

I assume you meant:

     lic_regex = re.compile(b'SPDX-License-Identifier:\s+([-A-Za-z\d. ]+)(?: |\n|\r\n)*?')

Not that it really matters though, as it will yield the same result as:

     lic_regex = re.compile(b'SPDX-License-Identifier:\s+([-A-Za-z\d. ]+)')

However, neither of the expressions above will correctly match all the SPDX-License-Identifier examples at https://spdx.dev/ids/#how.

Use this instead:

     lic_regex = re.compile(b'^\W*SPDX-License-Identifier:\s*([ \w\d.()+-]+?)(?:\s+\W*)?$', re.MULTILINE)
Quoted 29 lines
meta/classes/create-spdx.bbclass +18 -1
+
+    try:
+        with open(filename, 'rb') as f:
+            size = min(15000, os.stat(filename).st_size)
+            txt = f.read(size)
+            licenses = re.findall(lic_regex, txt)
+            if licenses:
+                ascii_licenses = [lic.decode('ascii') for lic in licenses]
+                return ascii_licenses
+    except Exception as e:
+        bb.warn(f"Exception reading {filename}: {e}")
+    return None
+
  def get_doc_namespace(d, doc):
      import uuid
      namespace_uuid = uuid.uuid5(uuid.NAMESPACE_DNS, d.getVar("SPDX_UUID_NAMESPACE"))
@@ -232,6 +250,11 @@ def add_package_files(d, doc, spdx_pkg, topdir, get_spdxid, get_types, *, archiv
                          checksumValue=bb.utils.sha256_file(filepath),
                      ))
 
+                if "SOURCE" in spdx_file.fileTypes:
+                    extracted_lics = extract_licenses(filepath)
+                    if extracted_lics:
+                        spdx_file.licenseInfoInFiles = extracted_lics
+
                  doc.files.append(spdx_file)
                  doc.add_relationship(spdx_pkg, "CONTAINS", spdx_file)
                  spdx_pkg.hasFiles.append(spdx_file.SPDXID)
--
Quoted 1 line

2.31.1

//Peter

Signature · 1 line
Sau!
#3 lore raw
Quoted 31 lines

-----Original Message-----
From: Saul Wold <[email protected]>
Sent: den 2 februari 2022 05:07
To: Peter Kjellerstedt <[email protected]>; openembedded- [email protected]; [email protected]
Subject: Re: [OE-core] [PATCH] create-spdx: Get SPDX-License-Identifier from source

Quoted 2 lines · Peter Kjellerstedt
Quoted 2 lines

-----Original Message-----
From: [email protected] <openembedded-

[email protected]> On Behalf Of Saul Wold

Quoted 4 lines
Quoted 4 lines

Sent: den 2 februari 2022 01:02
To: [email protected]; [email protected]
Cc: Saul Wold <[email protected]>
Subject: [OE-core] [PATCH] create-spdx: Get SPDX-License-Identifier

from source

Quoted 6 lines
Quoted 4 lines

This patch will read the begining of source files and try to find the SPDX-License-Identifier to populate the licenseInfoInFiles field for each source file. This does not populate licenseConculed

I assume that should be "licenseConcluded".

Well that depends on if "we" want to take some "ownership" of the conclusion as the "preparer". How would we handle the case of 2 SPDX-License-Identifiers tags in a file, is it an "AND" or an "OR"? Simple example.

The description of licenseConcluded is:

Quoted 8 lines

"License expression for licenseConcluded. The licensing that the preparer of this SPDX document has concluded, based on the evidence, actually applies to the package."

At somepoint we might be able to fill in that field, but for now I think we leave it as NOASSERTION.

Sau!

Sorry, you misunderstood. Since I do not know the specification, I could only assume that the field you intended to refer to was actually named "licenseConcluded" rather than "licenseConculed".

//Peter

Quoted 93 lines
Quoted 13 lines
Quoted 3 lines

at this time, nor rolls it up to package level.

We read as binary to since some source code seem to have some

to -> too

Quoted 7 lines

binary characters, the license is then converted to ascii strings.

---
Merge after Joshua's patch (spdx: Add set helper for list properties) merges

Quoted 4 lines

meta/classes/create-

Quoted 4 lines
  meta/classes/create-spdx.bbclass | 23 +++++++++++++++++++++++
  1 file changed, 23 insertions(+)
meta/classes/create- renamed from meta/classes/create-spdx.bbclass +0 -0

spdx.bbclass

Quoted 4 lines

meta/classes/create-spdx.bbclass SPDX_SUPPLIER[doc] = "The SPDX PackageSupplier field

Quoted 4 lines

index 8b4203fdb5d..588489cc2b0 100644

meta/classes/create-spdx.bbclass +0 -0
@@ -37,6 +37,24 @@ SPDX_SUPPLIER[doc] = "The SPDX PackageSupplier field

for SPDX packages created f

Quoted 11 lines

meta/classes/create-spdx.bbclass SPDX_SUPPLIER[doc] = "The SPDX PackageSupplier field for SPDX packages created f · line 42

Quoted 6 lines
meta/classes/create-spdx.bbclass +3 -0
  do_image_complete[depends] = "virtual/kernel:do_create_spdx"
 
+def extract_licenses(filename):
+    import re
+    import oe.spdx

You do not use oe.spdx in this function.

Quoted 2 lines
meta/classes/create-spdx.bbclass +2 -0
+
+    lic_regex = re.compile(b'SPDX-License-Identifier:\s+([-A-Za-z\d.

]+)[ |\n|\r\n]*?')

Quoted 4 lines

I assume you meant:

     lic_regex = re.compile(b'SPDX-License-Identifier:\s+([-A-Za-z\d.

]+)(?: |\n|\r\n)*?')

Quoted 4 lines

Not that it really matters though, as it will yield the same result as:

     lic_regex = re.compile(b'SPDX-License-Identifier:\s+([-A-Za-z\d.

]+)')

Quoted 7 lines

However, neither of the expressions above will correctly match all the SPDX-License-Identifier examples at https://spdx.dev/ids/#how.

Use this instead:

     lic_regex = re.compile(b'^\W*SPDX-License-Identifier:\s*([

\w\d.()+-]+?)(?:\s+\W*)?$', re.MULTILINE)

Quoted 9 lines

meta/classes/create-spdx.bbclass SPDX_SUPPLIER[doc] = "The SPDX PackageSupplier field for SPDX packages created f

Quoted 9 lines
meta/classes/create-spdx.bbclass +8 -0
+
+    try:
+        with open(filename, 'rb') as f:
+            size = min(15000, os.stat(filename).st_size)
+            txt = f.read(size)
+            licenses = re.findall(lic_regex, txt)
+            if licenses:
+                ascii_licenses = [lic.decode('ascii') for lic in

licenses]

Quoted 8 lines

meta/classes/create-spdx.bbclass SPDX_SUPPLIER[doc] = "The SPDX PackageSupplier field for SPDX packages created f

Quoted 8 lines
meta/classes/create-spdx.bbclass +5 -0
+                return ascii_licenses
+    except Exception as e:
+        bb.warn(f"Exception reading {filename}: {e}")
+    return None
+
  def get_doc_namespace(d, doc):
      import uuid
      namespace_uuid = uuid.uuid5(uuid.NAMESPACE_DNS,

d.getVar("SPDX_UUID_NAMESPACE"))

Quoted 1 line

meta/classes/create-spdx.bbclass def add_package_files(d, doc, spdx_pkg, topdir,

Quoted 1 line
meta/classes/create-spdx.bbclass +0 -0
@@ -232,6 +250,11 @@ def add_package_files(d, doc, spdx_pkg, topdir,

get_spdxid, get_types, *, archiv

Quoted 14 lines
Quoted 12 lines
meta/classes/create-spdx.bbclass +5 -1
                          checksumValue=bb.utils.sha256_file(filepath),
                      ))
 
+                if "SOURCE" in spdx_file.fileTypes:
+                    extracted_lics = extract_licenses(filepath)
+                    if extracted_lics:
+                        spdx_file.licenseInfoInFiles = extracted_lics
+
                  doc.files.append(spdx_file)
                  doc.add_relationship(spdx_pkg, "CONTAINS", spdx_file)
                  spdx_pkg.hasFiles.append(spdx_file.SPDXID)
--
Quoted 1 line

2.31.1

Quoted 2 lines

//Peter

--
Sau!