Skip to contentSkip to conversation

recipetool/create: Scan for SDPX-License-Identifier

PATCH In review Saul Wold 3 participants 8 messages Feb 3, 2022 – Feb 4, 2022

lore
Share & tools
lore

Saul Wold

author
#0 lore raw

PATCH 1/1 recipetool/create: Scan for SDPX-License-Identifier

When a file can not be identified by checksum and they contain an SPDX License-Identifier tag, use it as the found license.

[YOCTO #14529]

Tested with LICENSE files that contain 1 or more SPDX-License-Identifier tags

---
 scripts/lib/recipetool/create.py | 16 +++++++++++-----
 1 file changed, 11 insertions(+), 5 deletions(-)
scripts/lib/recipetool/create.py +11 -5
index 507a230511..9149c2d94f 100644
@@ -1221,14 +1221,20 @@ def guess_license(srctree, d):
     for licfile in sorted(licfiles):
         md5value = bb.utils.md5_file(licfile)
         license = md5sums.get(md5value, None)
+        license_list = []
         if not license:
             license, crunched_md5, lictext = crunch_license(licfile)
             if lictext and not license:
-                license = 'Unknown'
-                logger.info("Please add the following line for '%s' to a 'lib/recipetool/licenses.csv' " \
-                    "and replace `Unknown` with the license:\n" \
-                    "%s,Unknown" % (os.path.relpath(licfile, srctree), md5value))
-        if license:
+                spdx_re = re.compile('SPDX-License-Identifier:\s+([-A-Za-z\d. ]+)[ |\n|\r\n]*?')
+                license_list = re.findall(spdx_re, "\n".join(lictext))
+                if not license_list:
+                    license_list.append('Unknown')
+                    logger.info("Please add the following line for '%s' to a 'lib/recipetool/licenses.csv' " \
+                        "and replace `Unknown` with the license:\n" \
+                        "%s,Unknown" % (os.path.relpath(licfile, srctree), md5value))
+        else:
+            license_list.append(license)
+        for license in license_list:
             licenses.append((license, os.path.relpath(licfile, srctree), md5value))
 
     # FIXME should we grab at least one source file with a license header and add that too?
Signature · 1 line
2.25.1
#1 lore raw
Quoted 32 lines · Saul Wold

When a file can not be identified by checksum and they contain an SPDX License-Identifier tag, use it as the found license.

[YOCTO #14529]

Tested with LICENSE files that contain 1 or more SPDX-License-Identifier tags

---
 scripts/lib/recipetool/create.py | 16 +++++++++++-----
 1 file changed, 11 insertions(+), 5 deletions(-)
scripts/lib/recipetool/create.py +4 -5
index 507a230511..9149c2d94f 100644
@@ -1221,14 +1221,20 @@ def guess_license(srctree, d):
     for licfile in sorted(licfiles):
         md5value = bb.utils.md5_file(licfile)
         license = md5sums.get(md5value, None)
+        license_list = []
         if not license:
             license, crunched_md5, lictext = crunch_license(licfile)
             if lictext and not license:
-                license = 'Unknown'
-                logger.info("Please add the following line for '%s' to a 'lib/recipetool/licenses.csv' " \
-                    "and replace `Unknown` with the license:\n" \
-                    "%s,Unknown" % (os.path.relpath(licfile, srctree), md5value))
-        if license:
+                spdx_re = re.compile('SPDX-License-Identifier:\s+([-A-Za-z\d. ]+)[ |\n|\r\n]*?')
+                license_list = re.findall(spdx_re, "\n".join(lictext))
+                if not license_list:

scripts/lib/recipetool/create.py def guess_license(srctree, d): · line 1240

Quoted 10 lines
scripts/lib/recipetool/create.py +7 -0
+                    license_list.append('Unknown')
+                    logger.info("Please add the following line for '%s' to a 'lib/recipetool/licenses.csv' " \
+                        "and replace `Unknown` with the license:\n" \
+                        "%s,Unknown" % (os.path.relpath(licfile, srctree), md5value))
+        else:
+            license_list.append(license)
+        for license in license_list:
             licenses.append((license, os.path.relpath(licfile, srctree), md5value))
 
     # FIXME should we grab at least one source file with a license header and add that too?

I think to close this bug the code may need to go one step further and effectively grep over the source tree.

We'd probably want to list the value of any SPDX-License-Identifier: header found in any of the source files for the user to then decide upon?

Or am I misunderstanding?

Cheers,

Richard

author
#2 lore raw
Quoted 42 lines · Richard Purdie
Quoted 41 lines · Saul Wold

When a file can not be identified by checksum and they contain an SPDX License-Identifier tag, use it as the found license.

[YOCTO #14529]

Tested with LICENSE files that contain 1 or more SPDX-License-Identifier tags

---
  scripts/lib/recipetool/create.py | 16 +++++++++++-----
  1 file changed, 11 insertions(+), 5 deletions(-)
scripts/lib/recipetool/create.py +11 -5
index 507a230511..9149c2d94f 100644
@@ -1221,14 +1221,20 @@ def guess_license(srctree, d):
      for licfile in sorted(licfiles):
          md5value = bb.utils.md5_file(licfile)
          license = md5sums.get(md5value, None)
+        license_list = []
          if not license:
              license, crunched_md5, lictext = crunch_license(licfile)
              if lictext and not license:
-                license = 'Unknown'
-                logger.info("Please add the following line for '%s' to a 'lib/recipetool/licenses.csv' " \
-                    "and replace `Unknown` with the license:\n" \
-                    "%s,Unknown" % (os.path.relpath(licfile, srctree), md5value))
-        if license:
+                spdx_re = re.compile('SPDX-License-Identifier:\s+([-A-Za-z\d. ]+)[ |\n|\r\n]*?')
+                license_list = re.findall(spdx_re, "\n".join(lictext))
+                if not license_list:
+                    license_list.append('Unknown')
+                    logger.info("Please add the following line for '%s' to a 'lib/recipetool/licenses.csv' " \
+                        "and replace `Unknown` with the license:\n" \
+                        "%s,Unknown" % (os.path.relpath(licfile, srctree), md5value))
+        else:
+            license_list.append(license)
+        for license in license_list:
              licenses.append((license, os.path.relpath(licfile, srctree), md5value))
 
Quoted 8 lines

scripts/lib/recipetool/create.py def guess_license(srctree, d): · line 1240

Quoted 1 line
scripts/lib/recipetool/create.py +0 -0
      # FIXME should we grab at least one source file with a license header and add that too?

I think to close this bug the code may need to go one step further and effectively grep over the source tree.

We'd probably want to list the value of any SPDX-License-Identifier: header found in any of the source files for the user to then decide upon?

That's moving in to the create-spdx.bbclass territory I think. The change would need to be much larger. and I will likely have to shelve for a while.

Quoted 2 lines

Or am I misunderstanding?

Maybe it's my misunderstanding, Tim has mentioned the LICENSE related files in the bug report.

Sau!

Quoted 8 lines

Cheers,

Richard

Signature · 1 line
Sau!
#3 lore raw
Quoted 49 lines · Saul Wold
Quoted 47 lines · Richard Purdie
Quoted 32 lines · Saul Wold

When a file can not be identified by checksum and they contain an SPDX License-Identifier tag, use it as the found license.

[YOCTO #14529]

Tested with LICENSE files that contain 1 or more SPDX-License-Identifier tags

---
  scripts/lib/recipetool/create.py | 16 +++++++++++-----
  1 file changed, 11 insertions(+), 5 deletions(-)
scripts/lib/recipetool/create.py +4 -5
index 507a230511..9149c2d94f 100644
@@ -1221,14 +1221,20 @@ def guess_license(srctree, d):
      for licfile in sorted(licfiles):
          md5value = bb.utils.md5_file(licfile)
          license = md5sums.get(md5value, None)
+        license_list = []
          if not license:
              license, crunched_md5, lictext = crunch_license(licfile)
              if lictext and not license:
-                license = 'Unknown'
-                logger.info("Please add the following line for '%s' to a 'lib/recipetool/licenses.csv' " \
-                    "and replace `Unknown` with the license:\n" \
-                    "%s,Unknown" % (os.path.relpath(licfile, srctree), md5value))
-        if license:
+                spdx_re = re.compile('SPDX-License-Identifier:\s+([-A-Za-z\d. ]+)[ |\n|\r\n]*?')
+                license_list = re.findall(spdx_re, "\n".join(lictext))
+                if not license_list:

scripts/lib/recipetool/create.py def guess_license(srctree, d): · line 1240

Quoted 10 lines
scripts/lib/recipetool/create.py +7 -0
+                    license_list.append('Unknown')
+                    logger.info("Please add the following line for '%s' to a 'lib/recipetool/licenses.csv' " \
+                        "and replace `Unknown` with the license:\n" \
+                        "%s,Unknown" % (os.path.relpath(licfile, srctree), md5value))
+        else:
+            license_list.append(license)
+        for license in license_list:
              licenses.append((license, os.path.relpath(licfile, srctree), md5value))
 
      # FIXME should we grab at least one source file with a license header and add that too?

I think to close this bug the code may need to go one step further and effectively grep over the source tree.

Quoted 6 lines
Quoted 3 lines

We'd probably want to list the value of any SPDX-License-Identifier: header found in any of the source files for the user to then decide upon?

That's moving in to the create-spdx.bbclass territory I think. The change would need to be much larger. and I will likely have to shelve for a while.

This isn't related to create-spdx.

Quoted 5 lines
Quoted 3 lines

Or am I misunderstanding?

Maybe it's my misunderstanding, Tim has mentioned the LICENSE related files in the bug report.

Right, we want to "guess" what the right LICENSE is for the new recipe. To do that wouldn't we scan all the source for SPDX-License-Identifier: lines in the headers, add those all together and suggest that as the LICENSE field?

Cheers,

Richard

#4 lore raw
In reply to Richard Purdie

Hi Richard,

Am 03.02.2022 um 22:24 schrieb Richard Purdie via lists.openembedded.org:

Quoted 42 lines · Saul Wold
Quoted 41 lines

When a file can not be identified by checksum and they contain an SPDX License-Identifier tag, use it as the found license.

[YOCTO #14529]

Tested with LICENSE files that contain 1 or more SPDX-License-Identifier tags

---
  scripts/lib/recipetool/create.py | 16 +++++++++++-----
  1 file changed, 11 insertions(+), 5 deletions(-)
scripts/lib/recipetool/create.py +11 -5
index 507a230511..9149c2d94f 100644
@@ -1221,14 +1221,20 @@ def guess_license(srctree, d):
      for licfile in sorted(licfiles):
          md5value = bb.utils.md5_file(licfile)
          license = md5sums.get(md5value, None)
+        license_list = []
          if not license:
              license, crunched_md5, lictext = crunch_license(licfile)
              if lictext and not license:
-                license = 'Unknown'
-                logger.info("Please add the following line for '%s' to a 'lib/recipetool/licenses.csv' " \
-                    "and replace `Unknown` with the license:\n" \
-                    "%s,Unknown" % (os.path.relpath(licfile, srctree), md5value))
-        if license:
+                spdx_re = re.compile('SPDX-License-Identifier:\s+([-A-Za-z\d. ]+)[ |\n|\r\n]*?')
+                license_list = re.findall(spdx_re, "\n".join(lictext))
+                if not license_list:
+                    license_list.append('Unknown')
+                    logger.info("Please add the following line for '%s' to a 'lib/recipetool/licenses.csv' " \
+                        "and replace `Unknown` with the license:\n" \
+                        "%s,Unknown" % (os.path.relpath(licfile, srctree), md5value))
+        else:
+            license_list.append(license)
+        for license in license_list:
              licenses.append((license, os.path.relpath(licfile, srctree), md5value))
 
Quoted 4 lines

scripts/lib/recipetool/create.py def guess_license(srctree, d): · line 1240

Quoted 1 line
scripts/lib/recipetool/create.py +0 -0
      # FIXME should we grab at least one source file with a license header and add that too?

I think to close this bug the code may need to go one step further and effectively grep over the source tree.

Please keep in mind that we need a full license text and not only the license name for license compliance. The current function only search for license files with license text.

Quoted 2 lines

We'd probably want to list the value of any SPDX-License-Identifier: header found in any of the source files for the user to then decide upon?

I think this is an other feature like a license checker because if you have a SPDX-License-Identifier without a license text you have a license violation.

This brings us to the problem that this code will interpret a file with only a SPDX-License-Identifier as a license file with license text.

Regards
Stefan

#5 lore raw
Quoted 57 lines · Stefan Herbrechtsmeier

Hi Richard,

Am 03.02.2022 um 22:24 schrieb Richard Purdie via lists.openembedded.org:

Quoted 42 lines · Saul Wold
Quoted 41 lines

When a file can not be identified by checksum and they contain an SPDX License-Identifier tag, use it as the found license.

[YOCTO #14529]

Tested with LICENSE files that contain 1 or more SPDX-License-Identifier tags

---
  scripts/lib/recipetool/create.py | 16 +++++++++++-----
  1 file changed, 11 insertions(+), 5 deletions(-)
scripts/lib/recipetool/create.py +11 -5
index 507a230511..9149c2d94f 100644
@@ -1221,14 +1221,20 @@ def guess_license(srctree, d):
      for licfile in sorted(licfiles):
          md5value = bb.utils.md5_file(licfile)
          license = md5sums.get(md5value, None)
+        license_list = []
          if not license:
              license, crunched_md5, lictext = crunch_license(licfile)
              if lictext and not license:
-                license = 'Unknown'
-                logger.info("Please add the following line for '%s' to a 'lib/recipetool/licenses.csv' " \
-                    "and replace `Unknown` with the license:\n" \
-                    "%s,Unknown" % (os.path.relpath(licfile, srctree), md5value))
-        if license:
+                spdx_re = re.compile('SPDX-License-Identifier:\s+([-A-Za-z\d. ]+)[ |\n|\r\n]*?')
+                license_list = re.findall(spdx_re, "\n".join(lictext))
+                if not license_list:
+                    license_list.append('Unknown')
+                    logger.info("Please add the following line for '%s' to a 'lib/recipetool/licenses.csv' " \
+                        "and replace `Unknown` with the license:\n" \
+                        "%s,Unknown" % (os.path.relpath(licfile, srctree), md5value))
+        else:
+            license_list.append(license)
+        for license in license_list:
              licenses.append((license, os.path.relpath(licfile, srctree), md5value))
 
Quoted 4 lines

scripts/lib/recipetool/create.py def guess_license(srctree, d): · line 1240

Quoted 1 line
scripts/lib/recipetool/create.py +0 -0
      # FIXME should we grab at least one source file with a license header and add that too?

I think to close this bug the code may need to go one step further and effectively grep over the source tree.

Please keep in mind that we need a full license text and not only the license name for license compliance. The current function only search for license files with license text.

Quoted 2 lines

We'd probably want to list the value of any SPDX-License-Identifier: header found in any of the source files for the user to then decide upon?

Quoted 6 lines

I think this is an other feature like a license checker because if you have a SPDX-License-Identifier without a license text you have a license violation.

This brings us to the problem that this code will interpret a file with only a SPDX-License-Identifier as a license file with license text.

As I understand it the tool is there to help write a recipe so filling out LICENSE and highlighting a missing full license text would be a valid approach for the tool and helpful to the user?

It certainly isn't intended as full validation, just intended to assist the creation of a recipe.

Cheers,

Richard

#6 lore raw

Am 04.02.2022 um 14:41 schrieb Richard Purdie:

Quoted 60 lines · Stefan Herbrechtsmeier
Quoted 59 lines

Am 03.02.2022 um 22:24 schrieb Richard Purdie via lists.openembedded.org:

Quoted 42 lines · Saul Wold
Quoted 41 lines

When a file can not be identified by checksum and they contain an SPDX License-Identifier tag, use it as the found license.

[YOCTO #14529]

Tested with LICENSE files that contain 1 or more SPDX-License-Identifier tags

---
   scripts/lib/recipetool/create.py | 16 +++++++++++-----
   1 file changed, 11 insertions(+), 5 deletions(-)
scripts/lib/recipetool/create.py +11 -5
index 507a230511..9149c2d94f 100644
@@ -1221,14 +1221,20 @@ def guess_license(srctree, d):
       for licfile in sorted(licfiles):
           md5value = bb.utils.md5_file(licfile)
           license = md5sums.get(md5value, None)
+        license_list = []
           if not license:
               license, crunched_md5, lictext = crunch_license(licfile)
               if lictext and not license:
-                license = 'Unknown'
-                logger.info("Please add the following line for '%s' to a 'lib/recipetool/licenses.csv' " \
-                    "and replace `Unknown` with the license:\n" \
-                    "%s,Unknown" % (os.path.relpath(licfile, srctree), md5value))
-        if license:
+                spdx_re = re.compile('SPDX-License-Identifier:\s+([-A-Za-z\d. ]+)[ |\n|\r\n]*?')
+                license_list = re.findall(spdx_re, "\n".join(lictext))
+                if not license_list:
+                    license_list.append('Unknown')
+                    logger.info("Please add the following line for '%s' to a 'lib/recipetool/licenses.csv' " \
+                        "and replace `Unknown` with the license:\n" \
+                        "%s,Unknown" % (os.path.relpath(licfile, srctree), md5value))
+        else:
+            license_list.append(license)
+        for license in license_list:
               licenses.append((license, os.path.relpath(licfile, srctree), md5value))
 
Quoted 4 lines

scripts/lib/recipetool/create.py def guess_license(srctree, d): · line 1240

Quoted 1 line
scripts/lib/recipetool/create.py +0 -0
       # FIXME should we grab at least one source file with a license header and add that too?

I think to close this bug the code may need to go one step further and effectively grep over the source tree.

Please keep in mind that we need a full license text and not only the license name for license compliance. The current function only search for license files with license text.

Quoted 2 lines

We'd probably want to list the value of any SPDX-License-Identifier: header found in any of the source files for the user to then decide upon?

I think this is an other feature like a license checker because if you have a SPDX-License-Identifier without a license text you have a license violation.

Quoted 6 lines
Quoted 2 lines

This brings us to the problem that this code will interpret a file with only a SPDX-License-Identifier as a license file with license text.

As I understand it the tool is there to help write a recipe so filling out LICENSE and highlighting a missing full license text would be a valid approach for the tool and helpful to the user?

Yes, but we should distinguish between license files which are guess via hash of the content and SPDX-License-Identifier which labels the source code’s license. In this case the SPDX-License-Identifier is non-material text from a license file and should be filtered out inside crunch_license function.

The collection of all used licenses via SPDX-License-Identifier is an additional feature and we need a warning if a SPDX-License-Identifier exists without license file.

Quoted 2 lines

It certainly isn't intended as full validation, just intended to assist the creation of a recipe.

But this patch is an regress because it doesn't distinguish between a license file with a known hash and a mostly empty file with a SPDX-License-Identifier.

Regards
Stefan

#7 lore raw
In reply to Saul Wold

Hi Saul,

Am 03.02.2022 um 18:07 schrieb Saul Wold via lists.openembedded.org:

Quoted 6 lines

When a file can not be identified by checksum and they contain an SPDX License-Identifier tag, use it as the found license.

[YOCTO #14529]

Tested with LICENSE files that contain 1 or more SPDX-License-Identifier tags

Can you please give an example for an project with use a SPDX-License-Identifier inside a license file.

Quoted 10 lines
---
  scripts/lib/recipetool/create.py | 16 +++++++++++-----
  1 file changed, 11 insertions(+), 5 deletions(-)
scripts/lib/recipetool/create.py +0 -0
index 507a230511..9149c2d94f 100644

scripts/lib/recipetool/create.py def guess_license(srctree, d): · line 1224

Quoted 4 lines
scripts/lib/recipetool/create.py +1 -0
@@ -1221,14 +1221,20 @@ def guess_license(srctree, d):
      for licfile in sorted(licfiles):
          md5value = bb.utils.md5_file(licfile)
          license = md5sums.get(md5value, None)
+        license_list = []

Could you please use an other name. We already have licenses and it is hard to distinguish the difference between licenses and license_list.

Quoted 11 lines
scripts/lib/recipetool/create.py +3 -5
          if not license:
              license, crunched_md5, lictext = crunch_license(licfile)
              if lictext and not license:
-                license = 'Unknown'
-                logger.info("Please add the following line for '%s' to a 'lib/recipetool/licenses.csv' " \
-                    "and replace `Unknown` with the license:\n" \
-                    "%s,Unknown" % (os.path.relpath(licfile, srctree), md5value))
-        if license:
+                spdx_re = re.compile('SPDX-License-Identifier:\s+([-A-Za-z\d. ]+)[ |\n|\r\n]*?')
+                license_list = re.findall(spdx_re, "\n".join(lictext))
+                if not license_list:

scripts/lib/recipetool/create.py def guess_license(srctree, d): · line 1240

Quoted 10 lines
scripts/lib/recipetool/create.py +7 -0
+                    license_list.append('Unknown')
+                    logger.info("Please add the following line for '%s' to a 'lib/recipetool/licenses.csv' " \
+                        "and replace `Unknown` with the license:\n" \
+                        "%s,Unknown" % (os.path.relpath(licfile, srctree), md5value))
+        else:
+            license_list.append(license)
+        for license in license_list:
              licenses.append((license, os.path.relpath(licfile, srctree), md5value))
 
      # FIXME should we grab at least one source file with a license header and add that too?

Regards
Stefan