SPDX 3.0 improvements
cover SPDX 3.0 improvements
Various bug fixes and improvements to SPDX 3.0 support, notably upgrading to SPDX 3.0.1
Joshua Watt (4):
spdx 3.0: Find local sources when searching for debug sources spdx 3.0: Map gitsm URI to git
create-spdx-3.0: Upgrade to SPDX 3.0.1
Add script to make SPDX bindings
meta/classes/create-spdx-2.2.bbclass | 11 +- meta/classes/create-spdx-3.0.bbclass | 2 +- meta/lib/oe/sbom30.py | 15 +- meta/lib/oe/spdx30.py | 3375 +++++++++++-------------- meta/lib/oe/spdx30_tasks.py | 112 +- meta/lib/oe/spdx_common.py | 20 +- scripts/contrib/make-spdx-bindings.sh | 12 + 7 files changed, 1560 insertions(+), 1987 deletions(-) create mode 100755 scripts/contrib/make-spdx-bindings.sh
Signature · 1 line
2.46.0
PATCH 1/4 spdx 3.0: Find local sources when searching for debug sources
Include the local files when searching for matching debug sources
--- meta/lib/oe/spdx30_tasks.py | 59 ++++++++++++++++++++----------------- 1 file changed, 32 insertions(+), 27 deletions(-)
meta/lib/oe/spdx30_tasks.py +32 -27
@@ -119,7 +119,9 @@ def add_license_expression(d, objset, license_expression, license_data): ) spdx_license_expression = " ".join(convert(l) for l in lic_split) - return objset.new_license_expression(spdx_license_expression, license_data, license_text_map) + return objset.new_license_expression( + spdx_license_expression, license_data, license_text_map + ) def add_package_files(
@@ -202,6 +204,7 @@ def get_package_sources_from_debug( return False debug_search_paths = [ + Path(d.getVar("SPDXWORK")), Path(d.getVar("PKGD")), Path(d.getVar("STAGING_DIR_TARGET")), Path(d.getVar("STAGING_DIR_NATIVE")),
@@ -286,8 +289,24 @@ def collect_dep_objsets(d, build): return dep_objsets, dep_builds -def collect_dep_sources(dep_objsets): - sources = {} +def index_sources_by_hash(sources, dest): + for s in sources: + if not isinstance(s, oe.spdx30.software_File): + continue + + if s.software_primaryPurpose != oe.spdx30.software_SoftwarePurpose.source: + continue + + for v in s.verifiedUsing: + if v.algorithm == oe.spdx30.HashAlgorithm.sha256: + if not v.hashValue in dest: + dest[v.hashValue] = s + break + else: + bb.fatal(f"No SHA256 found for {s.name}") + + +def collect_dep_sources(dep_objsets, dest): for objset in dep_objsets: # Don't collect sources from native recipes as they # match non-native sources also.
@@ -307,26 +326,7 @@ def collect_dep_sources(dep_objsets): if e.relationshipType != oe.spdx30.RelationshipType.hasInputs: continue - for to in e.to: - if not isinstance(to, oe.spdx30.software_File): - continue - - if ( - to.software_primaryPurpose - != oe.spdx30.software_SoftwarePurpose.source - ): - continue - - for v in to.verifiedUsing: - if v.algorithm == oe.spdx30.HashAlgorithm.sha256: - sources[v.hashValue] = to - break - else: - bb.fatal( - "No SHA256 found for %s in %s" % (to.name, objset.doc.name) - ) - - return sources + index_sources_by_hash(e.to, dest) def add_download_files(d, objset):
@@ -511,18 +511,21 @@ def create_spdx(d): source_files = add_download_files(d, build_objset) build_inputs |= source_files - recipe_spdx_license = add_license_expression(d, build_objset, d.getVar("LICENSE"), license_data) + recipe_spdx_license = add_license_expression( + d, build_objset, d.getVar("LICENSE"), license_data + ) build_objset.new_relationship( source_files, oe.spdx30.RelationshipType.hasConcludedLicense, [recipe_spdx_license], ) + dep_sources = {} if oe.spdx_common.process_sources(d) and include_sources: bb.debug(1, "Adding source files to SPDX") oe.spdx_common.get_patched_src(d) - build_inputs |= add_package_files( + files = add_package_files( d, build_objset, spdx_workdir,
@@ -535,6 +538,8 @@ def create_spdx(d): ignore_top_level_dirs=["temp"], archive=None, ) + build_inputs |= files + index_sources_by_hash(files, dep_sources) dep_objsets, dep_builds = collect_dep_objsets(d, build) if dep_builds:
@@ -555,7 +560,7 @@ def create_spdx(d): # TODO: Handle native recipe output if not is_native: bb.debug(1, "Collecting Dependency sources files") - sources = collect_dep_sources(dep_objsets) + collect_dep_sources(dep_objsets, dep_sources) bb.build.exec_func("read_subpackage_metadata", d)
@@ -726,7 +731,7 @@ def create_spdx(d): if include_sources: debug_sources = get_package_sources_from_debug( - d, package, package_files, sources, source_hash_cache + d, package, package_files, dep_sources, source_hash_cache ) debug_source_ids |= set( oe.sbom30.get_element_link_id(d) for d in debug_sources
Signature · 1 line
2.46.0
PATCH 2/4 spdx 3.0: Map gitsm URI to git
In reply to Joshua Watt"gitsm" is not a recognized URI protocol (outside of bitbake), so map it to "git" when writing. This should be OK since we report all of the submodule source code (if enabled), and it's still possible for 3rd party analyzers to determine that submodules are in use by looking at .gitmodules.
The code to do the mapping is moved to a common location so it covers SPDX 2.2 also
[YOCTO #15582]
--- meta/classes/create-spdx-2.2.bbclass | 11 +---------- meta/lib/oe/spdx30_tasks.py | 13 +++---------- meta/lib/oe/spdx_common.py | 20 +++++++++++++++++++- 3 files changed, 23 insertions(+), 21 deletions(-)
meta/classes/create-spdx-2.2.bbclass +1 -10
@@ -354,15 +354,6 @@ def add_download_packages(d, doc, recipe): if f.type == "file": continue - uri = f.type - proto = getattr(f, "proto", None) - if proto is not None: - uri = uri + "+" + proto - uri = uri + "://" + f.host + f.path - - if f.method.supports_srcrev(): - uri = uri + "@" + f.revisions[name] - if f.method.supports_checksum(f): for checksum_id in CHECKSUM_LIST: if checksum_id.upper() not in oe.spdx.SPDXPackage.ALLOWED_CHECKSUMS:
@@ -377,7 +368,7 @@ def add_download_packages(d, doc, recipe): c.checksumValue = expected_checksum package.checksums.append(c) - package.downloadLocation = uri + package.downloadLocation = oe.spdx_common.fetch_data_to_uri(f, name) doc.packages.append(package) doc.add_relationship(doc, "DESCRIBES", package) # In the future, we might be able to do more fancy dependencies,
meta/lib/oe/spdx30_tasks.py +3 -10
@@ -379,22 +379,15 @@ def add_download_files(d, objset): inputs.add(file) else: - uri = fd.type - proto = getattr(fd, "proto", None) - if proto is not None: - uri = uri + "+" + proto - uri = uri + "://" + fd.host + fd.path - - if fd.method.supports_srcrev(): - uri = uri + "@" + fd.revisions[name] - dl = objset.add( oe.spdx30.software_Package( _id=objset.new_spdxid("source", str(download_idx + 1)), creationInfo=objset.doc.creationInfo, name=file_name, software_primaryPurpose=primary_purpose, - software_downloadLocation=uri, + software_downloadLocation=oe.spdx_common.fetch_data_to_uri( + fd, name + ), ) )
meta/lib/oe/spdx_common.py +19 -1
@@ -42,7 +42,6 @@ def is_work_shared_spdx(d): def load_spdx_license_data(d): - with open(d.getVar("SPDX_LICENSES"), "r") as f: data = json.load(f) # Transform the license array to a dictionary
@@ -225,3 +224,22 @@ def get_patched_src(d): bb.utils.mkdirhier(spdx_workdir) finally: d.setVar("WORKDIR", workdir) + + +def fetch_data_to_uri(fd, name): + """ + Translates a bitbake FetchData to a string URI + """ + uri = fd.type + # Map gitsm to git, since gitsm:// is not a valid URI protocol + if uri == "gitsm": + uri = "git" + proto = getattr(fd, "proto", None) + if proto is not None: + uri = uri + "+" + proto + uri = uri + "://" + fd.host + fd.path + + if fd.method.supports_srcrev(): + uri = uri + "@" + fd.revisions[name] + + return uri
Signature · 1 line
2.46.0
PATCH 3/4 create-spdx-3.0: Upgrade to SPDX 3.0.1
In reply to Joshua WattUpgrades the SPDX 3.0 implementation from 3.0.0 -> 3.0.1. This version introduced some breaking changes. Effectively, 3.0.0 was a pre-release version that we do not need to support any longer.
--- meta/classes/create-spdx-3.0.bbclass | 2 +- meta/lib/oe/sbom30.py | 15 +- meta/lib/oe/spdx30.py | 3375 +++++++++++--------------- meta/lib/oe/spdx30_tasks.py | 40 +- 4 files changed, 1493 insertions(+), 1939 deletions(-)
meta/classes/create-spdx-3.0.bbclass +1 -1
@@ -6,7 +6,7 @@ inherit spdx-common -SPDX_VERSION = "3.0.0" +SPDX_VERSION = "3.0.1" # The list of SPDX profiles generated documents will conform to SPDX_PROFILES ?= "core build software simpleLicensing security"
meta/lib/oe/sbom30.py +7 -8
@@ -359,7 +359,7 @@ class ObjectSet(oe.spdx30.SHACLObjectSet): if not spdxid: bb.fatal(f"{key} is not a valid SPDX_IMPORTS key") - for i in self.docs.imports: + for i in self.docs.import_: if i.externalSpdxId == spdxid: # Already imported return spdxid
@@ -380,7 +380,7 @@ class ObjectSet(oe.spdx30.SHACLObjectSet): ) ) - self.doc.imports.append(m) + self.doc.import_.append(m) return spdxid def new_agent(self, varname, *, creation_info=None, add=True):
@@ -521,8 +521,7 @@ class ObjectSet(oe.spdx30.SHACLObjectSet): return [] if not to: - # TODO: Switch to the code constant once SPDX 3.0.1 is released - to = ["https://spdx.org/rdf/3.0.0/terms/Core/NoneElement"] + to = [oe.spdx30.Element.NoneElement] ret = []
@@ -726,7 +725,7 @@ class ObjectSet(oe.spdx30.SHACLObjectSet): bb_objset = load_jsonld( self.d, deploy_dir_spdx / "bitbake.spdx.json", required=True ) - self.doc.imports.extend(bb_objset.doc.imports) + self.doc.import_.extend(bb_objset.doc.import_) self.update(bb_objset.objects) return bb_objset
@@ -827,12 +826,12 @@ class ObjectSet(oe.spdx30.SHACLObjectSet): Returns the set of ids that could not be found to link into the document """ missing_spdxids = set() - imports = {e.externalSpdxId: e for e in self.doc.imports} + imports = {e.externalSpdxId: e for e in self.doc.import_} def merge_doc(other): nonlocal imports - for e in other.doc.imports: + for e in other.doc.import_: if not e.externalSpdxId in imports: imports[e.externalSpdxId] = e
@@ -876,7 +875,7 @@ class ObjectSet(oe.spdx30.SHACLObjectSet): f"Linked document doesn't match missing SPDX ID list. Got: {missing}\nExpected: {missing_spdxids}" ) - self.doc.imports = sorted(imports.values(), key=lambda e: e.externalSpdxId) + self.doc.import_ = sorted(imports.values(), key=lambda e: e.externalSpdxId) return missing_spdxids
meta/lib/oe/spdx30.py +1463 -1912
2,937 lines not on this page · raw message
The rest of this diff is not shown. View the raw message.
Signature · 1 line
2.46.0
Part of the diff is not shown here. View the raw message.
PATCH 4/4 Add script to make SPDX bindings
In reply to Joshua WattAdds a script to generate the SPDX code bindings
--- scripts/contrib/make-spdx-bindings.sh | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100755 scripts/contrib/make-spdx-bindings.sh
scripts/contrib/make-spdx-bindings.sh new file +12 -0
@@ -0,0 +1,12 @@ +#! /bin/sh +# +# SPDX-License-Identifier: MIT + +THIS_DIR="$(dirname "$0")" + +VERSION="3.0.1" + +shacl2code generate --input https://spdx.org/rdf/$VERSION/spdx-model.ttl \ + --input https://spdx.org/rdf/$VERSION/spdx-json-serialize-annotations.ttl \ + --context https://spdx.org/rdf/$VERSION/spdx-context.jsonld \ + python -o $THIS_DIR/../../meta/lib/oe/spdx30.py
Signature · 1 line
2.46.0
Re: [OE-core][PATCH 3/4] create-spdx-3.0: Upgrade to SPDX 3.0.1
In reply to Joshua WattQuoted 4 lines · Joshua Watt via lists.openembedded.org
Upgrades the SPDX 3.0 implementation from 3.0.0 -> 3.0.1. This version introduced some breaking changes. Effectively, 3.0.0 was a pre-release version that we do not need to support any longer.
This sounds… important.
Should we ram this into styhead for the 5.1 release? Would upgrading to 3.0.1 in a styhead point release be reasonable if not? Or should this stay in master and not touch styhead?
Cheers,
Ross
Patchtest results for [OE-core][PATCH 3/4] create-spdx-3.0: Upgrade to SPDX 3.0.1
In reply to Joshua WattThank you for your submission. Patchtest identified one or more issues with the patch. Please see the log below for more information:
---
Testing patch /home/patchtest/share/mboxes/3-4-create-spdx-3.0-Upgrade-to-SPDX-3.0.1.patch
FAIL: test max line length: Patch line too long (current length 231, maximum is 200) (test_metadata.TestMetadata.test_max_line_length)
PASS: pretest pylint (test_python_pylint.PyLint.pretest_pylint)
PASS: test Signed-off-by presence (test_mbox.TestMbox.test_signed_off_by_presence)
PASS: test author valid (test_mbox.TestMbox.test_author_valid)
PASS: test commit message presence (test_mbox.TestMbox.test_commit_message_presence)
PASS: test mbox format (test_mbox.TestMbox.test_mbox_format)
PASS: test non-AUH upgrade (test_mbox.TestMbox.test_non_auh_upgrade)
PASS: test pylint (test_python_pylint.PyLint.test_pylint)
PASS: test shortlog format (test_mbox.TestMbox.test_shortlog_format)
PASS: test shortlog length (test_mbox.TestMbox.test_shortlog_length)
PASS: test target mailing list (test_mbox.TestMbox.test_target_mailing_list)
SKIP: pretest src uri left files: Patch cannot be merged (test_metadata.TestMetadata.pretest_src_uri_left_files)
SKIP: test CVE check ignore: No modified recipes or older target branch, skipping test (test_metadata.TestMetadata.test_cve_check_ignore)
SKIP: test CVE tag format: No new CVE patches introduced (test_patch.TestPatch.test_cve_tag_format)
SKIP: test Signed-off-by presence: No new CVE patches introduced (test_patch.TestPatch.test_signed_off_by_presence)
SKIP: test Upstream-Status presence: No new CVE patches introduced (test_patch.TestPatch.test_upstream_status_presence_format)
SKIP: test bugzilla entry format: No bug ID found (test_mbox.TestMbox.test_bugzilla_entry_format)
SKIP: test lic files chksum modified not mentioned: No modified recipes, skipping test (test_metadata.TestMetadata.test_lic_files_chksum_modified_not_mentioned)
SKIP: test lic files chksum presence: No added recipes, skipping test (test_metadata.TestMetadata.test_lic_files_chksum_presence)
SKIP: test license presence: No added recipes, skipping test (test_metadata.TestMetadata.test_license_presence)
SKIP: test series merge on head: Merge test is disabled for now (test_mbox.TestMbox.test_series_merge_on_head)
SKIP: test src uri left files: Patch cannot be merged (test_metadata.TestMetadata.test_src_uri_left_files)
SKIP: test summary presence: No added recipes, skipping test (test_metadata.TestMetadata.test_summary_presence)
---
Please address the issues identified and
submit a new revision of the patch, or alternatively, reply to this email with an explanation of why the patch should be accepted. If you believe these results are due to an error in patchtest, please submit a bug at https://bugzilla.yoctoproject.org/ (use the 'Patchtest' category under 'Yocto Project Subprojects'). For more information on specific failures, see: https://wiki.yoctoproject.org/wiki/Patchtest. Thank you!
Patchtest results for [OE-core][PATCH 4/4] Add script to make SPDX bindings
In reply to Joshua WattThank you for your submission. Patchtest identified one or more issues with the patch. Please see the log below for more information:
---
Testing patch /home/patchtest/share/mboxes/4-4-Add-script-to-make-SPDX-bindings.patch
FAIL: test shortlog format: Commit shortlog (first line of commit message) should follow the format "<target>: <summary>" (test_mbox.TestMbox.test_shortlog_format)
PASS: test Signed-off-by presence (test_mbox.TestMbox.test_signed_off_by_presence)
PASS: test author valid (test_mbox.TestMbox.test_author_valid)
PASS: test commit message presence (test_mbox.TestMbox.test_commit_message_presence)
PASS: test max line length (test_metadata.TestMetadata.test_max_line_length)
PASS: test mbox format (test_mbox.TestMbox.test_mbox_format)
PASS: test non-AUH upgrade (test_mbox.TestMbox.test_non_auh_upgrade)
PASS: test shortlog length (test_mbox.TestMbox.test_shortlog_length)
SKIP: pretest pylint: No python related patches, skipping test (test_python_pylint.PyLint.pretest_pylint)
SKIP: pretest src uri left files: No modified recipes, skipping pretest (test_metadata.TestMetadata.pretest_src_uri_left_files)
SKIP: test CVE check ignore: No modified recipes or older target branch, skipping test (test_metadata.TestMetadata.test_cve_check_ignore)
SKIP: test CVE tag format: No new CVE patches introduced (test_patch.TestPatch.test_cve_tag_format)
SKIP: test Signed-off-by presence: No new CVE patches introduced (test_patch.TestPatch.test_signed_off_by_presence)
SKIP: test Upstream-Status presence: No new CVE patches introduced (test_patch.TestPatch.test_upstream_status_presence_format)
SKIP: test bugzilla entry format: No bug ID found (test_mbox.TestMbox.test_bugzilla_entry_format)
SKIP: test lic files chksum modified not mentioned: No modified recipes, skipping test (test_metadata.TestMetadata.test_lic_files_chksum_modified_not_mentioned)
SKIP: test lic files chksum presence: No added recipes, skipping test (test_metadata.TestMetadata.test_lic_files_chksum_presence)
SKIP: test license presence: No added recipes, skipping test (test_metadata.TestMetadata.test_license_presence)
SKIP: test pylint: No python related patches, skipping test (test_python_pylint.PyLint.test_pylint)
SKIP: test series merge on head: Merge test is disabled for now (test_mbox.TestMbox.test_series_merge_on_head)
SKIP: test src uri left files: No modified recipes, skipping pretest (test_metadata.TestMetadata.test_src_uri_left_files)
SKIP: test summary presence: No added recipes, skipping test (test_metadata.TestMetadata.test_summary_presence)
SKIP: test target mailing list: Series merged, no reason to check other mailing lists (test_mbox.TestMbox.test_target_mailing_list)
---
Please address the issues identified and
submit a new revision of the patch, or alternatively, reply to this email with an explanation of why the patch should be accepted. If you believe these results are due to an error in patchtest, please submit a bug at https://bugzilla.yoctoproject.org/ (use the 'Patchtest' category under 'Yocto Project Subprojects'). For more information on specific failures, see: https://wiki.yoctoproject.org/wiki/Patchtest. Thank you!
Re: [OE-core][PATCH 3/4] create-spdx-3.0: Upgrade to SPDX 3.0.1
In reply to Ross BurtonQuoted 10 lines · Ross Burton via lists.openembedded.org
Quoted 4 lines · Joshua Watt via lists.openembedded.org
Upgrades the SPDX 3.0 implementation from 3.0.0 -> 3.0.1. This version introduced some breaking changes. Effectively, 3.0.0 was a pre-release version that we do not need to support any longer.
This sounds… important.
Should we ram this into styhead for the 5.1 release? Would upgrading to 3.0.1 in a styhead point release be reasonable if not? Or should this stay in master and not touch styhead?
Per discussion with Richard, we don't want to hold up the release for this, but it will need to be backported to styhead. I don't have a preference if that is before or after the release
Quoted 11 lines
Cheers,
Ross
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#205035): https://lists.openembedded.org/g/openembedded-core/message/205035
Mute This Topic: https://lists.openembedded.org/mt/108688994/3616693
Group Owner: [email protected]
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [[email protected]]
-=-=-=-=-=-=-=-=-=-=-=-