Add meta layer information to spdx
cover Add meta layer information to spdx
Hello,
as discussed before here
https://lists.openembedded.org/g/openembedded-core/message/226131
and here
https://lists.openembedded.org/g/openembedded-core/message/225774
it would be good to include information about the layers used in a build in the generated spdx documents.
My primary goal is to track where all the configuration files added via file protocol in SRC_URI originate from, but the approach could be extended to track recipes, classes, machine config etc.
These patches are meant as a basis to discuss the approach and should be considered draft status.
The approach is to generate a layers.spdx.json on each build invocation that lists each layer used in the build as a softwarePackage with version and source information as well as - if can be reasonably deducted - the downloadLocation. Each source file added via file protocol is then linked to the containing meta layer via a contains relationship.
Open Questions:
- is the overall architecture reasonable/correct?
- referring to layers is inconsistent across tooling. E.g. the core layer is
either named core as defined by BBFILE_COLLECTIONS or meta as that is
the layer directory in the openembedded-core repository.
I chose the latter since that is what the utility function
oe.buildcfg.get_layer_revisions returns. The package URL (PURL)
implementation uses the other naming scheme though. Any thoughts on which
should be chosen and why?
- I do not understand the alias linking yet. With the current implementation
I see many hash mismatches like
WARNING: gcc-runtime-15.3.0-r0 do_create_spdx: Element http://spdx.org/spdxdocs/layers-7ad0d8d7-57f9-5c3c-9411-49227dd50610/layers/layer/meta
has alias http://spdx.org/spdxdocs/UNIHASH-7ad0d8d7-57f9-5c3c-9411-49227dd50610/UNIHASH/layer/meta, but it should have prefix http://spdxdocs.org/openembedded-alias/by-doc-hash/e80f1d61f4749e663e80a4dfb25ebb1127b5f50600d3065de7ed91030942a1ef/
I've currently got some time available to work on this to get a proper solution integrated into openembedded-core.
Best Wishes
Daniel Wagenknecht
Daniel Wagenknecht (2):
spdx: add entries for meta-layers
spdx: link files to the meta-layer providing them
meta/classes/create-spdx-3.0.bbclass | 1 + meta/classes/spdx-common.bbclass | 3 + meta/lib/oe/sbom30.py | 12 +++ meta/lib/oe/spdx30_tasks.py | 118 ++++++++++++++++++++++++++- 4 files changed, 130 insertions(+), 4 deletions(-)
Signature · 1 line
2.54.0
PATCH 1/2 spdx: add entries for meta-layers
This adds support for determining the revision and downloadLocation of meta-layers and adds them to the OE generated SPDX document.
This downloadLocation will result in a NOASSERTION entry if
- the meta-layer is not in a git repo
- the git repo is dirty
- the git remote does not have a valid URL
- the git remote does not contain the used revision as far as can be eluded from the local repo
Variables to configure the behavior:
- SPDX_LAYER_REMOTE (default: origin): select the name of the git remote to use for layers. Set to an empty string to always return NOASSERTION
- SPDX_LAYER_REMOTE_layername: override SPDX_LAYER_REMOTE for a specific layer
- SPDX_REQUIRE_LAYER_ASSERTION: set to true to fail a build if determining downloadLocation fails for any layer
--- meta/classes/create-spdx-3.0.bbclass | 1 + meta/classes/spdx-common.bbclass | 3 ++ meta/lib/oe/sbom30.py | 12 +++++ meta/lib/oe/spdx30_tasks.py | 74 ++++++++++++++++++++++++++++ 4 files changed, 90 insertions(+)
meta/classes/create-spdx-3.0.bbclass +1 -0
@@ -285,6 +285,7 @@ addtask do_create_recipe_sbom_setscene python spdx30_build_started_handler () { import oe.spdx30_tasks d = e.data.createCopy() + oe.spdx30_tasks.write_layers_spdx(d) oe.spdx30_tasks.write_bitbake_spdx(d) }
meta/classes/spdx-common.bbclass +3 -0
@@ -89,6 +89,9 @@ SPDX_FILE_EXCLUDE_PATTERNS[doc] = "Space-separated list of Python regular \ (no filtering). Example: \ SPDX_FILE_EXCLUDE_PATTERNS = '\\.patch$ \\.diff$ /test/ \\.pyc$ \\.o$'" +SPDX_REQUIRE_LAYER_ASSERTION ??= "0" +SPDX_LAYER_REMOTE ??= "origin" + python () { from oe.cve_check import extend_cve_status extend_cve_status(d)
meta/lib/oe/sbom30.py +12 -0
@@ -752,6 +752,16 @@ class ObjectSet(oe.spdx30.SHACLObjectSet): return bb_objset + def import_layers_objset(self): + deploy_dir_spdx = Path(self.d.getVar("DEPLOY_DIR_SPDX")) + layers_objset = load_jsonld( + self.d, deploy_dir_spdx / "layers.spdx.json", required=True + ) + self.doc.import_.extend(layers_objset.doc.import_) + self.update(layers_objset.objects) + + return layers_objset + def import_bitbake_build(self): def find_bitbake_build(objset): return objset.find_filter(
@@ -793,6 +803,8 @@ class ObjectSet(oe.spdx30.SHACLObjectSet): [build], ) + layers = self.import_layers_objset() + if self.d.getVar("SPDX_INCLUDE_BUILD_VARIABLES") == "1": for varname in sorted(self.d.keys()): if varname.startswith("__"):
meta/lib/oe/spdx30_tasks.py +74 -0
80 lines not on this page · raw message
Signature · 1 line
2.54.0
PATCH 2/2 spdx: link files to the meta-layer providing them
In reply to Daniel WagenknechtMany recipes ship files that originate from a meta-layer. Link each file added via the file protocol to the spdx entry of the meta-layer containing that file.
Some files might not originate from a meta-layer (file protocol supports absolute paths), fail the build if SPDX_REQUIRE_LAYER_ASSERTION is set to true.
--- meta/lib/oe/spdx30_tasks.py | 44 +++++++++++++++++++++++++++++++++---- 1 file changed, 40 insertions(+), 4 deletions(-)
meta/lib/oe/spdx30_tasks.py +40 -4
@@ -497,6 +497,7 @@ def add_download_files(d, objset): primary_purpose = oe.spdx30.software_SoftwarePurpose.source if fd.type == "file": + (layer_path, layer_objset) = get_layer_info(d, fd.localpath) if os.path.isdir(fd.localpath): walk_idx = 1 for root, dirs, files in os.walk(fd.localpath, onerror=walk_error):
@@ -512,23 +513,36 @@ def add_download_files(d, objset): objset.new_spdxid( "source", str(download_idx + 1), str(walk_idx) ), - os.path.join( - file_name, os.path.relpath(f_path, fd.localpath) - ), + os.path.relpath(f_path, layer_path), f_path, purposes=[primary_purpose], ) + if layer_objset is not None: + objset.new_scoped_relationship( + [oe.sbom30.get_element_link_id(layer_objset)], + oe.spdx30.RelationshipType.contains, + oe.spdx30.LifecycleScopeType.build, + [file], + ) + inputs.add(file) walk_idx += 1 else: file = objset.new_file( objset.new_spdxid("source", str(download_idx + 1)), - file_name, + os.path.relpath(fd.localpath, layer_path), fd.localpath, purposes=[primary_purpose], ) + if layer_objset is not None: + objset.new_scoped_relationship( + [oe.sbom30.get_element_link_id(layer_objset)], + oe.spdx30.RelationshipType.contains, + oe.spdx30.LifecycleScopeType.build, + [file], + ) inputs.add(file) else:
@@ -803,6 +817,28 @@ def load_recipe_spdx(d): oe.spdx30.software_Package, ) +def get_layer_info(d, path): + layer = None + path = Path(path) + + layers = oe.buildcfg.get_layer_revisions(d) + for (l_path, l_name, l_branch, l_rev, l_ismodified) in layers: + l_path = Path(l_path) + if l_path in path.parents: + layer = l_name + break + + if not l_path: + if oe.utils.vartrue("SPDX_REQUIRE_LAYER_ASSERTION", True, False, d): + bb.fatal(f"Path {path} does not originate from a meta-layer!") + return ("/", None) + + deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX")) + objset = oe.sbom30.load_jsonld(d, deploy_dir_spdx / "layers.spdx.json", required=True) + spdx_obj = objset.find_root(oe.spdx30.software_Package, name=l_name) + + return (l_path, spdx_obj) + def create_spdx(d): def set_var_field(var, obj, name, package=None):
Signature · 1 line
2.54.0
Quoted 35 lines · Daniel Wagenknecht via lists.openembedded.org
Hello,
as discussed before here
https://lists.openembedded.org/g/openembedded-core/message/226131
and here
https://lists.openembedded.org/g/openembedded-core/message/225774it would be good to include information about the layers used in a build in the generated spdx documents.
My primary goal is to track where all the configuration files added via file protocol in
SRC_URIoriginate from, but the approach could be extended to track recipes, classes, machine config etc.These patches are meant as a basis to discuss the approach and should be considered draft status.
The approach is to generate a
layers.spdx.jsonon each build invocation that lists each layer used in the build as asoftwarePackagewith version and source information as well as - if can be reasonably deducted - the downloadLocation. Each source file added via file protocol is then linked to the containing meta layer via acontainsrelationship.Open Questions:
- is the overall architecture reasonable/correct?
- referring to layers is inconsistent across tooling. E.g. the core layer is
either namedcoreas defined byBBFILE_COLLECTIONSormetaas that is
the layer directory in the openembedded-core repository.
I chose the latter since that is what the utility function
oe.buildcfg.get_layer_revisionsreturns. The package URL (PURL)
implementation uses the other naming scheme though. Any thoughts on which
should be chosen and why?
- I do not understand the alias linking yet. With the current implementation
I see many hash mismatches like
WARNING: gcc-runtime-15.3.0-r0do_create_spdx: Element http://spdx.org/spdxdocs/layers-7ad0d8d7-57f9-5c3c-9411-49227dd50610/layers/layer/meta
Quoted 5 lines
has alias http://spdx.org/spdxdocs/UNIHASH-7ad0d8d7-57f9-5c3c-9411-49227dd50610/UNIHASH/layer/meta, but it should have prefix http://spdxdocs.org/openembedded-alias/by-doc-hash/e80f1d61f4749e663e80a4dfb25ebb1127b5f50600d3065de7ed91030942a1ef/I've currently got some time available to work on this to get a proper solution integrated into openembedded-core.
This series is causing the following issue on our Autobuilder:
ERROR: No SPDX document named /srv/pokybuild/yocto-worker/qemuarm-alt/build/build/tmp/deploy/spdx/3.0.1/bitbake.spdx.json found
See full log here: https://autobuilder.yoctoproject.org/valkyrie/#/builders/74/builds/4598
Can you look into it?
Thanks,
Antonin
v2 cover Add meta layer information to spdx
v2 of the patch series to fix the build failures on the autobuilder reported by Antonin Godard.
The questions from the initial patch series are still open and additionally the final document seems to be deployed from sstate cache even if inputs (layer revisions) change. Please continue to consider this as a draft for discussion only!
v2:
- fix build failures on clean build (layers.spdx.json created before bitbake.spdx.json)
- rebase onto current master branch
- remove commented out parts for overwriting the layer commit id
Daniel Wagenknecht (2):
spdx: add entries for meta-layers
spdx: link files to the meta-layer providing them
meta/classes/create-spdx-3.0.bbclass | 1 + meta/classes/spdx-common.bbclass | 3 + meta/lib/oe/sbom30.py | 12 +++ meta/lib/oe/spdx30_tasks.py | 116 ++++++++++++++++++++++++++- 4 files changed, 128 insertions(+), 4 deletions(-)
Signature · 1 line
2.54.0
Hi,
Quoted 1 line · Daniel Wagenknecht via lists.openembedded.org
Please continue to consider this as a draft for discussion only!
It is useful to mark these kinds of series with "RFC", if you don't consider it ready for merging. I must admit though I skipped over your mention of this being a draft in your cover letter.
Antonin
Hello Antonin,
thanks for the reply. I agree, some more prominent mark of this being a draft would have been helpful. OE does not have any defined prefixes / markers right? At least I could not find any in the OE contributor guide.
To me RFC is more of a formal process, but I'll include a [DRAFT] marker for future revisions of this patch series as long as they are not ready for merging yet.
Daniel
Quoted 24 lines · Antonin Godard via lists.openembedded.org
Hi,
Quoted 1 line · Daniel Wagenknecht via lists.openembedded.org
Please continue to consider this as a draft for discussion only!
It is useful to mark these kinds of series with "RFC", if you don't consider it
ready for merging. I must admit though I skipped over your mention of this being
a draft in your cover letter.Antonin
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group. View/Reply Online (#245930):
https://lists.openembedded.org/g/openembedded-core/message/245930
Mute This Topic: https://lists.openembedded.org/mt/121260101/5939739
Group Owner: [email protected] Unsubscribe:
https://lists.openembedded.org/g/openembedded-core/unsub [
[email protected]]
-=-=-=-=-=-=-=-=-=-=-=-
v3 cover Add meta layer information to spdx
In reply to Daniel Wagenknechtv3 of the patch series looking for feedback on the overall architecture. I worked out the problem in alias linking, so a major blocker is out of the way.
Please continue to consider this as a draft for discussion only!
During discussion on IRC some questions were raised especially regarding the interaction with sstate caching.
The layer information is only part of create_spdx, not create_recipe_spdx. It is thus only added to the final image spdx documents, not to the individual recipe or packagage spdx document. Images and their spdx documents are always rebuild, so with this approach no additional rebuilds occur even if a layers revision changes.
My still open questions are:
* is the overall architecture reasonable/correct?
* referring to layers is inconsistent across tooling. E.g. the core layer is either named core as defined by BBFILE_COLLECTIONS or meta as that is the layer directory in the openembedded-core repository.
I chose the latter since that is what the utility function oe.buildcfg.get_layer_revisions returns. The package URL (PURL) implementation uses the other naming scheme though. Any thoughts on which should be chosen and why?
v3:
* rebase onto current master branch
* fixed the alias linking
v2:
* fix build failures on clean build (layers.spdx.json created before bitbake.spdx.json)
* rebase onto current master branch
* remove commented out parts for overwriting the layer commit id
Daniel Wagenknecht (2):
spdx: add entries for meta-layers
spdx: link files to the meta-layer providing them
meta/classes/create-spdx-3.0.bbclass | 1 + meta/classes/spdx-common.bbclass | 3 + meta/lib/oe/sbom30.py | 12 +++ meta/lib/oe/spdx30_tasks.py | 118 ++++++++++++++++++++++++++- 4 files changed, 130 insertions(+), 4 deletions(-)
Signature · 1 line
2.54.0
PATCH v3 1/2 spdx: add entries for meta-layers
This adds support for determining the revision and downloadLocation of meta-layers and adds them to the OE generated SPDX document.
This downloadLocation will result in a NOASSERTION entry if
* the meta-layer is not in a git repo
* the git repo is dirty
* the git remote does not have a valid URL
* the git remote does not contain the used revision as far as can be eluded from the local repo
Variables to configure the behavior:
* SPDX_LAYER_REMOTE (default: origin): select the name of the git remote to use for layers. Set to an empty string to always return NOASSERTION
* SPDX_LAYER_REMOTE_layername: override SPDX_LAYER_REMOTE for a specific layer
* SPDX_REQUIRE_LAYER_ASSERTION: set to true to fail a build if determining downloadLocation fails for any layer
--- meta/classes/create-spdx-3.0.bbclass | 1 + meta/classes/spdx-common.bbclass | 3 ++ meta/lib/oe/sbom30.py | 12 +++++ meta/lib/oe/spdx30_tasks.py | 74 ++++++++++++++++++++++++++++ 4 files changed, 90 insertions(+)
meta/classes/create-spdx-3.0.bbclass +1 -0
@@ -288,6 +288,7 @@ python spdx30_build_started_handler () { import oe.spdx30_tasks d = e.data.createCopy() oe.spdx30_tasks.write_bitbake_spdx(d) + oe.spdx30_tasks.write_layers_spdx(d) } addhandler spdx30_build_started_handler
meta/classes/spdx-common.bbclass +3 -0
@@ -89,6 +89,9 @@ SPDX_FILE_EXCLUDE_PATTERNS[doc] = "Space-separated list of Python regular \ (no filtering). Example: \ SPDX_FILE_EXCLUDE_PATTERNS = '\\.patch$ \\.diff$ /test/ \\.pyc$ \\.o$'" +SPDX_REQUIRE_LAYER_ASSERTION ??= "0" +SPDX_LAYER_REMOTE ??= "origin" + python () { from oe.cve_check import extend_cve_status extend_cve_status(d)
meta/lib/oe/sbom30.py +12 -0
@@ -816,6 +816,16 @@ class ObjectSet(oe.spdx30.SHACLObjectSet): return bb_objset + def import_layers_objset(self): + deploy_dir_spdx = Path(self.d.getVar("DEPLOY_DIR_SPDX")) + layers_objset = load_jsonld( + self.d, deploy_dir_spdx / "layers.spdx.json", required=True + ) + self.doc.import_.extend(layers_objset.doc.import_) + self.update(layers_objset.objects) + + return layers_objset + def import_bitbake_build(self): def find_bitbake_build(objset): return objset.find_filter(
@@ -857,6 +867,8 @@ class ObjectSet(oe.spdx30.SHACLObjectSet): [build], ) + layers = self.import_layers_objset() + if self.d.getVar("SPDX_INCLUDE_BUILD_VARIABLES") == "1": for varname in sorted(self.d.keys()): if varname.startswith("__"):
meta/lib/oe/spdx30_tasks.py +74 -0
80 lines not on this page · raw message
Signature · 1 line
2.54.0
Re: [DRAFT PATCH v3 1/2] spdx: add entries for meta-layers
Quoted 18 lines · Daniel Wagenknecht
This adds support for determining the revision and downloadLocation of meta-layers and adds them to the OE generated SPDX document.
This downloadLocation will result in a NOASSERTION entry if
* the meta-layer is not in a git repo
* the git repo is dirty
* the git remote does not have a valid URL
* the git remote does not contain the used revision as far as can be eluded from the local repoVariables to configure the behavior:
*SPDX_LAYER_REMOTE(default: origin): select the name of the git remote to use for layers. Set to an empty string to always return NOASSERTION
*SPDX_LAYER_REMOTE_layername: overrideSPDX_LAYER_REMOTEfor a specific layer
*SPDX_REQUIRE_LAYER_ASSERTION: set to true to fail a build if determining downloadLocation fails for any layer
Quoted 1 line
Signed-off-by Daniel Wagenknecht <[email protected]>
Hi Daniel,
I don't think downloadLocation should be populated by default, it may refer to an internal network location for an organisations' private repositories and leaking those URLs may not be desired.
This probably needs to be opt-in.
Best regards,
Signature · 1 line
Paul Barker
Re: [DRAFT PATCH v3 1/2] spdx: add entries for meta-layers
Hello Paul,
Quoted 3 lines · Paul Barker
I don't think downloadLocation should be populated by default, it may refer to an internal network location for an organisations' private repositories
I already wondered about this when first implementing the approach. By SPDX spec it is OK to have a private URL here (SPDX 2 documentation explicitely noted private URLs being allowed for this property, in SPDX 3 I did not find such a clear statement but also nothing contradicting it)
Quoted 1 line
and leaking those URLs may not be desired.
by being implicit (parsed from the current repo checkout) it's different to private servers explicitely mentioned in SRC_URI, so I agree it may not be desired and considered an information leak.
Quoted 2 lines
This probably needs to be opt-in.
That would be OK with me, I'll make it opt-in in a next revision.
Best Wishes
Daniel
PATCH v3 2/2 spdx: link files to the meta-layer providing them
In reply to Daniel WagenknechtMany recipes ship files that originate from a meta-layer. Link each file added via the file protocol to the spdx entry of the meta-layer containing that file.
Some files might not originate from a meta-layer (file protocol supports absolute paths), fail the build if SPDX_REQUIRE_LAYER_ASSERTION is set to true.
--- meta/lib/oe/spdx30_tasks.py | 44 +++++++++++++++++++++++++++++++++---- 1 file changed, 40 insertions(+), 4 deletions(-)
meta/lib/oe/spdx30_tasks.py +40 -4
@@ -477,6 +477,7 @@ def add_download_files(d, objset): primary_purpose = oe.spdx30.software_SoftwarePurpose.source if fd.type == "file": + (layer_path, layer_objset) = get_layer_info(d, fd.localpath) if os.path.isdir(fd.localpath): walk_idx = 1 for root, dirs, files in os.walk(fd.localpath, onerror=walk_error):
@@ -492,23 +493,36 @@ def add_download_files(d, objset): objset.new_spdxid( "source", str(download_idx + 1), str(walk_idx) ), - os.path.join( - file_name, os.path.relpath(f_path, fd.localpath) - ), + os.path.relpath(f_path, layer_path), f_path, purposes=[primary_purpose], ) + if layer_objset is not None: + objset.new_scoped_relationship( + [oe.sbom30.get_element_link_id(layer_objset)], + oe.spdx30.RelationshipType.contains, + oe.spdx30.LifecycleScopeType.build, + [file], + ) + inputs.add(file) walk_idx += 1 else: file = objset.new_file( objset.new_spdxid("source", str(download_idx + 1)), - file_name, + os.path.relpath(fd.localpath, layer_path), fd.localpath, purposes=[primary_purpose], ) + if layer_objset is not None: + objset.new_scoped_relationship( + [oe.sbom30.get_element_link_id(layer_objset)], + oe.spdx30.RelationshipType.contains, + oe.spdx30.LifecycleScopeType.build, + [file], + ) inputs.add(file) else:
@@ -801,6 +815,28 @@ def load_recipe_spdx(d): oe.spdx30.software_Package, ) +def get_layer_info(d, path): + layer = None + path = Path(path) + + layers = oe.buildcfg.get_layer_revisions(d) + for (l_path, l_name, l_branch, l_rev, l_ismodified) in layers: + l_path = Path(l_path) + if l_path in path.parents: + layer = l_name + break + + if not l_path: + if oe.utils.vartrue("SPDX_REQUIRE_LAYER_ASSERTION", True, False, d): + bb.fatal(f"Path {path} does not originate from a meta-layer!") + return ("/", None) + + deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX")) + objset = oe.sbom30.load_jsonld(d, deploy_dir_spdx / "layers.spdx.json", required=True) + spdx_obj = objset.find_root(oe.spdx30.software_Package, name=l_name) + + return (l_path, spdx_obj) + def create_spdx(d): def set_var_field(var, obj, name, package=None):
Signature · 1 line
2.54.0
Quoted 13 lines · Daniel Wagenknecht
v3 of the patch series looking for feedback on the overall architecture. I worked out the problem in alias linking, so a major blocker is out of the way.
Please continue to consider this as a draft for discussion only!
During discussion on IRC some questions were raised especially regarding the interaction with sstate caching.
The layer information is only part ofcreate_spdx, notcreate_recipe_spdx. It is thus only added to the final image spdx documents, not to the individual recipe or packagage spdx document. Images and their spdx documents are always rebuild, so with this approach no additional rebuilds occur even if a layers revision changes.
Quoted 9 lines
My still open questions are:
* is the overall architecture reasonable/correct?
* referring to layers is inconsistent across tooling. E.g. the core layer is either namedcoreas defined byBBFILE_COLLECTIONSormetaas that is the layer directory in the openembedded-core repository.
I chose the latter since that is what the utility functionoe.buildcfg.get_layer_revisionsreturns. The package URL (PURL) implementation uses the other naming scheme though. Any thoughts on which should be chosen and why?
Hi Daniel,
It's worth thinking about how this will interact with the proposed changes to how layer.conf will work [1] [2]. That should hopefully improve consistency in how layers are referred to across tools, but may also impact other parts of these patches.
[1]: https://lists.openembedded.org/g/openembedded-architecture/topic/layers_and_layer_conf_time/121884200
[2]: https://lists.openembedded.org/g/openembedded-architecture/topic/layer_changes_poc_and_rfc/122113105
Best regards,
Signature · 1 line
Paul Barker
Quoted 19 lines · Daniel Wagenknecht via openembedded-core
v3 of the patch series looking for feedback on the overall architecture. I worked out the problem in alias linking, so a major blocker is out of the way.
Please continue to consider this as a draft for discussion only!
During discussion on IRC some questions were raised especially regarding the
interaction with sstate caching.
The layer information is only part ofcreate_spdx, notcreate_recipe_spdx.
It is thus only added to the final image spdx documents, not to the individual
recipe or packagage spdx document. Images and their spdx documents are always
rebuild, so with this approach no additional rebuilds occur even if a layers
revision changes.
Quoted 3 lines
My still open questions are:
* is the overall architecture reasonable/correct?
Overall, this is approach seems reasonable to me. It's similar to how I've thought about doing it in the past.
Quoted 9 lines
* referring to layers is inconsistent across tooling. E.g. the core layer is
either namedcoreas defined byBBFILE_COLLECTIONSormetaas that
is
the layer directory in the openembedded-core repository.
I chose the latter since that is what the utility functionoe.buildcfg.get_layer_revisionsreturns. The package URL (PURL) implementation uses the other naming scheme though. Any thoughts on which should be chosen and why?
This is very tricky. In an ideal world, we could use a "git" PURL as the official identifier:
https://github.com/package-url/purl-spec/blob/main/docs/types/definitions/git-definition.md
However, I'm not sure that we can reliably use the actual git URL of the layer to construct the PURL, since mirroring, caching, etc. are pretty common. Maybe there should be a per-layer variable that specifies the PURL(s) to use for the layer and would thus survive mirroring, caching, etc.? It would also present the option for a PURL to be something other than just a "git" type (e.g. it could be "github", "bitbucket", or other future PURL types).
Quoted 37 lines
v3:
* rebase onto current master branch
* fixed the alias linkingv2:
* fix build failures on clean build (layers.spdx.json created before bitbake.spdx.json)
* rebase onto current master branch
* remove commented out parts for overwriting the layer commit idDaniel Wagenknecht (2):
spdx: add entries for meta-layers
spdx: link files to the meta-layer providing themmeta/classes/create-spdx-3.0.bbclass | 1 + meta/classes/spdx-common.bbclass | 3 + meta/lib/oe/sbom30.py | 12 +++ meta/lib/oe/spdx30_tasks.py | 118 ++++++++++++++++++++++++++- 4 files changed, 130 insertions(+), 4 deletions(-)--
2.54.0-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group. View/Reply Online (#249634):
https://lists.openembedded.org/g/openembedded-core/message/249634
Mute This Topic: https://lists.openembedded.org/mt/122125585/3616693
Group Owner: [email protected]
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [ [email protected]]
-=-=-=-=-=-=-=-=-=-=-=-
PATCH v2 1/2 spdx: add entries for meta-layers
In reply to Antonin GodardThis adds support for determining the revision and downloadLocation of meta-layers and adds them to the OE generated SPDX document.
This downloadLocation will result in a NOASSERTION entry if
- the meta-layer is not in a git repo
- the git repo is dirty
- the git remote does not have a valid URL
- the git remote does not contain the used revision as far as can be eluded from the local repo
Variables to configure the behavior:
- SPDX_LAYER_REMOTE (default: origin): select the name of the git remote to use for layers. Set to an empty string to always return NOASSERTION
- SPDX_LAYER_REMOTE_layername: override SPDX_LAYER_REMOTE for a specific layer
- SPDX_REQUIRE_LAYER_ASSERTION: set to true to fail a build if determining downloadLocation fails for any layer
--- meta/classes/create-spdx-3.0.bbclass | 1 + meta/classes/spdx-common.bbclass | 3 ++ meta/lib/oe/sbom30.py | 12 +++++ meta/lib/oe/spdx30_tasks.py | 72 ++++++++++++++++++++++++++++ 4 files changed, 88 insertions(+)
meta/classes/create-spdx-3.0.bbclass +1 -0
@@ -286,6 +286,7 @@ python spdx30_build_started_handler () { import oe.spdx30_tasks d = e.data.createCopy() oe.spdx30_tasks.write_bitbake_spdx(d) + oe.spdx30_tasks.write_layers_spdx(d) } addhandler spdx30_build_started_handler
meta/classes/spdx-common.bbclass +3 -0
@@ -89,6 +89,9 @@ SPDX_FILE_EXCLUDE_PATTERNS[doc] = "Space-separated list of Python regular \ (no filtering). Example: \ SPDX_FILE_EXCLUDE_PATTERNS = '\\.patch$ \\.diff$ /test/ \\.pyc$ \\.o$'" +SPDX_REQUIRE_LAYER_ASSERTION ??= "0" +SPDX_LAYER_REMOTE ??= "origin" + python () { from oe.cve_check import extend_cve_status extend_cve_status(d)
meta/lib/oe/sbom30.py +12 -0
@@ -762,6 +762,16 @@ class ObjectSet(oe.spdx30.SHACLObjectSet): return bb_objset + def import_layers_objset(self): + deploy_dir_spdx = Path(self.d.getVar("DEPLOY_DIR_SPDX")) + layers_objset = load_jsonld( + self.d, deploy_dir_spdx / "layers.spdx.json", required=True + ) + self.doc.import_.extend(layers_objset.doc.import_) + self.update(layers_objset.objects) + + return layers_objset + def import_bitbake_build(self): def find_bitbake_build(objset): return objset.find_filter(
@@ -803,6 +813,8 @@ class ObjectSet(oe.spdx30.SHACLObjectSet): [build], ) + layers = self.import_layers_objset() + if self.d.getVar("SPDX_INCLUDE_BUILD_VARIABLES") == "1": for varname in sorted(self.d.keys()): if varname.startswith("__"):
meta/lib/oe/spdx30_tasks.py +72 -0
78 lines not on this page · raw message
Signature · 1 line
2.54.0
PATCH v2 2/2 spdx: link files to the meta-layer providing them
In reply to Antonin GodardMany recipes ship files that originate from a meta-layer. Link each file added via the file protocol to the spdx entry of the meta-layer containing that file.
Some files might not originate from a meta-layer (file protocol supports absolute paths), fail the build if SPDX_REQUIRE_LAYER_ASSERTION is set to true.
--- meta/lib/oe/spdx30_tasks.py | 44 +++++++++++++++++++++++++++++++++---- 1 file changed, 40 insertions(+), 4 deletions(-)
meta/lib/oe/spdx30_tasks.py +40 -4
@@ -453,6 +453,7 @@ def add_download_files(d, objset): primary_purpose = oe.spdx30.software_SoftwarePurpose.source if fd.type == "file": + (layer_path, layer_objset) = get_layer_info(d, fd.localpath) if os.path.isdir(fd.localpath): walk_idx = 1 for root, dirs, files in os.walk(fd.localpath, onerror=walk_error):
@@ -468,23 +469,36 @@ def add_download_files(d, objset): objset.new_spdxid( "source", str(download_idx + 1), str(walk_idx) ), - os.path.join( - file_name, os.path.relpath(f_path, fd.localpath) - ), + os.path.relpath(f_path, layer_path), f_path, purposes=[primary_purpose], ) + if layer_objset is not None: + objset.new_scoped_relationship( + [oe.sbom30.get_element_link_id(layer_objset)], + oe.spdx30.RelationshipType.contains, + oe.spdx30.LifecycleScopeType.build, + [file], + ) + inputs.add(file) walk_idx += 1 else: file = objset.new_file( objset.new_spdxid("source", str(download_idx + 1)), - file_name, + os.path.relpath(fd.localpath, layer_path), fd.localpath, purposes=[primary_purpose], ) + if layer_objset is not None: + objset.new_scoped_relationship( + [oe.sbom30.get_element_link_id(layer_objset)], + oe.spdx30.RelationshipType.contains, + oe.spdx30.LifecycleScopeType.build, + [file], + ) inputs.add(file) else:
@@ -774,6 +788,28 @@ def load_recipe_spdx(d): oe.spdx30.software_Package, ) +def get_layer_info(d, path): + layer = None + path = Path(path) + + layers = oe.buildcfg.get_layer_revisions(d) + for (l_path, l_name, l_branch, l_rev, l_ismodified) in layers: + l_path = Path(l_path) + if l_path in path.parents: + layer = l_name + break + + if not l_path: + if oe.utils.vartrue("SPDX_REQUIRE_LAYER_ASSERTION", True, False, d): + bb.fatal(f"Path {path} does not originate from a meta-layer!") + return ("/", None) + + deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX")) + objset = oe.sbom30.load_jsonld(d, deploy_dir_spdx / "layers.spdx.json", required=True) + spdx_obj = objset.find_root(oe.spdx30.software_Package, name=l_name) + + return (l_path, spdx_obj) + def create_spdx(d): def set_var_field(var, obj, name, package=None):
Signature · 1 line
2.54.0