Add SPDX 3.0 support
cover Add SPDX 3.0 support
This patch series add support for SPDX 3.0. Currently it is not possible to have SPDX 2.2 and SPDX 3.0 enabled at the same time; users may choose to specifically use SPDX 3.0 instead of 2.2 by doing INHERIT += "create-spdx-3.0"
The default version is not changed yet, although that will likely happen soon
Joshua Watt (6):
classes-recipe/image: Add image file manifest classes/spdx-common: Move common SPDX to new class classes/spdx-common: Add SPDX version to path classes/spdx-common: Return empty list from extract_licenses classes/create-spdx-3.0: Add class classes-recipe/image_types: Add SPDX_IMAGE_PURPOSE to images
meta/classes-recipe/image.bbclass | 61 + meta/classes-recipe/image_types.bbclass | 2 + meta/classes-recipe/image_types_wic.bbclass | 1 + meta/classes/create-spdx-2.2.bbclass | 261 +- meta/classes/create-spdx-3.0.bbclass | 1284 +++++ meta/classes/spdx-common.bbclass | 280 + meta/lib/oe/sbom30.py | 993 ++++ meta/lib/oe/spdx30.py | 5413 +++++++++++++++++++ 8 files changed, 8046 insertions(+), 249 deletions(-) create mode 100644 meta/classes/create-spdx-3.0.bbclass create mode 100644 meta/classes/spdx-common.bbclass create mode 100644 meta/lib/oe/sbom30.py create mode 100644 meta/lib/oe/spdx30.py
Signature · 1 line
2.43.2
PATCH 1/6 classes-recipe/image: Add image file manifest
Downstream tasks may want to know what image files were written by the do_image family of tasks (e.g. SPDX) so have each task write out a manifest file that describes the files it produced, then aggregate them in do_image_complete
--- meta/classes-recipe/image.bbclass | 61 +++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+)
meta/classes-recipe/image.bbclass +61 -0
@@ -88,6 +88,11 @@ PACKAGE_INSTALL_ATTEMPTONLY ?= "${FEATURE_INSTALL_OPTIONAL}" IMGDEPLOYDIR = "${WORKDIR}/deploy-${PN}-image-complete" +IMGMANIFESTDIR = "${WORKDIR}/image-task-manifest" + +IMAGE_FILE_MANIFEST_DIR = "${WORKDIR}/deploy-image-file-manifest" +IMAGE_FILE_MANIFEST = "${IMAGE_FILE_MANIFEST_DIR}/manifest.json" + # Images are generally built explicitly, do not need to be part of world. EXCLUDE_FROM_WORLD = "1"
@@ -277,14 +282,31 @@ fakeroot python do_image () { execute_pre_post_process(d, pre_process_cmds) } do_image[dirs] = "${TOPDIR}" +do_image[cleandirs] = "${IMGMANIFESTDIR}" addtask do_image after do_rootfs fakeroot python do_image_complete () { from oe.utils import execute_pre_post_process + from pathlib import Path + import json post_process_cmds = d.getVar("IMAGE_POSTPROCESS_COMMAND") execute_pre_post_process(d, post_process_cmds) + + image_manifest_dir = Path(d.getVar('IMGMANIFESTDIR')) + + data = [] + + for taskname in bb.build.preceedtask("do_image_complete", False, d): + manifest_path = image_manifest_dir / (taskname + ".json") + if not manifest_path.exists(): + continue + with manifest_path.open("r") as f: + data.extend(json.load(f)) + + with open(d.getVar("IMAGE_FILE_MANIFEST"), "w") as f: + json.dump(data, f) } do_image_complete[dirs] = "${TOPDIR}" SSTATETASKS += "do_image_complete"
@@ -292,6 +314,8 @@ SSTATE_SKIP_CREATION:task-image-complete = '1' do_image_complete[sstate-inputdirs] = "${IMGDEPLOYDIR}" do_image_complete[sstate-outputdirs] = "${DEPLOY_DIR_IMAGE}" do_image_complete[stamp-extra-info] = "${MACHINE_ARCH}" +do_image_complete[sstate-plaindirs] = "${IMAGE_FILE_MANIFEST_DIR}" +do_image_complete[dirs] = "${IMAGE_FILE_MANIFEST_DIR}" addtask do_image_complete after do_image before do_build python do_image_complete_setscene () { sstate_setscene(d)
@@ -513,6 +537,8 @@ python () { d.appendVarFlag(task, 'subimages', ' ' + ' '.join(subimages)) d.appendVarFlag(task, 'vardeps', ' ' + ' '.join(vardeps)) d.appendVarFlag(task, 'vardepsexclude', ' DATETIME DATE ' + ' '.join(vardepsexclude)) + d.appendVarFlag(task, 'postfuncs', ' write_image_file_manifest') + d.appendVarFlag(task, 'dirs', ' ' + d.getVar('IMGMANIFESTDIR')) bb.debug(2, "Adding task %s before %s, after %s" % (task, 'do_image_complete', after)) bb.build.addtask(task, 'do_image_complete', after, d)
@@ -610,6 +636,41 @@ python create_symlinks() { bb.note("Skipping symlink, source does not exist: %s -> %s" % (dst, src)) } +python write_image_file_manifest() { + import json + from pathlib import Path + + taskname = d.getVar("BB_CURRENTTASK") + image_deploy_dir = Path(d.getVar('IMGDEPLOYDIR')) + image_manifest_dir = Path(d.getVar('IMGMANIFESTDIR')) + manifest_path = image_manifest_dir / ("do_" + d.getVar("BB_CURRENTTASK") + ".json") + + image_name = d.getVar("IMAGE_NAME") + image_basename = d.getVar("IMAGE_BASENAME") + machine = d.getVar("MACHINE") + + subimages = (d.getVarFlag("do_" + taskname, 'subimages', False) or "").split() + imagetype = d.getVarFlag("do_" + taskname, 'imagetype', False) + + data = { + "taskname": taskname, + "imagetype": imagetype, + "images": [] + } + + for type in subimages: + image_filename = image_name + "." + type + image_path = image_deploy_dir / image_filename + if not image_path.exists(): + continue + data["images"].append({ + "filename": image_filename, + }) + + with manifest_path.open("w") as f: + json.dump([data], f) +} + MULTILIBRE_ALLOW_REP += "${base_bindir} ${base_sbindir} ${bindir} ${sbindir} ${libexecdir} ${sysconfdir} ${nonarch_base_libdir}/udev /lib/modules/[^/]*/modules.*" MULTILIB_CHECK_FILE = "${WORKDIR}/multilib_check.py" MULTILIB_TEMP_ROOTFS = "${WORKDIR}/multilib"
Signature · 1 line
2.43.2
Re: [OE-core][PATCH 1/6] classes-recipe/image: Add image file manifest
Hi Joshua,
Two minor nits below.
Quoted 35 lines · Joshua Watt
Downstream tasks may want to know what image files were written by the
do_imagefamily of tasks (e.g. SPDX) so have each task write out a manifest file that describes the files it produced, then aggregate them
indo_image_completeSigned-off-by Joshua Watt <[email protected]>---
meta/classes-recipe/image.bbclass | 61
+++++++++++++++++++++++++++++++1 file changed, 61 insertions(+)meta/classes- renamed from meta/classes-recipe/image.bbclass +0
-0recipe/image.bbclass
index28be6c63623..122af2b0328100644meta/classes-recipe/image.bbclass +0
-0@@ -88,6 +88,11 @@ PACKAGE_INSTALL_ATTEMPTONLY ?="${FEATURE_INSTALL_OPTIONAL}"
IMGDEPLOYDIR = "${WORKDIR}/deploy-${PN}-image-complete"meta/classes-recipe/image.bbclass +5
-0+IMGMANIFESTDIR = "${WORKDIR}/image-task-manifest" + +IMAGE_FILE_MANIFEST_DIR = "${WORKDIR}/deploy-image-file-manifest" +IMAGE_FILE_MANIFEST = "${IMAGE_FILE_MANIFEST_DIR}/manifest.json" +# Images are generally built explicitly, do not need to be part of world.
EXCLUDE_FROM_WORLD = "1"meta/classes-recipe/image.bbclass +0
-0@@ -277,14 +282,31 @@ fakeroot python do_image () { execute_pre_post_process(d, pre_process_cmds) }
Quoted 2 lines
do_image[dirs] = "${TOPDIR}"
+do_image[cleandirs] = "${IMGMANIFESTDIR}"
Should this be an append (i.e. +=) to avoid conflicting with recipes ?
Quoted 32 lines
addtask
do_imageafterdo_rootfs
fakeroot pythondo_image_complete() {
from oe.utils importexecute_pre_post_process
+ from pathlib import Path
+ import json
post_process_cmds=d.getVar("IMAGE_POSTPROCESS_COMMAND")
execute_pre_post_process(d, post_process_cmds)meta/classes-recipe/image.bbclass +6
-0+ + image_manifest_dir = Path(d.getVar('IMGMANIFESTDIR')) + + data = [] + + for taskname in bb.build.preceedtask("do_image_complete", False,d):
+manifest_path=image_manifest_dir/ (taskname + ".json") + if notmanifest_path.exists():
+ continue
+ withmanifest_path.open("r")as f:
+ data.extend(json.load(f))
+
+ withopen(d.getVar("IMAGE_FILE_MANIFEST"), "w")as f:
+json.dump(data, f)
}
do_image_complete[dirs] = "${TOPDIR}"
SSTATETASKS += "do_image_complete"meta/classes-recipe/image.bbclass +0
-0@@ -292,6 +314,8 @@ SSTATE_SKIP_CREATION:task-image-complete = '1'do_image_complete[sstate-inputdirs] = "${IMGDEPLOYDIR}" do_image_complete[sstate-outputdirs] = "${DEPLOY_DIR_IMAGE}" do_image_complete[stamp-extra-info] = "${MACHINE_ARCH}"
meta/classes-recipe/image.bbclass SSTATE_SKIP_CREATION:task-image-complete = '1' · line 318
Quoted 2 lines
meta/classes-recipe/image.bbclass +2
-0+do_image_complete[sstate-plaindirs] = "${IMAGE_FILE_MANIFEST_DIR}" +do_image_complete[dirs] = "${IMAGE_FILE_MANIFEST_DIR}"
Same here...
Thanks,
Martin
Quoted 76 lines
addtask
do_image_completeafterdo_imagebeforedo_buildpythondo_image_complete_setscene() {sstate_setscene(d)meta/classes-recipe/image.bbclass +0
-0@@ -513,6 +537,8 @@ python () {
d.appendVarFlag(task, 'subimages', ' ' + ' '.join(subimages))
d.appendVarFlag(task, 'vardeps', ' ' + ' '.join(vardeps))d.appendVarFlag(task, 'vardepsexclude', ' DATETIME DATE ' + ' '.join(vardepsexclude))
+d.appendVarFlag(task, 'postfuncs', ' write_image_file_manifest')
+d.appendVarFlag(task, 'dirs', ' ' + d.getVar('IMGMANIFESTDIR'))
bb.debug(2, "Adding task %s before %s, after %s" % (task, 'do_image_complete', after))
bb.build.addtask(task, 'do_image_complete', after, d)meta/classes-recipe/image.bbclass +0
-0@@ -610,6 +636,41 @@ python create_symlinks() {
bb.note("Skipping symlink, source does not exist: %s -> %s" % (dst, src))
}meta/classes-recipe/image.bbclass +8
-0+python write_image_file_manifest() { + import json + from pathlib import Path + + taskname = d.getVar("BB_CURRENTTASK") + image_deploy_dir = Path(d.getVar('IMGDEPLOYDIR')) + image_manifest_dir = Path(d.getVar('IMGMANIFESTDIR')) + manifest_path = image_manifest_dir / ("do_" +
d.getVar("BB_CURRENTTASK")+ ".json")meta/classes-recipe/image.bbclass +6
-0+ + image_name = d.getVar("IMAGE_NAME") + image_basename = d.getVar("IMAGE_BASENAME") + machine = d.getVar("MACHINE") + + subimages = (d.getVarFlag("do_" + taskname, 'subimages', False)or "").split()
+ imagetype =d.getVarFlag("do_" + taskname, 'imagetype', False)meta/classes-recipe/image.bbclass +20
-0+ + data = { + "taskname": taskname, + "imagetype": imagetype, + "images": [] + } + + for type in subimages: + image_filename = image_name + "." + type + image_path = image_deploy_dir / image_filename + if not image_path.exists(): + continue + data["images"].append({ + "filename": image_filename, + }) + + with manifest_path.open("w") as f: + json.dump([data], f) +} +MULTILIBRE_ALLOW_REP += "${base_bindir} ${base_sbindir} ${bindir} ${sbindir} ${libexecdir} ${sysconfdir} ${nonarch_base_libdir}/udev /lib/modules/[^/]*/modules.*"
MULTILIB_CHECK_FILE = "${WORKDIR}/multilib_check.py"MULTILIB_TEMP_ROOTFS= "${WORKDIR}/multilib"-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group. View/Reply Online (#200485):
https://lists.openembedded.org/g/openembedded-core/message/200485
Mute This Topic: https://lists.openembedded.org/mt/106602515/3618414
Group Owner: [email protected] Unsubscribe:
https://lists.openembedded.org/g/openembedded-core/unsub [
[email protected]]
-=-=-=-=-=-=-=-=-=-=-=-
PATCH 2/6 classes/spdx-common: Move common SPDX to new class
In reply to Joshua WattMoves SPDX code that can be shared between different SPDX versions into a common class
--- meta/classes/create-spdx-2.2.bbclass | 259 ++------------------------- meta/classes/spdx-common.bbclass | 256 ++++++++++++++++++++++++++ 2 files changed, 266 insertions(+), 249 deletions(-) create mode 100644 meta/classes/spdx-common.bbclass
meta/classes/create-spdx-2.2.bbclass +10 -249
@@ -4,65 +4,13 @@ # SPDX-License-Identifier: GPL-2.0-only # -DEPLOY_DIR_SPDX ??= "${DEPLOY_DIR}/spdx" - -# The product name that the CVE database uses. Defaults to BPN, but may need to -# be overriden per recipe (for example tiff.bb sets CVE_PRODUCT=libtiff). -CVE_PRODUCT ??= "${BPN}" -CVE_VERSION ??= "${PV}" - -SPDXDIR ??= "${WORKDIR}/spdx" -SPDXDEPLOY = "${SPDXDIR}/deploy" -SPDXWORK = "${SPDXDIR}/work" -SPDXIMAGEWORK = "${SPDXDIR}/image-work" -SPDXSDKWORK = "${SPDXDIR}/sdk-work" -SPDXDEPS = "${SPDXDIR}/deps.json" - -SPDX_TOOL_NAME ??= "oe-spdx-creator" -SPDX_TOOL_VERSION ??= "1.0" - -SPDXRUNTIMEDEPLOY = "${SPDXDIR}/runtime-deploy" - -SPDX_INCLUDE_SOURCES ??= "0" -SPDX_ARCHIVE_SOURCES ??= "0" -SPDX_ARCHIVE_PACKAGED ??= "0" - -SPDX_UUID_NAMESPACE ??= "sbom.openembedded.org" -SPDX_NAMESPACE_PREFIX ??= "http://spdx.org/spdxdocs" -SPDX_PRETTY ??= "0" - -SPDX_LICENSES ??= "${COREBASE}/meta/files/spdx-licenses.json" - -SPDX_CUSTOM_ANNOTATION_VARS ??= "" - -SPDX_ORG ??= "OpenEmbedded ()" -SPDX_SUPPLIER ??= "Organization: ${SPDX_ORG}" -SPDX_SUPPLIER[doc] = "The SPDX PackageSupplier field for SPDX packages created from \ - this recipe. For SPDX documents create using this class during the build, this \ - is the contact information for the person or organization who is doing the \ - build." - -def extract_licenses(filename): - import re - - lic_regex = re.compile(rb'^\W*SPDX-License-Identifier:\s*([ \w\d.()+-]+?)(?:\s+\W*)?$', re.MULTILINE) - - try: - with open(filename, 'rb') as f: - size = min(15000, os.stat(filename).st_size) - txt = f.read(size) - licenses = re.findall(lic_regex, txt) - if licenses: - ascii_licenses = [lic.decode('ascii') for lic in licenses] - return ascii_licenses - except Exception as e: - bb.warn(f"Exception reading {filename}: {e}") - return None - -def get_doc_namespace(d, doc): +inherit spdx-common + +def get_namespace(d, name): import uuid namespace_uuid = uuid.uuid5(uuid.NAMESPACE_DNS, d.getVar("SPDX_UUID_NAMESPACE")) - return "%s/%s-%s" % (d.getVar("SPDX_NAMESPACE_PREFIX"), doc.name, str(uuid.uuid5(namespace_uuid, doc.name))) + return "%s/%s-%s" % (d.getVar("SPDX_NAMESPACE_PREFIX"), name, str(uuid.uuid5(namespace_uuid, name))) + def create_annotation(d, comment): from datetime import datetime, timezone
@@ -80,26 +28,6 @@ def recipe_spdx_is_native(d, recipe): a.annotator == "Tool: %s - %s" % (d.getVar("SPDX_TOOL_NAME"), d.getVar("SPDX_TOOL_VERSION")) and a.comment == "isNative" for a in recipe.annotations) -def is_work_shared_spdx(d): - return bb.data.inherits_class('kernel', d) or ('work-shared' in d.getVar('WORKDIR')) - -def get_json_indent(d): - if d.getVar("SPDX_PRETTY") == "1": - return 2 - return None - -python() { - import json - if d.getVar("SPDX_LICENSE_DATA"): - return - - with open(d.getVar("SPDX_LICENSES"), "r") as f: - data = json.load(f) - # Transform the license array to a dictionary - data["licenses"] = {l["licenseId"]: l for l in data["licenses"]} - d.setVar("SPDX_LICENSE_DATA", data) -} - def convert_license_to_spdx(lic, document, d, existing={}): from pathlib import Path import oe.spdx
@@ -172,34 +100,6 @@ def convert_license_to_spdx(lic, document, d, existing={}): return ' '.join(convert(l) for l in lic_split) -def process_sources(d): - pn = d.getVar('PN') - assume_provided = (d.getVar("ASSUME_PROVIDED") or "").split() - if pn in assume_provided: - for p in d.getVar("PROVIDES").split(): - if p != pn: - pn = p - break - - # glibc-locale: do_fetch, do_unpack and do_patch tasks have been deleted, - # so avoid archiving source here. - if pn.startswith('glibc-locale'): - return False - if d.getVar('PN') == "libtool-cross": - return False - if d.getVar('PN') == "libgcc-initial": - return False - if d.getVar('PN') == "shadow-sysroot": - return False - - # We just archive gcc-source for all the gcc related recipes - if d.getVar('BPN') in ['gcc', 'libgcc']: - bb.debug(1, 'spdx: There is bug in scan of %s is, do nothing' % pn) - return False - - return True - - def add_package_files(d, doc, spdx_pkg, topdir, get_spdxid, get_types, *, archive=None, ignore_dirs=[], ignore_top_level_dirs=[]): from pathlib import Path import oe.spdx
@@ -348,14 +248,12 @@ def collect_dep_recipes(d, doc, spdx_recipe): import oe.spdx deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX")) - spdx_deps_file = Path(d.getVar("SPDXDEPS")) package_archs = d.getVar("SSTATE_ARCHS").split() package_archs.reverse() dep_recipes = [] - with spdx_deps_file.open("r") as f: - deps = json.load(f) + deps = get_spdx_deps(d) for dep_pn, dep_hashfn, in_taskhash in deps: # If this dependency is not calculated in the taskhash skip it.
@@ -468,51 +366,6 @@ def add_download_packages(d, doc, recipe): # but this should be sufficient for now doc.add_relationship(package, "BUILD_DEPENDENCY_OF", recipe) -def collect_direct_deps(d, dep_task): - current_task = "do_" + d.getVar("BB_CURRENTTASK") - pn = d.getVar("PN") - - taskdepdata = d.getVar("BB_TASKDEPDATA", False) - - for this_dep in taskdepdata.values(): - if this_dep[0] == pn and this_dep[1] == current_task: - break - else: - bb.fatal(f"Unable to find this {pn}:{current_task} in taskdepdata") - - deps = set() - for dep_name in this_dep[3]: - dep_data = taskdepdata[dep_name] - if dep_data[1] == dep_task and dep_data[0] != pn: - deps.add((dep_data[0], dep_data[7], dep_name in this_dep[8])) - - return sorted(deps) - -collect_direct_deps[vardepsexclude] += "BB_TASKDEPDATA" -collect_direct_deps[vardeps] += "DEPENDS" - -python do_collect_spdx_deps() { - # This task calculates the build time dependencies of the recipe, and is - # required because while a task can deptask on itself, those dependencies - # do not show up in BB_TASKDEPDATA. To work around that, this task does the - # deptask on do_create_spdx and writes out the dependencies it finds, then - # do_create_spdx reads in the found dependencies when writing the actual - # SPDX document - import json - from pathlib import Path - - spdx_deps_file = Path(d.getVar("SPDXDEPS")) - - deps = collect_direct_deps(d, "do_create_spdx") - - with spdx_deps_file.open("w") as f: - json.dump(deps, f) -} -# NOTE: depending on do_unpack is a hack that is necessary to get it's dependencies for archive the source -addtask do_collect_spdx_deps after do_unpack -do_collect_spdx_deps[depends] += "${PATCHDEPENDENCY}" -do_collect_spdx_deps[deptask] = "do_create_spdx" -do_collect_spdx_deps[dirs] = "${SPDXDIR}" python do_create_spdx() { from datetime import datetime, timezone
@@ -551,7 +404,7 @@ python do_create_spdx() { doc = oe.spdx.SPDXDocument() doc.name = "recipe-" + d.getVar("PN") - doc.documentNamespace = get_doc_namespace(d, doc) + doc.documentNamespace = get_namespace(d, doc.name) doc.creationInfo.created = creation_time doc.creationInfo.comment = "This document was created by analyzing recipe files during the build." doc.creationInfo.licenseListVersion = d.getVar("SPDX_LICENSE_DATA")["licenseListVersion"]
@@ -655,7 +508,7 @@ python do_create_spdx() { package_doc = oe.spdx.SPDXDocument() pkg_name = d.getVar("PKG:%s" % package) or package package_doc.name = pkg_name - package_doc.documentNamespace = get_doc_namespace(d, package_doc) + package_doc.documentNamespace = get_namespace(d, package_doc.name) package_doc.creationInfo.created = creation_time package_doc.creationInfo.comment = "This document was created by analyzing packages created during the build." package_doc.creationInfo.licenseListVersion = d.getVar("SPDX_LICENSE_DATA")["licenseListVersion"]
@@ -716,44 +569,6 @@ do_create_spdx[dirs] = "${SPDXWORK}" do_create_spdx[cleandirs] = "${SPDXDEPLOY} ${SPDXWORK}" do_create_spdx[depends] += "${PATCHDEPENDENCY}" -def collect_package_providers(d): - from pathlib import Path - import oe.sbom - import oe.spdx - import json - - deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX")) - - providers = {} - - deps = collect_direct_deps(d, "do_create_spdx") - deps.append((d.getVar("PN"), d.getVar("BB_HASHFILENAME"), True)) - - for dep_pn, dep_hashfn, _ in deps: - localdata = d - recipe_data = oe.packagedata.read_pkgdata(dep_pn, localdata) - if not recipe_data: - localdata = bb.data.createCopy(d) - localdata.setVar("PKGDATA_DIR", "${PKGDATA_DIR_SDK}") - recipe_data = oe.packagedata.read_pkgdata(dep_pn, localdata) - - for pkg in recipe_data.get("PACKAGES", "").split(): - - pkg_data = oe.packagedata.read_subpkgdata_dict(pkg, localdata) - rprovides = set(n for n, _ in bb.utils.explode_dep_versions2(pkg_data.get("RPROVIDES", "")).items()) - rprovides.add(pkg) - - if "PKG" in pkg_data: - pkg = pkg_data["PKG"] - rprovides.add(pkg) - - for r in rprovides: - providers[r] = (pkg, dep_hashfn) - - return providers - -collect_package_providers[vardepsexclude] += "BB_TASKDEPDATA" - python do_create_runtime_spdx() { from datetime import datetime, timezone import oe.sbom
@@ -800,7 +615,7 @@ python do_create_runtime_spdx() { runtime_doc = oe.spdx.SPDXDocument() runtime_doc.name = "runtime-" + pkg_name - runtime_doc.documentNamespace = get_doc_namespace(localdata, runtime_doc) + runtime_doc.documentNamespace = get_namespace(localdata, runtime_doc.name) runtime_doc.creationInfo.created = creation_time runtime_doc.creationInfo.comment = "This document was created by analyzing package runtime dependencies." runtime_doc.creationInfo.licenseListVersion = d.getVar("SPDX_LICENSE_DATA")["licenseListVersion"]
@@ -891,60 +706,6 @@ do_create_runtime_spdx[dirs] = "${SPDXRUNTIMEDEPLOY}" do_create_runtime_spdx[cleandirs] = "${SPDXRUNTIMEDEPLOY}" do_create_runtime_spdx[rdeptask] = "do_create_spdx" -def spdx_get_src(d): - """ - save patched source of the recipe in SPDX_WORKDIR. - """ - import shutil - spdx_workdir = d.getVar('SPDXWORK') - spdx_sysroot_native = d.getVar('STAGING_DIR_NATIVE') - pn = d.getVar('PN') - - workdir = d.getVar("WORKDIR") - - try: - # The kernel class functions require it to be on work-shared, so we dont change WORKDIR - if not is_work_shared_spdx(d): - # Change the WORKDIR to make do_unpack do_patch run in another dir. - d.setVar('WORKDIR', spdx_workdir) - # Restore the original path to recipe's native sysroot (it's relative to WORKDIR). - d.setVar('STAGING_DIR_NATIVE', spdx_sysroot_native) - - # The changed 'WORKDIR' also caused 'B' changed, create dir 'B' for the - # possibly requiring of the following tasks (such as some recipes's - # do_patch required 'B' existed). - bb.utils.mkdirhier(d.getVar('B')) - - bb.build.exec_func('do_unpack', d) - # Copy source of kernel to spdx_workdir - if is_work_shared_spdx(d): - share_src = d.getVar('WORKDIR') - d.setVar('WORKDIR', spdx_workdir) - d.setVar('STAGING_DIR_NATIVE', spdx_sysroot_native) - src_dir = spdx_workdir + "/" + d.getVar('PN')+ "-" + d.getVar('PV') + "-" + d.getVar('PR') - bb.utils.mkdirhier(src_dir) - if bb.data.inherits_class('kernel',d): - share_src = d.getVar('STAGING_KERNEL_DIR') - cmd_copy_share = "cp -rf " + share_src + "/* " + src_dir + "/" - cmd_copy_shared_res = os.popen(cmd_copy_share).read() - bb.note("cmd_copy_shared_result = " + cmd_copy_shared_res) - - git_path = src_dir + "/.git" - if os.path.exists(git_path): - shutils.rmtree(git_path) - - # Make sure gcc and kernel sources are patched only once - if not (d.getVar('SRC_URI') == "" or is_work_shared_spdx(d)): - bb.build.exec_func('do_patch', d) - - # Some userland has no source. - if not os.path.exists( spdx_workdir ): - bb.utils.mkdirhier(spdx_workdir) - finally: - d.setVar("WORKDIR", workdir) - -spdx_get_src[vardepsexclude] += "STAGING_KERNEL_DIR" - do_rootfs[recrdeptask] += "do_create_spdx do_create_runtime_spdx" do_rootfs[cleandirs] += "${SPDXIMAGEWORK}"
@@ -1019,7 +780,7 @@ def combine_spdx(d, rootfs_name, rootfs_deploydir, rootfs_spdxid, packages, spdx doc = oe.spdx.SPDXDocument() doc.name = rootfs_name - doc.documentNamespace = get_doc_namespace(d, doc) + doc.documentNamespace = get_namespace(d, doc.name) doc.creationInfo.created = creation_time doc.creationInfo.comment = "This document was created by analyzing the source of the Yocto recipe during the build." doc.creationInfo.licenseListVersion = d.getVar("SPDX_LICENSE_DATA")["licenseListVersion"]
meta/classes/spdx-common.bbclass new file +256 -0
256 lines not on this page · raw message
Signature · 1 line
2.43.2
Re: [OE-core][PATCH 2/6] classes/spdx-common: Move common SPDX to new class
Quoted 378 lines · Joshua Watt via lists.openembedded.org
Moves SPDX code that can be shared between different SPDX versions into
a common classSigned-off-by Joshua Watt <[email protected]>---
meta/classes/create-spdx-2.2.bbclass | 259 ++----------------------- --
meta/classes/spdx-common.bbclass | 256
++++++++++++++++++++++++++2 files changed, 266 insertions(+), 249 deletions(-)create mode 100644 meta/classes/spdx-common.bbclass
meta/classes/create-spdx-2.2.bbclass +0
-0@@ -4,65 +4,13 @@ # SPDX-License-Identifier: GPL-2.0-only #meta/classes/create-spdx-2.2.bbclass +0
-3-DEPLOY_DIR_SPDX ??= "${DEPLOY_DIR}/spdx" - -# The product name that the CVE database uses. Defaults to BPN, butmay need to
-# be overriden per recipe (for example tiff.bb sets CVE_PRODUCT=libtiff).meta/classes/create-spdx-2.2.bbclass +0
-30-CVE_PRODUCT ??= "${BPN}" -CVE_VERSION ??= "${PV}" - -SPDXDIR ??= "${WORKDIR}/spdx" -SPDXDEPLOY = "${SPDXDIR}/deploy" -SPDXWORK = "${SPDXDIR}/work" -SPDXIMAGEWORK = "${SPDXDIR}/image-work" -SPDXSDKWORK = "${SPDXDIR}/sdk-work" -SPDXDEPS = "${SPDXDIR}/deps.json" - -SPDX_TOOL_NAME ??= "oe-spdx-creator" -SPDX_TOOL_VERSION ??= "1.0" - -SPDXRUNTIMEDEPLOY = "${SPDXDIR}/runtime-deploy" - -SPDX_INCLUDE_SOURCES ??= "0" -SPDX_ARCHIVE_SOURCES ??= "0" -SPDX_ARCHIVE_PACKAGED ??= "0" - -SPDX_UUID_NAMESPACE ??= "sbom.openembedded.org" -SPDX_NAMESPACE_PREFIX ??= "http://spdx.org/spdxdocs" -SPDX_PRETTY ??= "0" - -SPDX_LICENSES ??= "${COREBASE}/meta/files/spdx-licenses.json" - -SPDX_CUSTOM_ANNOTATION_VARS ??= "" - -SPDX_ORG ??= "OpenEmbedded ()" -SPDX_SUPPLIER ??= "Organization: ${SPDX_ORG}" -SPDX_SUPPLIER[doc] = "The SPDX PackageSupplier field for SPDXpackages created from \
- this recipe. For SPDX documents create using this class during the build, this \
- is the contact information for the person or organization who is doing the \
- build."meta/classes/create-spdx-2.2.bbclass +0
-5- -def extract_licenses(filename): - import re - - lic_regex = re.compile(rb'^\W*SPDX-License-Identifier:\s*([\w\d.()+-]+?)(?:\s+\W*)?$', re.MULTILINE)
-
- try:
- withopen(filename, 'rb')as f:
- size =min(15000, os.stat(filename).st_size)- txt =f.read(size)
- licenses =re.findall(lic_regex, txt)
- if licenses:
-ascii_licenses= [lic.decode('ascii')for lic in licenses]
- returnascii_licenses
- except Exception as e:
-bb.warn(f"Exception reading {filename}: {e}")- return Nonemeta/classes/create-spdx-2.2.bbclass +3
-2- -def get_doc_namespace(d, doc): +inherit spdx-common + +def get_namespace(d, name):import uuid
namespace_uuid=uuid.uuid5(uuid.NAMESPACE_DNS, d.getVar("SPDX_UUID_NAMESPACE"))
- return "%s/%s-%s" % (d.getVar("SPDX_NAMESPACE_PREFIX"), doc.name,str(uuid.uuid5(namespace_uuid, doc.name)))
+ return "%s/%s-%s" % (d.getVar("SPDX_NAMESPACE_PREFIX"), name,str(uuid.uuid5(namespace_uuid, name)))
+
defcreate_annotation(d, comment):
from datetime import datetime, timezonemeta/classes/create-spdx-2.2.bbclass +0
-0@@ -80,26 +28,6 @@ def recipe_spdx_is_native(d, recipe): a.annotator == "Tool: %s - %s" % (d.getVar("SPDX_TOOL_NAME"),
d.getVar("SPDX_TOOL_VERSION")) and
a.comment == "isNative" for a in recipe.annotations)
-defis_work_shared_spdx(d):
- returnbb.data.inherits_class('kernel', d)or ('work-shared' ind.getVar('WORKDIR'))meta/classes/create-spdx-2.2.bbclass +0
-15- -def get_json_indent(d): - if d.getVar("SPDX_PRETTY") == "1": - return 2 - return None - -python() { - import json - if d.getVar("SPDX_LICENSE_DATA"): - return - - with open(d.getVar("SPDX_LICENSES"), "r") as f: - data = json.load(f) - # Transform the license array to a dictionary - data["licenses"] = {l["licenseId"]: l for l indata["licenses"]}
-d.setVar("SPDX_LICENSE_DATA", data)meta/classes/create-spdx-2.2.bbclass +0
-2-} -def
convert_license_to_spdx(lic, document, d, existing={}):
from pathlib import Path
import oe.spdxmeta/classes/create-spdx-2.2.bbclass +0
-0@@ -172,34 +100,6 @@ def convert_license_to_spdx(lic, document, d,existing={}):
return ' '.join(convert(l) for l inlic_split)meta/classes/create-spdx-2.2.bbclass +0
-10-def process_sources(d): - pn = d.getVar('PN') - assume_provided = (d.getVar("ASSUME_PROVIDED") or "").split() - if pn in assume_provided: - for p in d.getVar("PROVIDES").split(): - if p != pn: - pn = p - break - - # glibc-locale: do_fetch, do_unpack and do_patch tasks have beendeleted,
- # so avoid archiving source here.
- ifpn.startswith('glibc-locale'):
- return False
- ifd.getVar('PN')== "libtool-cross":
- return False
- ifd.getVar('PN')== "libgcc-initial":
- return False
- ifd.getVar('PN')== "shadow-sysroot":
- return False
-
- # We just archive gcc-source for all the gcc related recipes - ifd.getVar('BPN')in ['gcc', 'libgcc']:
-bb.debug(1, 'spdx: There is bug in scan of %s is, do nothing' % pn)
- return Falsemeta/classes/create-spdx-2.2.bbclass +0
-4- - return True - -def add_package_files(d, doc,
spdx_pkg, topdir,get_spdxid,get_types, *, archive=None, ignore_dirs=[], ignore_top_level_dirs=[]):
from pathlib import Path
import oe.spdxmeta/classes/create-spdx-2.2.bbclass +0
-0@@ -348,14 +248,12 @@ def collect_dep_recipes(d, doc, spdx_recipe):import oe.spdx
deploy_dir_spdx= Path(d.getVar("DEPLOY_DIR_SPDX")) -spdx_deps_file= Path(d.getVar("SPDXDEPS"))package_archs= d.getVar("SSTATE_ARCHS").split()package_archs.reverse()
dep_recipes= []
- withspdx_deps_file.open("r")as f:
- deps =json.load(f)
+ deps =get_spdx_deps(d)
fordep_pn,dep_hashfn,in_taskhashin deps:
# If this dependency is not calculated in the taskhash skip it.meta/classes/create-spdx-2.2.bbclass +0
-0@@ -468,51 +366,6 @@ def add_download_packages(d, doc, recipe): # but this should be sufficient for now doc.add_relationship(package, "BUILD_DEPENDENCY_OF", recipe)meta/classes/create-spdx-2.2.bbclass +0
-11-def collect_direct_deps(d, dep_task): - current_task = "do_" + d.getVar("BB_CURRENTTASK") - pn = d.getVar("PN") - - taskdepdata = d.getVar("BB_TASKDEPDATA", False) - - for this_dep in taskdepdata.values(): - if this_dep[0] == pn and this_dep[1] == current_task: - break - else: - bb.fatal(f"Unable to find this {pn}:{current_task} intaskdepdata")
-
- deps =set()
- fordep_namein this_dep[3]:
-dep_data= taskdepdata[dep_name]
- if dep_data[1] ==dep_taskand dep_data[0] != pn:
-deps.add((dep_data[0], dep_data[7], dep_name in this_dep[8]))meta/classes/create-spdx-2.2.bbclass +0
-8- - return sorted(deps) - -collect_direct_deps[vardepsexclude] += "BB_TASKDEPDATA" -collect_direct_deps[vardeps] += "DEPENDS" - -python do_collect_spdx_deps() { - # This task calculates the build time dependencies of therecipe, and is
- # required because while a task can deptask on itself, those dependencies
- # do not show up inBB_TASKDEPDATA. To work around that, this task does the
- # deptask ondo_create_spdxand writes out the dependencies it finds, then
- #do_create_spdxreads in the found dependencies when writing the actual
- # SPDX document
- import json
- from pathlib import Pathmeta/classes/create-spdx-2.2.bbclass +0
-9- - spdx_deps_file = Path(d.getVar("SPDXDEPS")) - - deps = collect_direct_deps(d, "do_create_spdx") - - with spdx_deps_file.open("w") as f: - json.dump(deps, f) -} -# NOTE: depending on do_unpack is a hack that is necessary to getit's dependencies for archive the source
meta/classes/create-spdx-2.2.bbclass +0
-4-addtask do_collect_spdx_deps after do_unpack -do_collect_spdx_deps[depends] += "${PATCHDEPENDENCY}" -do_collect_spdx_deps[deptask] = "do_create_spdx" -do_collect_spdx_deps[dirs] = "${SPDXDIR}"
pythondo_create_spdx(){
from datetime import datetime, timezonemeta/classes/create-spdx-2.2.bbclass +0
-0@@ -551,7 +404,7 @@ python do_create_spdx() { doc = oe.spdx.SPDXDocument()
doc.name = "recipe-" +d.getVar("PN")
- doc.documentNamespace =get_doc_namespace(d, doc)+ doc.documentNamespace =get_namespace(d, doc.name)doc.creationInfo.created =creation_timedoc.creationInfo.comment = "This document was created by analyzing recipe files during the build." doc.creationInfo.licenseListVersion = d.getVar("SPDX_LICENSE_DATA")["licenseListVersion"]meta/classes/create-spdx-2.2.bbclass +0
-0@@ -655,7 +508,7 @@ python do_create_spdx() { package_doc = oe.spdx.SPDXDocument()
pkg_name=d.getVar("PKG:%s" % package)or package package_doc.name =pkg_name
- package_doc.documentNamespace =get_doc_namespace(d, package_doc)
+ package_doc.documentNamespace =get_namespace(d, package_doc.name)
package_doc.creationInfo.created =creation_timepackage_doc.creationInfo.comment = "This document was created by analyzing packages created during the build." package_doc.creationInfo.licenseListVersion = d.getVar("SPDX_LICENSE_DATA")["licenseListVersion"]meta/classes/create-spdx-2.2.bbclass +0
-0@@ -716,44 +569,6 @@ do_create_spdx[dirs] = "${SPDXWORK}"do_create_spdx[cleandirs] = "${SPDXDEPLOY} ${SPDXWORK}" do_create_spdx[depends] += "${PATCHDEPENDENCY}"
meta/classes/create-spdx-2.2.bbclass +0
-20-def collect_package_providers(d): - from pathlib import Path - import oe.sbom - import oe.spdx - import json - - deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX")) - - providers = {} - - deps = collect_direct_deps(d, "do_create_spdx") - deps.append((d.getVar("PN"), d.getVar("BB_HASHFILENAME"), True)) - - for dep_pn, dep_hashfn, _ in deps: - localdata = d - recipe_data = oe.packagedata.read_pkgdata(dep_pn, localdata) - if not recipe_data: - localdata = bb.data.createCopy(d) - localdata.setVar("PKGDATA_DIR", "${PKGDATA_DIR_SDK}") - recipe_data = oe.packagedata.read_pkgdata(dep_pn,localdata)
meta/classes/create-spdx-2.2.bbclass +0
-4- - for pkg in recipe_data.get("PACKAGES", "").split(): - - pkg_data = oe.packagedata.read_subpkgdata_dict(pkg,localdata)
- rprovides =set(n for n, _ in bb.utils.explode_dep_versions2(pkg_data.get("RPROVIDES", "")).items())
-rprovides.add(pkg)meta/classes/create-spdx-2.2.bbclass +0
-12- - if "PKG" in pkg_data: - pkg = pkg_data["PKG"] - rprovides.add(pkg) - - for r in rprovides: - providers[r] = (pkg, dep_hashfn) - - return providers - -collect_package_providers[vardepsexclude] += "BB_TASKDEPDATA" -python
do_create_runtime_spdx(){
from datetime import datetime, timezone
import oe.sbommeta/classes/create-spdx-2.2.bbclass +0
-0@@ -800,7 +615,7 @@ python do_create_runtime_spdx() {
runtime_doc=oe.spdx.SPDXDocument()runtime_doc.name = "runtime-" +pkg_name
- runtime_doc.documentNamespace =get_doc_namespace(localdata, runtime_doc)
+ runtime_doc.documentNamespace =get_namespace(localdata, runtime_doc.name)
runtime_doc.creationInfo.created =creation_timeruntime_doc.creationInfo.comment = "This document was created by analyzing package runtime dependencies." runtime_doc.creationInfo.licenseListVersion = d.getVar("SPDX_LICENSE_DATA")["licenseListVersion"]meta/classes/create-spdx-2.2.bbclass +0
-0@@ -891,60 +706,6 @@ do_create_runtime_spdx[dirs] ="${SPDXRUNTIMEDEPLOY}" do_create_runtime_spdx[cleandirs] = "${SPDXRUNTIMEDEPLOY}" do_create_runtime_spdx[rdeptask] = "
do_create_spdx"meta/classes/create-spdx-2.2.bbclass +0
-13-def spdx_get_src(d): - """ - save patched source of the recipe in SPDX_WORKDIR. - """ - import shutil - spdx_workdir = d.getVar('SPDXWORK') - spdx_sysroot_native = d.getVar('STAGING_DIR_NATIVE') - pn = d.getVar('PN') - - workdir = d.getVar("WORKDIR") - - try: - # The kernel class functions require it to be on work-shared, so we dont change WORKDIR
- if notis_work_shared_spdx(d):
- # Change the WORKDIR to makedo_unpackdo_patchrun in another dir.
-d.setVar('WORKDIR', spdx_workdir)
- # Restore the original path to recipe's native sysroot (it's relative to WORKDIR).
-d.setVar('STAGING_DIR_NATIVE', spdx_sysroot_native)-
- # The changed 'WORKDIR' also caused 'B' changed, create dir 'B' for the
- # possibly requiring of the following tasks (such as some recipes's
- #do_patchrequired 'B' existed).
- bb.utils.mkdirhier(d.getVar('B'))
-
-bb.build.exec_func('do_unpack', d)
- # Copy source of kernel tospdx_workdir
- ifis_work_shared_spdx(d):
-share_src=d.getVar('WORKDIR')
-d.setVar('WORKDIR', spdx_workdir)
-d.setVar('STAGING_DIR_NATIVE', spdx_sysroot_native)-src_dir=spdx_workdir+ "/" + d.getVar('PN')+ "-" +d.getVar('PV')+ "-" +d.getVar('PR')
-bb.utils.mkdirhier(src_dir)
- ifbb.data.inherits_class('kernel',d):
-share_src=d.getVar('STAGING_KERNEL_DIR')
Quoted 8 lines
-
cmd_copy_share= "cp -rf " +share_src+ "/* " +src_dir
+ "/"
-cmd_copy_shared_res= os.popen(cmd_copy_share).read() -bb.note("cmd_copy_shared_result = " + cmd_copy_shared_res)
-
-git_path=src_dir+ "/.git"
- ifos.path.exists(git_path):
import os.path is missing
Quoted 1 line
-
shutils.rmtree(git_path)
Should be: shutil.rmtree(git_path)
Thank you for SPDX 3.0 support.
Adrian
Quoted 336 lines
-
- # Make sure gcc and kernel sources are patched only once - if not (d.getVar('SRC_URI')== "" oris_work_shared_spdx(d)):
-bb.build.exec_func('do_patch', d)meta/classes/create-spdx-2.2.bbclass +0
-9- - # Some userland has no source. - if not os.path.exists( spdx_workdir ): - bb.utils.mkdirhier(spdx_workdir) - finally: - d.setVar("WORKDIR", workdir) - -spdx_get_src[vardepsexclude] += "STAGING_KERNEL_DIR" -do_rootfs[recrdeptask] += "
do_create_spdxdo_create_runtime_spdx" do_rootfs[cleandirs] += "${SPDXIMAGEWORK}"meta/classes/create-spdx-2.2.bbclass +0
-0@@ -1019,7 +780,7 @@ def combine_spdx(d, rootfs_name,
rootfs_deploydir,rootfs_spdxid, packages, spdx
doc =oe.spdx.SPDXDocument()
doc.name =rootfs_name
- doc.documentNamespace =get_doc_namespace(d, doc)+ doc.documentNamespace =get_namespace(d, doc.name)doc.creationInfo.created =creation_timedoc.creationInfo.comment = "This document was created by analyzing the source of the Yocto recipe during the build." doc.creationInfo.licenseListVersion = d.getVar("SPDX_LICENSE_DATA")["licenseListVersion"]meta/classes/spdx- renamed from meta/classes/spdx-common.bbclass +0
-0common.bbclass
new file mode 100644
index 00000000000..468a11ca3e2meta/classes/spdx-common.bbclass new file +9
-0@@ -0,0 +1,256 @@ +# +# Copyright OpenEmbedded Contributors +# +# SPDX-License-Identifier: GPL-2.0-only +# + +DEPLOY_DIR_SPDX ??= "${DEPLOY_DIR}/spdx" + +# The product name that the CVE database uses. Defaults to BPN, butmay need to
+# be overriden per recipe (for example tiff.bb sets CVE_PRODUCT=libtiff).meta/classes/spdx-common.bbclass new file +30
-0+CVE_PRODUCT ??= "${BPN}" +CVE_VERSION ??= "${PV}" + +SPDXDIR ??= "${WORKDIR}/spdx" +SPDXDEPLOY = "${SPDXDIR}/deploy" +SPDXWORK = "${SPDXDIR}/work" +SPDXIMAGEWORK = "${SPDXDIR}/image-work" +SPDXSDKWORK = "${SPDXDIR}/sdk-work" +SPDXDEPS = "${SPDXDIR}/deps.json" + +SPDX_TOOL_NAME ??= "oe-spdx-creator" +SPDX_TOOL_VERSION ??= "1.0" + +SPDXRUNTIMEDEPLOY = "${SPDXDIR}/runtime-deploy" + +SPDX_INCLUDE_SOURCES ??= "0" +SPDX_ARCHIVE_SOURCES ??= "0" +SPDX_ARCHIVE_PACKAGED ??= "0" + +SPDX_UUID_NAMESPACE ??= "sbom.openembedded.org" +SPDX_NAMESPACE_PREFIX ??= "http://spdx.org/spdxdocs" +SPDX_PRETTY ??= "0" + +SPDX_LICENSES ??= "${COREBASE}/meta/files/spdx-licenses.json" + +SPDX_CUSTOM_ANNOTATION_VARS ??= "" + +SPDX_ORG ??= "OpenEmbedded ()" +SPDX_SUPPLIER ??= "Organization: ${SPDX_ORG}" +SPDX_SUPPLIER[doc] = "The SPDX PackageSupplier field for SPDXpackages created from \
+ this recipe. For SPDX documents create using this class during the build, this \
+ is the contact information for the person or organization who is doing the \
+ build."meta/classes/spdx-common.bbclass new file +5
-0+ +def extract_licenses(filename): + import re + + lic_regex = re.compile(rb'^\W*SPDX-License-Identifier:\s*([\w\d.()+-]+?)(?:\s+\W*)?$', re.MULTILINE)
+
+ try:
+ withopen(filename, 'rb')as f:
+ size =min(15000, os.stat(filename).st_size)+ txt =f.read(size)
+ licenses =re.findall(lic_regex, txt)
+ if licenses:
+ascii_licenses= [lic.decode('ascii')for lic in licenses]
+ returnascii_licenses
+ except Exception as e:
+bb.warn(f"Exception reading {filename}: {e}")+ return Nonemeta/classes/spdx-common.bbclass new file +3
-0+ +def is_work_shared_spdx(d): + return bb.data.inherits_class('kernel', d) or ('work-shared' in
d.getVar('WORKDIR'))meta/classes/spdx-common.bbclass new file +15
-0+ +def get_json_indent(d): + if d.getVar("SPDX_PRETTY") == "1": + return 2 + return None + +python() { + import json + if d.getVar("SPDX_LICENSE_DATA"): + return + + with open(d.getVar("SPDX_LICENSES"), "r") as f: + data = json.load(f) + # Transform the license array to a dictionary + data["licenses"] = {l["licenseId"]: l for l indata["licenses"]}
+d.setVar("SPDX_LICENSE_DATA", data)meta/classes/spdx-common.bbclass new file +12
-0+} + +def process_sources(d): + pn = d.getVar('PN') + assume_provided = (d.getVar("ASSUME_PROVIDED") or "").split() + if pn in assume_provided: + for p in d.getVar("PROVIDES").split(): + if p != pn: + pn = p + break + + # glibc-locale: do_fetch, do_unpack and do_patch tasks have beendeleted,
+ # so avoid archiving source here.
+ ifpn.startswith('glibc-locale'):
+ return False
+ ifd.getVar('PN')== "libtool-cross":
+ return False
+ ifd.getVar('PN')== "libgcc-initial":
+ return False
+ ifd.getVar('PN')== "shadow-sysroot":
+ return False
+
+ # We just archive gcc-source for all the gcc related recipes + ifd.getVar('BPN')in ['gcc', 'libgcc']:
+bb.debug(1, 'spdx: There is bug in scan of %s is, do nothing' % pn)
+ return Falsemeta/classes/spdx-common.bbclass new file +14
-0+ + return True + +def collect_direct_deps(d, dep_task): + current_task = "do_" + d.getVar("BB_CURRENTTASK") + pn = d.getVar("PN") + + taskdepdata = d.getVar("BB_TASKDEPDATA", False) + + for this_dep in taskdepdata.values(): + if this_dep[0] == pn and this_dep[1] == current_task: + break + else: + bb.fatal(f"Unable to find this {pn}:{current_task} intaskdepdata")
meta/classes/spdx-common.bbclass new file +7
-0+ + deps = set() + + for dep_name in this_dep.deps: + dep_data = taskdepdata[dep_name] + if dep_data.taskname == dep_task and dep_data.pn != pn: + deps.add((dep_data.pn, dep_data.hashfn, dep_name inthis_dep.taskhash_deps))
meta/classes/spdx-common.bbclass new file +8
-0+ + return sorted(deps) + +collect_direct_deps[vardepsexclude] += "BB_TASKDEPDATA" +collect_direct_deps[vardeps] += "DEPENDS" + +python do_collect_spdx_deps() { + # This task calculates the build time dependencies of therecipe, and is
+ # required because while a task can deptask on itself, those dependencies
+ # do not show up inBB_TASKDEPDATA. To work around that, this task does the
+ # deptask ondo_create_spdxand writes out the dependencies it finds, then
+ #do_create_spdxreads in the found dependencies when writing the actual
+ # SPDX document
+ import json
+ from pathlib import Pathmeta/classes/spdx-common.bbclass new file +9
-0+ + spdx_deps_file = Path(d.getVar("SPDXDEPS")) + + deps = collect_direct_deps(d, "do_create_spdx") + + with spdx_deps_file.open("w") as f: + json.dump(deps, f) +} +# NOTE: depending on do_unpack is a hack that is necessary to getit's dependencies for archive the source
meta/classes/spdx-common.bbclass new file +34
-0+addtask do_collect_spdx_deps after do_unpack +do_collect_spdx_deps[depends] += "${PATCHDEPENDENCY}" +do_collect_spdx_deps[deptask] = "do_create_spdx" +do_collect_spdx_deps[dirs] = "${SPDXDIR}" + +def get_spdx_deps(d): + import json + from pathlib import Path + + spdx_deps_file = Path(d.getVar("SPDXDEPS")) + + with spdx_deps_file.open("r") as f: + return json.load(f) + +def collect_package_providers(d): + from pathlib import Path + import oe.sbom + import oe.spdx + import json + + deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX")) + + providers = {} + + deps = collect_direct_deps(d, "do_create_spdx") + deps.append((d.getVar("PN"), d.getVar("BB_HASHFILENAME"), True)) + + for dep_pn, dep_hashfn, _ in deps: + localdata = d + recipe_data = oe.packagedata.read_pkgdata(dep_pn, localdata) + if not recipe_data: + localdata = bb.data.createCopy(d) + localdata.setVar("PKGDATA_DIR", "${PKGDATA_DIR_SDK}") + recipe_data = oe.packagedata.read_pkgdata(dep_pn,localdata)
meta/classes/spdx-common.bbclass new file +4
-0+ + for pkg in recipe_data.get("PACKAGES", "").split(): + + pkg_data = oe.packagedata.read_subpkgdata_dict(pkg,localdata)
+ rprovides =set(n for n, _ in bb.utils.explode_dep_versions2(pkg_data.get("RPROVIDES", "")).items())
+rprovides.add(pkg)meta/classes/spdx-common.bbclass new file +25
-0+ + if "PKG" in pkg_data: + pkg = pkg_data["PKG"] + rprovides.add(pkg) + + for r in rprovides: + providers[r] = (pkg, dep_hashfn) + + return providers + +collect_package_providers[vardepsexclude] += "BB_TASKDEPDATA" + +def spdx_get_src(d): + """ + save patched source of the recipe in SPDX_WORKDIR. + """ + import shutil + spdx_workdir = d.getVar('SPDXWORK') + spdx_sysroot_native = d.getVar('STAGING_DIR_NATIVE') + pn = d.getVar('PN') + + workdir = d.getVar("WORKDIR") + + try: + # The kernel class functions require it to be on work-shared, so we dont change WORKDIR
+ if notis_work_shared_spdx(d):
+ # Change the WORKDIR to makedo_unpackdo_patchrun in another dir.
+d.setVar('WORKDIR', spdx_workdir)
+ # Restore the original path to recipe's native sysroot (it's relative to WORKDIR).
+d.setVar('STAGING_DIR_NATIVE', spdx_sysroot_native)+
+ # The changed 'WORKDIR' also caused 'B' changed, create dir 'B' for the
+ # possibly requiring of the following tasks (such as some recipes's
+ #do_patchrequired 'B' existed).
+ bb.utils.mkdirhier(d.getVar('B'))
+
+bb.build.exec_func('do_unpack', d)
+ # Copy source of kernel tospdx_workdir
+ ifis_work_shared_spdx(d):
+share_src=d.getVar('WORKDIR')
+d.setVar('WORKDIR', spdx_workdir)
+d.setVar('STAGING_DIR_NATIVE', spdx_sysroot_native)+src_dir=spdx_workdir+ "/" + d.getVar('PN')+ "-" +d.getVar('PV')+ "-" +d.getVar('PR')
+bb.utils.mkdirhier(src_dir)
+ ifbb.data.inherits_class('kernel',d):
+share_src=d.getVar('STAGING_KERNEL_DIR')+cmd_copy_share= "cp -rf " +share_src+ "/* " +src_dir
+ "/"
+cmd_copy_shared_res= os.popen(cmd_copy_share).read() +bb.note("cmd_copy_shared_result = " + cmd_copy_shared_res)meta/classes/spdx-common.bbclass new file +7
-0+ + git_path = src_dir + "/.git" + if os.path.exists(git_path): + shutils.rmtree(git_path) + + # Make sure gcc and kernel sources are patched only once + if not (d.getVar('SRC_URI') == "" or
is_work_shared_spdx(d)):
+bb.build.exec_func('do_patch', d)meta/classes/spdx-common.bbclass new file +9
-1+ + # Some userland has no source. + if not os.path.exists( spdx_workdir ): + bb.utils.mkdirhier(spdx_workdir) + finally: + d.setVar("WORKDIR", workdir) + +spdx_get_src[vardepsexclude] += "STAGING_KERNEL_DIR" + -=-=-=-=-=-=-=-=-=-=-=-Links: You receive all messages sent to this group. View/Reply Online (#200486):
https://lists.openembedded.org/g/openembedded-core/message/200486
Mute This Topic: https://lists.openembedded.org/mt/106602516/4454582
Group Owner: [email protected] Unsubscribe:
https://lists.openembedded.org/g/openembedded-core/unsub [
[email protected]]
-=-=-=-=-=-=-=-=-=-=-=-
PATCH 3/6 classes/spdx-common: Add SPDX version to path
In reply to Joshua WattSince multiple versions of SPDX are available, use the version in the path
--- meta/classes/create-spdx-2.2.bbclass | 2 ++ meta/classes/spdx-common.bbclass | 5 +++-- 2 files changed, 5 insertions(+), 2 deletions(-)
meta/classes/create-spdx-2.2.bbclass +2 -0
@@ -6,6 +6,8 @@ inherit spdx-common +SPDX_VERSION = "2.2" + def get_namespace(d, name): import uuid namespace_uuid = uuid.uuid5(uuid.NAMESPACE_DNS, d.getVar("SPDX_UUID_NAMESPACE"))
meta/classes/spdx-common.bbclass +3 -2
@@ -4,14 +4,15 @@ # SPDX-License-Identifier: GPL-2.0-only # -DEPLOY_DIR_SPDX ??= "${DEPLOY_DIR}/spdx" +SPDX_VERSION ??= "" +DEPLOY_DIR_SPDX ??= "${DEPLOY_DIR}/spdx/${SPDX_VERSION}" # The product name that the CVE database uses. Defaults to BPN, but may need to # be overriden per recipe (for example tiff.bb sets CVE_PRODUCT=libtiff). CVE_PRODUCT ??= "${BPN}" CVE_VERSION ??= "${PV}" -SPDXDIR ??= "${WORKDIR}/spdx" +SPDXDIR ??= "${WORKDIR}/spdx/${SPDX_VERSION}" SPDXDEPLOY = "${SPDXDIR}/deploy" SPDXWORK = "${SPDXDIR}/work" SPDXIMAGEWORK = "${SPDXDIR}/image-work"
Signature · 1 line
2.43.2
PATCH 4/6 classes/spdx-common: Return empty list from extract_licenses
In reply to Joshua WattThis is nicer as the normal return type is a list, so the calling code doesn't have to deal with a None sometimes and a list others.
--- meta/classes/spdx-common.bbclass | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
meta/classes/spdx-common.bbclass +1 -1
@@ -58,7 +58,7 @@ def extract_licenses(filename): return ascii_licenses except Exception as e: bb.warn(f"Exception reading {filename}: {e}") - return None + return [] def is_work_shared_spdx(d): return bb.data.inherits_class('kernel', d) or ('work-shared' in d.getVar('WORKDIR'))
Signature · 1 line
2.43.2
PATCH 5/6 classes/create-spdx-3.0: Add class
In reply to Joshua WattAdds a class to generate SPDX 3.0 output.
--- meta/classes/create-spdx-3.0.bbclass | 1284 ++++++ meta/classes/spdx-common.bbclass | 25 +- meta/lib/oe/sbom30.py | 993 +++++ meta/lib/oe/spdx30.py | 5413 ++++++++++++++++++++++++++ 4 files changed, 7714 insertions(+), 1 deletion(-) create mode 100644 meta/classes/create-spdx-3.0.bbclass create mode 100644 meta/lib/oe/sbom30.py create mode 100644 meta/lib/oe/spdx30.py
meta/classes/create-spdx-3.0.bbclass new file +1284 -0
@@ -0,0 +1,1284 @@ +# +# Copyright OpenEmbedded Contributors +# +# SPDX-License-Identifier: GPL-2.0-only +# + +inherit spdx-common + +SPDX_VERSION = "3.0.0" + +# The list of SPDX profiles generated documents will conform to +SPDX_PROFILES ?= "core build software simpleLicensing security" + +SPDX_INCLUDE_BUILD_VARIABLES ??= "0" +SPDX_INCLUDE_BUILD_VARIABLES[doc] = "If set to '1', the bitbake variables for a \ + recipe will be included in the Build object" + +SPDX_IMPORTS ??= "" +SPDX_IMPORTS[doc] = "SPDX_IMPORTS is the base variable that describes how to \ + reference external SPDX ids. Each import is defined as a key in this \ + variable with a suffix to describe to as a suffix to look up more \ + information about the import. Each key can have the following variables: \ + SPDX_IMPORTS_<key>_spdxid: The Fully qualified SPDX ID of the object \ + SPDX_IMPORTS_<key>_uri: The URI where the SPDX Document that contains \ + the external object can be found. Optional but recommended \ + SPDX_IMPORTS_<key>_hash_<hash>: The Checksum of the SPDX Document that \ + contains the External ID. <hash> must be one the valid SPDX hashing \ + algorithms, as described by the HashAlgorithm vocabulary in the\ + SPDX 3 spec. Optional but recommended \ +" + +# Agents +# Bitbake variables can be used to describe an SPDX Agent that may be used +# during the build. An Agent is specified using a set of variables which all +# start with some common base name: +# +# <BASE>_name: The name of the Agent (required) +# <BASE>_type: The type of Agent. Must be one of "person", "organization", +# "software", or "agent" (the default if not specified) +# <BASE>_comment: The comment for the Agent (optional) +# <BASE>_id_<ID>: And External Identifier for the Agent. <ID> must be a valid +# ExternalIdentifierType from the SPDX 3 spec. Commonly, an E-mail address +# can be specified with <BASE>_id_email +# +# Alternatively, an Agent can be an external reference by referencing a key +# in SPDX_IMPORTS like so: +# +# <BASE>_import = "<key>" +# +# Finally, the same agent described by another set of agent variables can be +# referenced by specifying the basename of the variable that should be +# referenced: +# +# SPDX_SUPPLIER_ref = "SPDX_AUTHORS_openembedded" + +SPDX_AUTHORS ??= "openembedded" +SPDX_AUTHORS[doc] = "A space separated list of the document authors. Each item \ + is used to name a base variable like SPDX_AUTHORS_<AUTHOR> that \ + describes the author." + +SPDX_AUTHORS_openembedded_name = "OpenEmbedded" +SPDX_AUTHORS_openembedded_type = "organization" + +SPDX_BUILD_HOST[doc] = "The base variable name to describe the build host on \ + which a build is running. Must be an SPDX_IMPORTS key" + +SPDX_INVOKED_BY[doc] = "The base variable name to describe the Agent that \ + invoked the build, which builds will link to if specified." + +SPDX_ON_BEHALF_OF[doc] = "The base variable name to describe the Agent on who's \ + behalf the invoking Agent (SPDX_INVOKED_BY) is running the build." + +SPDX_SUPPLIER[doc] = "The base variable name to describe the Agent who is supplying \ + artifacts produced by the build" + +SPDX_PACKAGE_ADDITIONAL_PURPOSE ?= "" +SPDX_PACKAGE_ADDITIONAL_PURPOSE[doc] = "The list of additional purposes to assign to \ + the generated packages for a recipe. The primary purpose is always `install`. \ + Packages overrides are allowed to override the additional purposes for \ + individual packages." + +SPDX_IMAGE_PURPOSE ?= "filesystemImage" +SPDX_IMAGE_PURPOSE[doc] = "The list of purposes to assign to the generated images. \ + The first listed item will be the Primary Purpose and all additional items will \ + be added as additional purposes" + +SPDX_SDK_PURPOSE ?= "install" +SPDX_SDK_PURPOSE[doc] = "The list of purposes to assign to the generate SDK installer. \ + The first listed item will be the Primary Purpose and all additional items will \ + be added as additional purposes" + +SPDX_HASH_ALGORITHMS ?= "sha256 sha1" + + +def add_license_expression(d, objset, license_expression): + from pathlib import Path + import oe.spdx30 + import oe.sbom30 + + license_data = d.getVar("SPDX_LICENSE_DATA") + simple_license_text = {} + license_text_map = {} + license_ref_idx = 0 + + def add_license_text(name): + nonlocal objset + nonlocal simple_license_text + + if name in simple_license_text: + return name + + lic = objset.find_filter( + oe.spdx30.simplelicensing_SimpleLicensingText, + name=name, + ) + + if lic is not None: + simple_license_text[name] = lic + return lic + + lic = objset.add(oe.spdx30.simplelicensing_SimpleLicensingText( + _id=objset.new_spdxid("license-text", name), + creationInfo=objset.doc.creationInfo, + name=name, + )) + simple_license_text[name] = lic + + if name == "PD": + lic.simplelicensing_licenseText = "Software released to the public domain" + return lic + + # Seach for the license in COMMON_LICENSE_DIR and LICENSE_PATH + for directory in [d.getVar('COMMON_LICENSE_DIR')] + (d.getVar('LICENSE_PATH') or '').split(): + try: + with (Path(directory) / name).open(errors="replace") as f: + lic.simplelicensing_licenseText = f.read() + return lic + + except FileNotFoundError: + pass + + # If it's not SPDX or PD, then NO_GENERIC_LICENSE must be set + filename = d.getVarFlag('NO_GENERIC_LICENSE', name) + if filename: + filename = d.expand("${S}/" + filename) + with open(filename, errors="replace") as f: + lic.simplelicensing_licenseText = f.read() + return lic + else: + bb.fatal("Cannot find any text for license %s" % name) + + def convert(l): + nonlocal license_text_map + nonlocal license_ref_idx + + if l == "(" or l == ")": + return l + + if l == "&": + return "AND" + + if l == "|": + return "OR" + + if l == "CLOSED": + return "NONE" + + spdx_license = d.getVarFlag("SPDXLICENSEMAP", l) or l + if spdx_license in license_data["licenses"]: + return spdx_license + + spdx_license = "LicenseRef-" + l + license_text_map[spdx_license] = add_license_text(l)._id + + return spdx_license + + lic_split = license_expression.replace("(", " ( ").replace(")", " ) ").replace("|", " | ").replace("&", " & ").split() + spdx_license_expression = ' '.join(convert(l) for l in lic_split) + + return objset.new_license_expression(spdx_license_expression, license_text_map) + + +def add_package_files(d, objset, topdir, get_spdxid, get_purposes, *, archive=None, ignore_dirs=[], ignore_top_level_dirs=[]): + from pathlib import Path + import oe.spdx30 + import oe.sbom30 + + license_cache = {} + + source_date_epoch = d.getVar("SOURCE_DATE_EPOCH") + if source_date_epoch: + source_date_epoch = int(source_date_epoch) + + spdx_files = set() + + file_counter = 1 + for subdir, dirs, files in os.walk(topdir): + dirs[:] = [d for d in dirs if d not in ignore_dirs] + if subdir == str(topdir): + dirs[:] = [d for d in dirs if d not in ignore_top_level_dirs] + + for file in files: + filepath = Path(subdir) / file + if filepath.is_symlink() or not filepath.is_file(): + continue + + filename = str(filepath.relative_to(topdir)) + + spdx_file = oe.spdx30.software_File( + _id=get_spdxid(file_counter), + creationInfo=objset.doc.creationInfo, + name=filename, + ) + + file_purposes = get_purposes(filepath) + is_source = oe.spdx30.software_SoftwarePurpose.source in file_purposes + if file_purposes: + spdx_file.software_primaryPurpose = file_purposes[0] + spdx_file.software_additionalPurpose = file_purposes[1:] + + objset.add(spdx_file) + spdx_files.add(spdx_file) + + if archive is not None: + with filepath.open("rb") as f: + info = archive.gettarinfo(fileobj=f) + info.name = filename + info.uid = 0 + info.gid = 0 + info.uname = "root" + info.gname = "root" + + if source_date_epoch is not None and info.mtime > source_date_epoch: + info.mtime = source_date_epoch + + archive.addfile(info, f) + + spdx_file.verifiedUsing.append( + oe.spdx30.Hash( + algorithm=oe.spdx30.HashAlgorithm.sha1, + hashValue=bb.utils.sha1_file(filepath), + ) + ) + + spdx_file.verifiedUsing.append( + oe.spdx30.Hash( + algorithm=oe.spdx30.HashAlgorithm.sha256, + hashValue=bb.utils.sha256_file(filepath), + ) + ) + + bb.debug(1, "Adding file %s to %s" % (filepath, objset.doc._id)) + + if is_source: + file_licenses = set() + for extracted_lic in extract_licenses(filepath): + file_licenses.add(objset.new_license_expression(extracted_lic)) + + if file_licenses: + objset.new_relationship( + [spdx_file], + oe.spdx30.RelationshipType.hasDeclaredLicense, + file_licenses + ) + + file_counter += 1 + + return spdx_files + + +def get_package_sources_from_debug(d, package, package_files, sources, source_hash_cache): + from pathlib import Path + import oe.packagedata + + debug_search_paths = [ + Path(d.getVar('PKGD')), + Path(d.getVar('STAGING_DIR_TARGET')), + Path(d.getVar('STAGING_DIR_NATIVE')), + Path(d.getVar('STAGING_KERNEL_DIR')), + ] + + pkg_data = oe.packagedata.read_subpkgdata_extended(package, d) + + if pkg_data is None: + return + + dep_source_files = set() + + for file_path, file_data in pkg_data["files_info"].items(): + if not "debugsrc" in file_data: + continue + + for pkg_file in package_files: + if file_path.lstrip("/") == pkg_file.name.lstrip("/"): + break + else: + bb.fatal("No package file found for %s in %s; SPDX found: %s" % (str(file_path), package, + " ".join(p.name for p in package_files))) + continue + + for debugsrc in file_data["debugsrc"]: + for search in debug_search_paths: + if debugsrc.startswith("/usr/src/kernel"): + debugsrc_path = search / debugsrc.replace('/usr/src/kernel/', '') + else: + debugsrc_path = search / debugsrc.lstrip("/") + + if debugsrc_path in source_hash_cache: + file_sha256 = source_hash_cache[debugsrc_path] + if file_sha256 is None: + continue + else: + if not debugsrc_path.exists(): + source_hash_cache[debugsrc_path] = None + continue + + file_sha256 = bb.utils.sha256_file(debugsrc_path) + source_hash_cache[debugsrc_path] = file_sha256 + + if file_sha256 in sources: + source_file = sources[file_sha256] + dep_source_files.add(source_file) + else: + bb.debug(1, "Debug source %s with SHA256 %s not found in any dependency" % (str(debugsrc_path), file_sha256)) + break + else: + bb.debug(1, "Debug source %s not found" % debugsrc) + + return dep_source_files + +get_package_sources_from_debug[vardepsexclude] += "STAGING_KERNEL_DIR" + +def collect_dep_objsets(d, build): + import json + from pathlib import Path + import oe.sbom30 + import oe.spdx30 + + deps = get_spdx_deps(d) + + dep_objsets = [] + dep_builds = set() + + dep_build_spdxids = set() + for dep_pn, _, in_taskhash in deps: + bb.debug(1, "Fetching SPDX for dependency %s" % (dep_pn)) + dep_build, dep_objset = oe.sbom30.find_root_obj_in_jsonld(d, "recipes", dep_pn, oe.spdx30.build_Build) + # If the dependency is part of the taskhash, return it to be linked + # against. Otherwise, it cannot be linked against because this recipe + # will not rebuilt if dependency changes + if in_taskhash: + dep_objsets.append(dep_objset) + + # The build _can_ be linked against (by alias) + dep_builds.add(dep_build) + + return dep_objsets, dep_builds + +collect_dep_objsets[vardepsexclude] = "SSTATE_ARCHS" + +def collect_dep_sources(dep_objsets): + import oe.spdx30 + import oe.sbom30 + + sources = {} + for objset in dep_objsets: + # Don't collect sources from native recipes as they + # match non-native sources also. + if objset.is_native(): + continue + + bb.debug(1, "Fetching Sources for dependency %s" % (objset.doc.name)) + + dep_build = objset.find_root(oe.spdx30.build_Build) + if not dep_build: + bb.fatal("Unable to find a build") + + for e in objset.foreach_type(oe.spdx30.Relationship): + if dep_build is not e.from_: + continue + + if e.relationshipType != oe.spdx30.RelationshipType.hasInputs: + continue + + for to in e.to: + if not isinstance(to, oe.spdx30.software_File): + continue + + if to.software_primaryPurpose != oe.spdx30.software_SoftwarePurpose.source: + continue + + for v in to.verifiedUsing: + if v.algorithm == oe.spdx30.HashAlgorithm.sha256: + sources[v.hashValue] = to + break + else: + bb.fatal("No SHA256 found for %s in %s" % (to.name, objset.doc.name)) + + return sources + +def add_hashes(d, element, filepath): + import hashlib + + algorithms = d.getVar("SPDX_HASH_ALGORITHMS").split() + algorithms.sort() + + hashes = [hashlib.new(a) for a in algorithms] + + with open(filepath, "rb") as f: + while True: + data = f.read(4096) + if not data: + break + + for h in hashes: + h.update(data) + + element.verifiedUsing.extend( + oe.spdx30.Hash( + algorithm=getattr(oe.spdx30.HashAlgorithm, h.name), + hashValue=h.hexdigest(), + ) for h in hashes + ) + + +def add_download_files(d, objset): + import oe.patch + import oe.spdx30 + import os + + inputs = set() + + urls = d.getVar("SRC_URI").split() + fetch = bb.fetch2.Fetch(urls, d) + + for download_idx, src_uri in enumerate(urls): + fd = fetch.ud[src_uri] + + for name in fd.names: + file_name = os.path.basename(fetch.localpath(src_uri)) + if oe.patch.patch_path(src_uri, fetch, '', expand=False): + primary_purpose = oe.spdx30.software_SoftwarePurpose.patch + else: + primary_purpose = oe.spdx30.software_SoftwarePurpose.source + + if fd.type == "file": + file = objset.add(oe.spdx30.software_File( + _id=objset.new_spdxid("source", str(download_idx + 1)), + creationInfo=objset.doc.creationInfo, + name=file_name, + software_primaryPurpose=primary_purpose, + )) + add_hashes(d, file, fd.localpath) + + inputs.add(file) + else: + uri = fd.type + proto = getattr(fd, "proto", None) + if proto is not None: + uri = uri + "+" + proto + uri = uri + "://" + fd.host + fd.path + + if fd.method.supports_srcrev(): + uri = uri + "@" + fd.revisions[name] + + dl = objset.add(oe.spdx30.software_Package( + _id=objset.new_spdxid("source", str(download_idx + 1)), + creationInfo=objset.doc.creationInfo, + name=file_name, + software_primaryPurpose=primary_purpose, + software_downloadLocation=uri, + )) + + if fd.method.supports_checksum(fd): + # TODO Need something better than hard coding this + for checksum_id in ["sha256", "sha1"]: + expected_checksum = getattr(fd, "%s_expected" % checksum_id, None) + if expected_checksum is None: + continue + + dl.verifiedUsing.append( + oe.spdx30.Hash( + algorithm=getattr(oe.spdx30.HashAlgorithm, checksum_id), + hashValue=expected_checksum, + ) + ) + + inputs.add(dl) + + return inputs + + +def collect_build_package_inputs(d, objset, build, packages): + providers = collect_package_providers(d) + + build_deps = set() + + for name in sorted(packages.keys()): + if name not in providers: + bb.fatal("Unable to find SPDX provider for '%s'" % name) + + pkg_name, pkg_hashfn = providers[name] + + # Copy all of the package SPDX files into the Sbom elements + pkg_spdx, _ = oe.sbom30.find_root_obj_in_jsonld( + d, + "packages", + pkg_name, + oe.spdx30.software_Package, + software_primaryPurpose=oe.spdx30.software_SoftwarePurpose.install, + ) + build_deps.add(pkg_spdx._id) + + objset.new_scoped_relationship( + [build], + oe.spdx30.RelationshipType.hasInputs, + oe.spdx30.LifecycleScopeType.build, + sorted(list(build_deps)), + ) + +def set_purposes(d, element, *var_names, force_purposes=[]): + purposes = force_purposes[:] + + for var_name in var_names: + val = d.getVar(var_name) + if val: + purposes.extend(val.split()) + break + + if not purposes: + bb.warn("No SPDX purposes found in %s" % " ".join(var_names)) + return + + element.software_primaryPurpose = getattr(oe.spdx30.software_SoftwarePurpose, purposes[0]) + element.software_additionalPurpose = [getattr(oe.spdx30.software_SoftwarePurpose, p) for p in purposes[1:]] + + +python do_create_spdx() { + import oe.sbom30 + import oe.spdx30 + from pathlib import Path + from contextlib import contextmanager + import oe.cve_check + from datetime import datetime + + + def set_var_field(var, obj, name, package=None): + val = None + if package: + val = d.getVar("%s:%s" % (var, package)) + + if not val: + val = d.getVar(var) + + if val: + setattr(obj, name, val) + + deploydir = Path(d.getVar("SPDXDEPLOY")) + deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX")) + spdx_workdir = Path(d.getVar("SPDXWORK")) + include_sources = d.getVar("SPDX_INCLUDE_SOURCES") == "1" + pkg_arch = d.getVar("SSTATE_PKGARCH") + is_native = bb.data.inherits_class("native", d) or bb.data.inherits_class("cross", d) + + build_objset = oe.sbom30.ObjectSet.new_objset(d, d.getVar("PN")) + + build = build_objset.new_sub_build("recipe", "recipe") + build_objset.doc.rootElement.append(build) + + build_objset.set_is_native(is_native) + + for var in (d.getVar('SPDX_CUSTOM_ANNOTATION_VARS') or "").split(): + new_annotation( + d, + build_objset, + build, + "%s=%s" % (var, d.getVar(var)), + oe.spdx30.AnnotationType.other + ) + + build_inputs = set() + + # Add CVEs + cve_by_status = {} + cve_by_status["Patched"] = {cve: build_objset.new_cve_vuln(cve) for cve in oe.cve_check.get_patched_cves(d)} + + for cve in (d.getVarFlags("CVE_STATUS") or {}): + status, _, _ = oe.cve_check.decode_cve_status(d, cve) + # Already added above + if status == "Patched": + continue + + cve_by_status.setdefault(status, {})[cve] = build_objset.new_cve_vuln(cve) + + cpe_ids = oe.cve_check.get_cpe_ids(d.getVar("CVE_PRODUCT"), d.getVar("CVE_VERSION")) + + source_files = add_download_files(d, build_objset) + build_inputs |= source_files + + recipe_spdx_license = add_license_expression(d, build_objset, d.getVar("LICENSE")) + build_objset.new_relationship( + source_files, + oe.spdx30.RelationshipType.hasConcludedLicense, + [recipe_spdx_license], + ) + + if process_sources(d) and include_sources: + bb.debug(1, "Adding source files to SPDX") + spdx_get_src(d) + + build_inputs |= add_package_files( + d, + build_objset, + spdx_workdir, + lambda file_counter: build_objset.new_spdxid("sourcefile", str(file_counter)), + lambda filepath: [oe.spdx30.software_SoftwarePurpose.source], + ignore_dirs=[".git"], + ignore_top_level_dirs=["temp"], + archive=None, + ) + + dep_objsets, dep_builds = collect_dep_objsets(d, build) + build_objset.new_scoped_relationship( + [build], + oe.spdx30.RelationshipType.dependsOn, + oe.spdx30.LifecycleScopeType.build, + sorted(oe.sbom30.get_element_link_id(b) for b in dep_builds), + ) + + debug_source_ids = set() + source_hash_cache = {} + + # Write out the package SPDX data now. It is not complete as we cannot + # write the runtime data, so write it to a staging area and a later task + # will write out the final collection + + # TODO: Handle native recipe output + if not is_native: + bb.debug(1, "Collecting Dependency sources files") + sources = collect_dep_sources(dep_objsets) + + bb.build.exec_func("read_subpackage_metadata", d) + + pkgdest = Path(d.getVar("PKGDEST")) + for package in d.getVar("PACKAGES").split(): + if not oe.packagedata.packaged(package, d): + continue + + pkg_name = d.getVar("PKG:%s" % package) or package + + bb.debug(1, "Creating SPDX for package %s" % pkg_name) + + pkg_objset = oe.sbom30.ObjectSet.new_objset(d, pkg_name) + + spdx_package = pkg_objset.add_root(oe.spdx30.software_Package( + _id=pkg_objset.new_spdxid("package", pkg_name), + creationInfo=pkg_objset.doc.creationInfo, + name=pkg_name, + software_packageVersion=d.getVar("PV"), + builtTime=oe.sbom30.spdx_now(), + )) + + set_purposes( + d, + spdx_package, + "SPDX_PACKAGE_ADDITIONAL_PURPOSE:%s" % package, + "SPDX_PACKAGE_ADDITIONAL_PURPOSE", + force_purposes=["install"], + ) + + + supplier = build_objset.new_agent("SPDX_SUPPLIER") + if supplier is not None: + spdx_package.supplier = supplier if isinstance(supplier, str) else supplier._id + + set_var_field("HOMEPAGE", spdx_package, "software_homePage", package=package) + set_var_field("SUMMARY", spdx_package, "summary", package=package) + set_var_field("DESCRIPTION", spdx_package, "description", package=package) + + pkg_objset.new_scoped_relationship( + [build._id], + oe.spdx30.RelationshipType.hasOutputs, + oe.spdx30.LifecycleScopeType.build, + [spdx_package], + ) + + if cpe_ids: + for cpe_id in cpe_ids: + spdx_package.externalIdentifier.append( + oe.spdx30.ExternalIdentifier( + externalIdentifierType=oe.spdx30.ExternalIdentifierType.cpe23, + identifier=cpe_id, + )) + + # TODO: Generate a file for each actual IPK/DEB/RPM/TGZ file + # generated and link it to the package + #spdx_package_file = pkg_objset.add(oe.spdx30.software_File( + # _id=pkg_objset.new_spdxid("distribution", pkg_name), + # creationInfo=pkg_objset.doc.creationInfo, + # name=pkg_name, + # builtTime=oe.sbom30.spdx_now(), + # software_primaryPurpose=spdx_package.software_primaryPurpose, + # software_additionalPurpose=spdx_package.software_additionalPurpose, + #)) + + ## TODO add hashes + #pkg_objset.new_relationship( + # [spdx_package], + # oe.spdx30.RelationshipType.hasDistributionArtifact, + # [spdx_package_file], + #) + + # NOTE: licenses live in the recipe collection and are referenced + # by ID in the package collection(s). This helps reduce duplication + # (since a lot of packages will have the same license), and also + # prevents duplicate license SPDX IDs in the packages + package_license = d.getVar("LICENSE:%s" % package) + if package_license and package_license != d.getVar("LICENSE"): + package_spdx_license = add_license_expression(d, build_objset, package_license) + else: + package_spdx_license = recipe_spdx_license + + pkg_objset.new_relationship( + [spdx_package], + oe.spdx30.RelationshipType.hasConcludedLicense, + [package_spdx_license._id], + ) + + # NOTE: CVE Elements live in the recipe collection + for status, cves in cve_by_status.items(): + pkg_objset.new_relationship( + [spdx_package], + oe.spdx30.RelationshipType.hasAssociatedVulnerability, + sorted(spdx_cve._id for _, spdx_cve in cves.items()), + ) + + for cve, spdx_cve in cves.items(): + if status == "Patched": + pkg_objset.new_vex_patched_relationship(spdx_cve._id, [spdx_package]) + elif status == "Unpatched": + pkg_objset.new_vex_unpatched_relationship(spdx_cve._id, [spdx_package]) + elif status == "Ignored": + pkg_objset.new_vex_ignored_relationship( + spdx_cve._id, + [spdx_package], + impact_statement=d.getVarFlag("CVE_STATUS", cve).split(":", 1)[1], + ) + else: + bb.fatal(f"Unknown CVE status {status}") + + bb.debug(1, "Adding package files to SPDX for package %s" % pkg_name) + package_files = add_package_files( + d, + pkg_objset, + pkgdest / package, + lambda file_counter: pkg_objset.new_spdxid("package", pkg_name, "file", str(file_counter)), + # TODO: Can we know the purpose here? + lambda filepath: [], + ignore_top_level_dirs=['CONTROL', 'DEBIAN'], + archive=None, + ) + + pkg_objset.new_relationship( + [spdx_package], + oe.spdx30.RelationshipType.contains, + sorted(list(package_files)), + ) + + debug_sources = get_package_sources_from_debug(d, package, package_files, sources, source_hash_cache) + debug_source_ids |= set(oe.sbom30.get_element_link_id(d) for d in debug_sources) + + oe.sbom30.write_recipe_jsonld_doc(d, pkg_objset, "packages-staging", deploydir, create_spdx_id_links=False) + + if include_sources: + bb.debug(1, "Adding sysroot files to SPDX") + sysroot_files = add_package_files( + d, + build_objset, + d.expand("${COMPONENTS_DIR}/${PACKAGE_ARCH}/${PN}"), + lambda file_counter: build_objset.new_spdxid("sysroot", str(file_counter)), + lambda filepath: [], + archive=None, + ) + + if sysroot_files: + build_objset.new_scoped_relationship( + [build], + oe.spdx30.RelationshipType.hasOutputs, + oe.spdx30.LifecycleScopeType.build, + sorted(list(sysroot_files)), + ) + + build_objset.new_scoped_relationship( + [build], + oe.spdx30.RelationshipType.hasInputs, + oe.spdx30.LifecycleScopeType.build, + sorted(list(build_inputs)) + sorted(list(debug_source_ids)), + ) + + oe.sbom30.write_recipe_jsonld_doc(d, build_objset, "recipes", deploydir) +} +do_create_spdx[vardepsexclude] += "BB_NUMBER_THREADS" +# NOTE: depending on do_unpack is a hack that is necessary to get it's dependencies for archive the source +addtask do_create_spdx after do_populate_sysroot do_package do_packagedata do_unpack do_collect_spdx_deps do_package_write_ipk do_pacakge_write_deb do_package_write_rpm before do_populate_sdk do_build do_rm_work + +SSTATETASKS += "do_create_spdx" +do_create_spdx[sstate-inputdirs] = "${SPDXDEPLOY}" +do_create_spdx[sstate-outputdirs] = "${DEPLOY_DIR_SPDX}" + +python do_create_spdx_setscene () { + sstate_setscene(d) +} +addtask do_create_spdx_setscene + +do_create_spdx[dirs] = "${SPDXWORK}" +do_create_spdx[cleandirs] = "${SPDXDEPLOY} ${SPDXWORK}" +do_create_spdx[depends] += "${PATCHDEPENDENCY}" + +python do_create_package_spdx() { + import oe.sbom30 + import oe.spdx30 + import oe.packagedata + from pathlib import Path + + deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX")) + deploydir = Path(d.getVar("SPDXRUNTIMEDEPLOY")) + is_native = bb.data.inherits_class("native", d) or bb.data.inherits_class("cross", d) + + providers = collect_package_providers(d) + pkg_arch = d.getVar("SSTATE_PKGARCH") + + if not is_native: + bb.build.exec_func("read_subpackage_metadata", d) + + dep_package_cache = {} + + # Any element common to all packages that need to be referenced by ID + # should be written into this objset set + common_objset = oe.sbom30.ObjectSet.new_objset(d, "%s-package-common" % d.getVar("PN")) + + pkgdest = Path(d.getVar("PKGDEST")) + for package in d.getVar("PACKAGES").split(): + bb.debug(1, "Processing package %s" % package) + localdata = bb.data.createCopy(d) + pkg_name = d.getVar("PKG:%s" % package) or package + localdata.setVar("PKG", pkg_name) + localdata.setVar('OVERRIDES', d.getVar("OVERRIDES", False) + ":" + package) + + if not oe.packagedata.packaged(package, localdata): + continue + + spdx_package, pkg_objset = oe.sbom30.load_obj_in_jsonld( + d, + pkg_arch, + "packages-staging", + pkg_name, + oe.spdx30.software_Package, + software_primaryPurpose=oe.spdx30.software_SoftwarePurpose.install, + ) + + # We will write out a new collection, so link it to the new + # creation info in the common package data. The old creation info + # should still exist and be referenced by all the existing elements + # in the package + pkg_objset.creationInfo = pkg_objset.copy_creation_info(common_objset.doc.creationInfo) + + runtime_spdx_deps = set() + + deps = bb.utils.explode_dep_versions2(localdata.getVar("RDEPENDS") or "") + seen_deps = set() + for dep, _ in deps.items(): + if dep in seen_deps: + continue + + if dep not in providers: + continue + + (dep, _) = providers[dep] + + if not oe.packagedata.packaged(dep, localdata): + continue + + dep_pkg_data = oe.packagedata.read_subpkgdata_dict(dep, d) + dep_pkg = dep_pkg_data["PKG"] + + if dep in dep_package_cache: + dep_spdx_package = dep_package_cache[dep] + else: + bb.debug(1, "Searching for %s" % dep_pkg) + dep_spdx_package, _ = oe.sbom30.find_root_obj_in_jsonld( + d, + "packages-staging", + dep_pkg, + oe.spdx30.software_Package, + software_primaryPurpose=oe.spdx30.software_SoftwarePurpose.install, + ) + dep_package_cache[dep] = dep_spdx_package + + runtime_spdx_deps.add(dep_spdx_package) + seen_deps.add(dep) + + pkg_objset.new_scoped_relationship( + [spdx_package], + oe.spdx30.RelationshipType.dependsOn, + oe.spdx30.LifecycleScopeType.runtime, + [oe.sbom30.get_element_link_id(dep) for dep in runtime_spdx_deps], + ) + + oe.sbom30.write_recipe_jsonld_doc(d, pkg_objset, "packages", deploydir) + + oe.sbom30.write_recipe_jsonld_doc(d, common_objset, "common-package", deploydir) +} + +do_create_package_spdx[vardepsexclude] += "OVERRIDES SSTATE_ARCHS" + +addtask do_create_package_spdx after do_create_spdx before do_build do_rm_work +SSTATETASKS += "do_create_package_spdx" +do_create_package_spdx[sstate-inputdirs] = "${SPDXRUNTIMEDEPLOY}" +do_create_package_spdx[sstate-outputdirs] = "${DEPLOY_DIR_SPDX}" + +python do_create_package_spdx_setscene () { + sstate_setscene(d) +} +addtask do_create_package_spdx_setscene + +do_create_package_spdx[dirs] = "${SPDXRUNTIMEDEPLOY}" +do_create_package_spdx[cleandirs] = "${SPDXRUNTIMEDEPLOY}" +do_create_package_spdx[rdeptask] = "do_create_spdx" + +python do_create_rootfs_spdx() { + import json + from pathlib import Path + import oe.spdx30 + import oe.sbom30 + from datetime import datetime + + deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX")) + deploydir = Path(d.getVar("SPDXROOTFSDEPLOY")) + root_packages_file = Path(d.getVar("SPDX_ROOTFS_PACKAGES")) + image_basename = d.getVar("IMAGE_BASENAME") + machine = d.getVar("MACHINE") + + with root_packages_file.open("r") as f: + packages = json.load(f) + + objset = oe.sbom30.ObjectSet.new_objset(d, "%s-%s" % (image_basename, machine)) + + rootfs = objset.add_root(oe.spdx30.software_SoftwareArtifact( + _id=objset.new_spdxid("rootfs", image_basename), + creationInfo=objset.doc.creationInfo, + name=image_basename, + software_primaryPurpose=oe.spdx30.software_SoftwarePurpose.archive, + builtTime=oe.sbom30.spdx_now() + )) + + rootfs_build = objset.add_root(objset.new_sub_build("rootfs", "rootfs")) + rootfs_build.build_buildEndTime = oe.sbom30.spdx_now() + + objset.new_scoped_relationship( + [rootfs_build], + oe.spdx30.RelationshipType.hasOutputs, + oe.spdx30.LifecycleScopeType.build, + [rootfs], + ) + + collect_build_package_inputs(d, objset, rootfs_build, packages) + + oe.sbom30.write_recipe_jsonld_doc(d, objset, "rootfs", deploydir) +} +addtask do_create_rootfs_spdx after do_rootfs before do_image +SSTATETASKS += "do_create_rootfs_spdx" +do_create_rootfs_spdx[sstate-inputdirs] = "${SPDXROOTFSDEPLOY}" +do_create_rootfs_spdx[sstate-outputdirs] = "${DEPLOY_DIR_SPDX}" +do_create_rootfs_spdx[recrdeptask] += "do_create_spdx do_create_package_spdx" +do_create_rootfs_spdx[cleandirs] += "${SPDXROOTFSDEPLOY}" + +python do_create_rootfs_spdx_setscene() { + sstate_setscene(d) +} +addtask do_create_rootfs_spdx_setscene + +python do_create_image_spdx() { + import oe.spdx30 + import oe.sbom30 + import json + from pathlib import Path + + image_deploy_dir = Path(d.getVar('IMGDEPLOYDIR')) + manifest_path = Path(d.getVar("IMAGE_FILE_MANIFEST")) + spdx_work_dir = Path(d.getVar('SPDXIMAGEWORK')) + + image_basename = d.getVar('IMAGE_BASENAME') + machine = d.getVar("MACHINE") + + objset = oe.sbom30.ObjectSet.new_objset(d, "%s-%s" % (image_basename, machine)) + + with manifest_path.open("r") as f: + manifest = json.load(f) + + builds = [] + for task in manifest: + imagetype = task["imagetype"] + taskname = task["taskname"] + + image_build = objset.add_root(objset.new_sub_build(taskname, "image/%s" % imagetype)) + image_build.build_buildEndTime = oe.sbom30.spdx_now() + builds.append(image_build) + + artifacts = [] + + for image in task["images"]: + image_filename = image["filename"] + image_path = image_deploy_dir / image_filename + a = objset.add_root(oe.spdx30.software_File( + _id=objset.new_spdxid("image", image_filename), + creationInfo=objset.doc.creationInfo, + name=image_filename, + builtTime=oe.sbom30.spdx_now() + )) + set_purposes(d, a, "SPDX_IMAGE_PURPOSE:%s" % imagetype, "SPDX_IMAGE_PURPOSE") + add_hashes(d, a, image_path) + + artifacts.append(a) + + if artifacts: + objset.new_scoped_relationship( + [image_build], + oe.spdx30.RelationshipType.hasOutputs, + oe.spdx30.LifecycleScopeType.build, + artifacts, + ) + + if builds: + rootfs_image, _ = oe.sbom30.find_root_obj_in_jsonld( + d, + "rootfs", + "%s-%s" % (image_basename, machine), + oe.spdx30.software_SoftwareArtifact, + # TODO: Should use a purpose to filter here? + ) + objset.new_scoped_relationship( + builds, + oe.spdx30.RelationshipType.hasInputs, + oe.spdx30.LifecycleScopeType.build, + [rootfs_image._id], + ) + + objset.add_aliases() + objset.link() + oe.sbom30.write_recipe_jsonld_doc(d, objset, "image", spdx_work_dir) +} +addtask do_create_image_spdx after do_image_complete do_create_rootfs_spdx before do_build +SSTATETASKS += "do_create_image_spdx" +SSTATE_SKIP_CREATION:task-combine-image-type-spdx = "1" +do_create_image_spdx[sstate-inputdirs] = "${SPDXIMAGEWORK}" +do_create_image_spdx[sstate-outputdirs] = "${DEPLOY_DIR_SPDX}" +do_create_image_spdx[cleandirs] = "${SPDXIMAGEWORK}" +do_create_image_spdx[dirs] = "${SPDXIMAGEWORK}" + +python do_create_image_spdx_setscene() { + sstate_setscene(d) +} +addtask do_create_image_spdx_setscene + + +python do_create_image_sbom() { + import os + from pathlib import Path + import oe.spdx30 + import oe.sbom30 + + image_name = d.getVar("IMAGE_NAME") + image_basename = d.getVar("IMAGE_BASENAME") + image_link_name = d.getVar("IMAGE_LINK_NAME") + imgdeploydir = Path(d.getVar("SPDXIMAGEDEPLOYDIR")) + machine = d.getVar("MACHINE") + + spdx_path = imgdeploydir / (image_name + ".spdx.json") + + root_elements = [] + + # TODO: Do we need to add the rootfs or are the image files sufficient? + rootfs_image, _ = oe.sbom30.find_root_obj_in_jsonld( + d, + "rootfs", + "%s-%s" % (image_basename, machine), + oe.spdx30.software_SoftwareArtifact, + # TODO: Should use a purpose here? + ) + root_elements.append(rootfs_image._id) + + image_objset, _ = oe.sbom30.find_jsonld(d, "image", "%s-%s" % (image_basename, machine), required=True) + for o in image_objset.foreach_root(oe.spdx30.software_File): + root_elements.append(o._id) + + objset, sbom = oe.sbom30.create_sbom(d, image_name, root_elements) + + oe.sbom30.write_jsonld_doc(d, objset, spdx_path) + + def make_image_link(target_path, suffix): + if image_link_name: + link = imgdeploydir / (image_link_name + suffix) + if link != target_path: + link.symlink_to(os.path.relpath(target_path, link.parent)) + + make_image_link(spdx_path, ".spdx.json") +} +addtask do_create_image_sbom after do_create_rootfs_spdx do_create_image_spdx before do_build +SSTATETASKS += "do_create_image_sbom" +SSTATE_SKIP_CREATION:task-create-image-sbom = "1" +do_create_image_sbom[sstate-inputdirs] = "${SPDXIMAGEDEPLOYDIR}" +do_create_image_sbom[sstate-outputdirs] = "${DEPLOY_DIR_IMAGE}" +do_create_image_sbom[stamp-extra-info] = "${MACHINE_ARCH}" +do_create_image_sbom[cleandirs] = "${SPDXIMAGEDEPLOYDIR}" + +python do_create_image_sbom_setscene() { + sstate_setscene(d) +} +addtask do_create_image_sbom_setscene + +do_populate_sdk[recrdeptask] += "do_create_spdx do_create_package_spdx" +do_populate_sdk[cleandirs] += "${SPDXSDKWORK}" +do_populate_sdk[postfuncs] += "sdk_create_sbom" +POPULATE_SDK_POST_HOST_COMMAND:append:task-populate-sdk = " sdk_host_create_spdx" +POPULATE_SDK_POST_TARGET_COMMAND:append:task-populate-sdk = " sdk_target_create_spdx" + +python sdk_host_create_spdx() { + sdk_create_spdx(d, "host") +} + +python sdk_target_create_spdx() { + sdk_create_spdx(d, "target") +} + +def sdk_create_spdx(d, sdk_type): + from pathlib import Path + from oe.sdk import sdk_list_installed_packages + import oe.spdx30 + import oe.sbom30 + from datetime import datetime + + sdk_name = d.getVar("TOOLCHAIN_OUTPUTNAME") + "-" + sdk_type + sdk_packages = sdk_list_installed_packages(d, sdk_type == "target") + spdx_work_dir = Path(d.getVar('SPDXSDKWORK')) + + objset = oe.sbom30.ObjectSet.new_objset(d, sdk_name) + + sdk_rootfs = objset.add_root(oe.spdx30.software_SoftwareArtifact( + _id=objset.new_spdxid("sdk-rootfs", sdk_name), + creationInfo=objset.doc.creationInfo, + name=sdk_name, + softwre_primaryPurpose=oe.spdx30.software_SoftwarePurpose.archive, + builtTime=oe.sbom30.spdx_now(), + )) + + sdk_build = objset.add_root(objset.new_sub_build("sdk-rootfs", "sdk-rootfs")) + sdk_build.build_buildEndTime = oe.sbom30.spdx_now() + + objset.new_scoped_relationship( + [sdk_build], + oe.spdx30.RelationshipType.hasOutputs, + oe.spdx30.LifecycleScopeType.build, + [sdk_rootfs], + ) + + collect_build_package_inputs(d, objset, sdk_build, sdk_packages) + + objset.add_aliases() + oe.sbom30.write_jsonld_doc(d, objset, spdx_work_dir / "sdk-rootfs.spdx.json") + +python sdk_create_sbom() { + import oe.spdx30 + import oe.sbom30 + from pathlib import Path + from datetime import datetime + + toolchain_outputname = d.getVar("TOOLCHAIN_OUTPUTNAME") + sdk_filename = toolchain_outputname + ".sh" + sdk_deploydir = Path(d.getVar("SDKDEPLOYDIR")) + spdx_work_dir = Path(d.getVar('SPDXSDKWORK')) + + # Load the document written earlier + rootfs_objset = oe.sbom30.load_jsonld(d, spdx_work_dir / "sdk-rootfs.spdx.json", required=True) + + # Create a new build for the SDK installer + sdk_build = rootfs_objset.new_sub_build("sdk-populate", "sdk-populate") + sdk_build.buildEndTime = oe.sbom30.spdx_now() + + rootfs = rootfs_objset.find_root(oe.spdx30.software_SoftwareArtifact) + if rootfs is None: + bb.fatal("Unable to find rootfs artifact") + + rootfs_objset.new_scoped_relationship( + [sdk_build], + oe.spdx30.RelationshipType.hasInputs, + oe.spdx30.LifecycleScopeType.build, + [rootfs] + ) + + # Create a file for the SDK installer + sdk_installer = objset.add(oe.spdx30.software_File( + _id=objset.new_spdxid("sdk-installer", toolchain_outputname), + creationInfo=rootfs_objset.doc.creationInfo, + name=sdk_filename, + builtTime=oe.sbom30.spdx_now(), + )) + set_purposes(d, sdk_installer, "SPDX_SDK_PURPOSE") + add_hashes(d, sdk_installer, sdk_deploydir / sdk_filename) + + rootfs_objset.new_scoped_relationship( + [sdk_build], + oe.spdx30.RelationshipType.hasOutputs, + oe.spdx30.LifecycleScopeType.build, + [sdk_installer], + ) + + objset, sbom = oe.sbom30.create_sbom(d, sdk_filename, [sdk_installer], [rootfs_objset]) + + oe.sbom30.write_jsonld_doc(d, objset, sdk_deploydir / (toolchain_outputname + ".spdx.json")) +} + +python spdx30_build_started_handler () { + import oe.spdx30 + import oe.sbom30 + import os + from pathlib import Path + + # Create a copy of the datastore. Set PN to "bitbake" so that SPDX IDs can + # be generated + d = e.data.createCopy() + d.setVar("PN", "bitbake") + d.setVar("BB_TASKHASH", "bitbake") + load_spdx_license_data(d) + + deploy_dir_spdx = Path(e.data.getVar("DEPLOY_DIR_SPDX")) + + nonce = os.urandom(16).hex() + + objset = oe.sbom30.ObjectSet.new_objset(d, "bitbake", False) + + build = objset.add_root(oe.spdx30.build_Build( + _id=objset.new_spdxid(nonce, include_unihash=False), + creationInfo=objset.doc.creationInfo, + build_buildType=oe.sbom30.SPDX_BUILD_TYPE, + build_buildStartTime=oe.sbom30.spdx_now() + )) + + host_import_key = d.getVar("SPDX_BUILD_HOST") + if host_import_key: + objset.new_scoped_relationship( + [build], + oe.spdx30.RelationshipType.hasHost, + oe.spdx30.LifecycleScopeType.build, + [objset.new_import("SPDX_BUILD_HOST")], + ) + + invoked_by = objset.new_agent("SPDX_INVOKED_BY") + if invoked_by: + invoked_by_spdx = objset.new_scoped_relationship( + [build], + oe.spdx30.RelationshipType.invokedBy, + oe.spdx30.LifecycleScopeType.build, + [invoked_by], + ) + + on_behalf_of = objset.new_agent("SPDX_ON_BEHALF_OF") + if on_behalf_of: + objset.new_scoped_relationship( + [on_behalf_of], + oe.spdx30.RelationshipType.delegatedTo, + oe.spdx30.LifecycleScopeType.build, + invoked_by_spdx, + ) + + for obj in objset.foreach_type(oe.spdx30.Element): + obj.extension.append(oe.sbom30.OELinkExtension(link_spdx_id=False)) + obj.extension.append(oe.sbom30.OEIdAliasExtension()) + + oe.sbom30.write_jsonld_doc(d, objset, deploy_dir_spdx / "bitbake.spdx.json") +} + +addhandler spdx30_build_started_handler +spdx30_build_started_handler[eventmask] = "bb.event.ConfigParsed" +
meta/classes/spdx-common.bbclass +24 -1
@@ -16,13 +16,17 @@ SPDXDIR ??= "${WORKDIR}/spdx/${SPDX_VERSION}" SPDXDEPLOY = "${SPDXDIR}/deploy" SPDXWORK = "${SPDXDIR}/work" SPDXIMAGEWORK = "${SPDXDIR}/image-work" +SPDXROOTFSWORK = "${SPDXDIR}/rootfs-work" SPDXSDKWORK = "${SPDXDIR}/sdk-work" SPDXDEPS = "${SPDXDIR}/deps.json" +SPDXIMAGEDEPLOYDIR = "${SPDXDIR}/image-deploy" +SPDX_ROOTFS_PACKAGES = "${SPDXDIR}/rootfs-packages.json" SPDX_TOOL_NAME ??= "oe-spdx-creator" SPDX_TOOL_VERSION ??= "1.0" SPDXRUNTIMEDEPLOY = "${SPDXDIR}/runtime-deploy" +SPDXROOTFSDEPLOY = "${SPDXDIR}/rootfs-deploy" SPDX_INCLUDE_SOURCES ??= "0" SPDX_ARCHIVE_SOURCES ??= "0"
@@ -68,7 +72,7 @@ def get_json_indent(d): return 2 return None -python() { +def load_spdx_license_data(d): import json if d.getVar("SPDX_LICENSE_DATA"): return
@@ -78,6 +82,9 @@ python() { # Transform the license array to a dictionary data["licenses"] = {l["licenseId"]: l for l in data["licenses"]} d.setVar("SPDX_LICENSE_DATA", data) + +python() { + load_spdx_license_data(d) } def process_sources(d):
@@ -255,3 +262,19 @@ def spdx_get_src(d): spdx_get_src[vardepsexclude] += "STAGING_KERNEL_DIR" + +python spdx_collect_rootfs_packages() { + import json + from pathlib import Path + from oe.rootfs import image_list_installed_packages + + root_packages_file = Path(d.getVar("SPDX_ROOTFS_PACKAGES")) + + packages = image_list_installed_packages(d) + + root_packages_file.parent.mkdir(parents=True, exist_ok=True) + with root_packages_file.open("w") as f: + json.dump(packages, f) +} +ROOTFS_POSTUNINSTALL_COMMAND =+ "spdx_collect_rootfs_packages" +
meta/lib/oe/sbom30.py new file +993 -0
993 lines not on this page · raw message
meta/lib/oe/spdx30.py new file +5413 -0
670 lines not on this page · raw message
The rest of this diff is not shown. View the raw message.
Signature · 1 line
2.43.2
Part of the diff is not shown here. View the raw message.
Patchtest results for [OE-core][PATCH 5/6] classes/create-spdx-3.0: Add class
Thank you for your submission. Patchtest identified one or more issues with the patch. Please see the log below for more information:
---
Testing patch /home/patchtest/share/mboxes/5-6-classes-create-spdx-3.0-Add-class.patch
FAIL: test max line length: Patch line too long (current length 216, maximum is 200) (test_metadata.TestMetadata.test_max_line_length)
PASS: pretest pylint (test_python_pylint.PyLint.pretest_pylint)
PASS: test Signed-off-by presence (test_mbox.TestMbox.test_signed_off_by_presence)
PASS: test author valid (test_mbox.TestMbox.test_author_valid)
PASS: test commit message presence (test_mbox.TestMbox.test_commit_message_presence)
PASS: test mbox format (test_mbox.TestMbox.test_mbox_format)
PASS: test non-AUH upgrade (test_mbox.TestMbox.test_non_auh_upgrade)
PASS: test pylint (test_python_pylint.PyLint.test_pylint)
PASS: test shortlog format (test_mbox.TestMbox.test_shortlog_format)
PASS: test shortlog length (test_mbox.TestMbox.test_shortlog_length)
PASS: test target mailing list (test_mbox.TestMbox.test_target_mailing_list)
SKIP: pretest src uri left files: Patch cannot be merged (test_metadata.TestMetadata.pretest_src_uri_left_files)
SKIP: test CVE check ignore: No modified recipes or older target branch, skipping test (test_metadata.TestMetadata.test_cve_check_ignore)
SKIP: test CVE tag format: No new CVE patches introduced (test_patch.TestPatch.test_cve_tag_format)
SKIP: test Signed-off-by presence: No new CVE patches introduced (test_patch.TestPatch.test_signed_off_by_presence)
SKIP: test Upstream-Status presence: No new CVE patches introduced (test_patch.TestPatch.test_upstream_status_presence_format)
SKIP: test bugzilla entry format: No bug ID found (test_mbox.TestMbox.test_bugzilla_entry_format)
SKIP: test lic files chksum modified not mentioned: No modified recipes, skipping test (test_metadata.TestMetadata.test_lic_files_chksum_modified_not_mentioned)
SKIP: test lic files chksum presence: No added recipes, skipping test (test_metadata.TestMetadata.test_lic_files_chksum_presence)
SKIP: test license presence: No added recipes, skipping test (test_metadata.TestMetadata.test_license_presence)
SKIP: test series merge on head: Merge test is disabled for now (test_mbox.TestMbox.test_series_merge_on_head)
SKIP: test src uri left files: Patch cannot be merged (test_metadata.TestMetadata.test_src_uri_left_files)
SKIP: test summary presence: No added recipes, skipping test (test_metadata.TestMetadata.test_summary_presence)
---
Please address the issues identified and
submit a new revision of the patch, or alternatively, reply to this email with an explanation of why the patch should be accepted. If you believe these results are due to an error in patchtest, please submit a bug at https://bugzilla.yoctoproject.org/ (use the 'Patchtest' category under 'Yocto Project Subprojects'). For more information on specific failures, see: https://wiki.yoctoproject.org/wiki/Patchtest. Thank you!
Re: [OE-core][PATCH 5/6] classes/create-spdx-3.0: Add class
In reply to Joshua WattHello,
This is my first round of the review, before applying the code.
On Mon, Jun 10, 2024 at 11:45 PM Joshua Watt via lists.openembedded.org <[email protected]> wrote:
| +SPDX_PROFILES ?= "core build software simpleLicensing security"
I would like to discuss this choice by default. People who do security won't need the 'build' profile (and this likely adds
much data). On the other hand, people who do licensing won't need the 'security' profile.
It would be really nice to generate a basic SPDX without doing the build...
Review
Quoted 3 lines
(quoted diff) +2
-0+ +SPDX_HASH_ALGORITHMS ?= "sha256 sha1"
Any reason to still use sha1? It is obsolete. We could use sha3 instead if we want to have multiple ones.
Quoted 64 lines
meta/classes/create-spdx-3.0.bbclass new file +28
-0+ + +python do_create_spdx() { + import oe.sbom30 + import oe.spdx30 + from pathlib import Path + from contextlib import contextmanager + import oe.cve_check + from datetime import datetime + + + def set_var_field(var, obj, name, package=None): + val = None + if package: + val = d.getVar("%s:%s" % (var, package)) + + if not val: + val = d.getVar(var) + + if val: + setattr(obj, name, val) + + deploydir = Path(d.getVar("SPDXDEPLOY")) + deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX")) + spdx_workdir = Path(d.getVar("SPDXWORK")) + include_sources = d.getVar("SPDX_INCLUDE_SOURCES") == "1" + pkg_arch = d.getVar("SSTATE_PKGARCH") + is_native = bb.data.inherits_class("native", d) or
bb.data.inherits_class("cross", d)meta/classes/create-spdx-3.0.bbclass new file +22
-0+ + build_objset = oe.sbom30.ObjectSet.new_objset(d, d.getVar("PN")) + + build = build_objset.new_sub_build("recipe", "recipe") + build_objset.doc.rootElement.append(build) + + build_objset.set_is_native(is_native) + + for var in (d.getVar('SPDX_CUSTOM_ANNOTATION_VARS') or "").split(): + new_annotation( + d, + build_objset, + build, + "%s=%s" % (var, d.getVar(var)), + oe.spdx30.AnnotationType.other + ) + + build_inputs = set() + + # Add CVEs + cve_by_status = {} + cve_by_status["Patched"] = {cve: build_objset.new_cve_vuln(cve) forcve in
oe.cve_check.get_patched_cves(d)}meta/classes/create-spdx-3.0.bbclass new file +8
-0+ + for cve in (d.getVarFlags("CVE_STATUS") or {}): + status, _, _ = oe.cve_check.decode_cve_status(d, cve) + # Already added above + if status == "Patched": + continue + + cve_by_status.setdefault(status, {})[cve] =
build_objset.new_cve_vuln(cve)(quoted diff) +2
-0+ + cpe_ids = oe.cve_check.get_cpe_ids(d.getVar("CVE_PRODUCT"),
Quoted 2 lines
d.getVar("CVE_VERSION"))
This will be adding only those from patches and everything with CVE_STATUS. In some cases (extra-exclusions) vulnerabilities
that have no real link with the given package. Also, the list isn't complete, does not contain most of the unpatched ones.
+do_create_spdx[vardepsexclude] += "BB_NUMBER_THREADS"
Quoted 8 lines
+# NOTE: depending on
do_unpackis a hack that is necessary to get it's dependencies for archive the source
+addtaskdo_create_spdxafterdo_populate_sysrootdo_packagedo_packagedatado_unpackdo_collect_spdx_depsdo_package_write_ipkdo_pacakge_write_debdo_package_write_rpmbeforedo_populate_sdkdo_builddo_rm_work
+
This is really late... I fear that if we get that in, it will be too hard to make the generation mode modulable and output
certain profiles at a different place than the whole build. With the same problem for the world build as we have today
with 2.2.
+def spdxid_hash(*items):
meta/lib/oe/sbom30.py line 138
Quoted 9 lines
meta/lib/oe/sbom30.py new file +8
-0+ h = hashlib.md5() + for i in items: + if isinstance(i, oe.spdx30.Element): + h.update(i._id.encode("utf-8")) + else: + h.update(i.encode("utf-8")) + return h.hexdigest() +
Can we avoid MD5?
Quoted 14 lines
meta/lib/oe/sbom30.py new file +8
-0+ def new_cve_vuln(self, cve): + v = oe.spdx30.security_Vulnerability() + v._id = self.new_spdxid("vulnerability", cve) + v.creationInfo = self.doc.creationInfo + + v.externalIdentifier.append( + oe.spdx30.ExternalIdentifier( +externalIdentifierType=oe.spdx30.ExternalIdentifierType.cve,
meta/lib/oe/sbom30.py new file +3
-0+ identifier=cve, + identifierLocator=[ + f"
meta/lib/oe/sbom30.py line 565
Quoted 2 lines
meta/lib/oe/sbom30.py new file +2
-0+ f"https://www.cve.org/CVERecord?id={cve}", + ],
Better to link to the JSON format https://cveawg.mitre.org/api/cve/{cve}
Quoted 6 lines
meta/lib/oe/sbom30.py new file +5
-0+ ) + ) + return self.add(v) + + def new_vex_patched_relationship(self, from_, to):
meta/lib/oe/sbom30.py line 580
Quoted 10 lines
meta/lib/oe/sbom30.py new file +10
-0+ self.add( + oe.spdx30.security_VexFixedVulnAssessmentRelationship( + _id=self.new_spdxid("vex-fixed", spdxid_hash(from_, *to)), + creationInfo=self.doc.creationInfo, + from_=from_, + relationshipType=oe.spdx30.RelationshipType.fixedIn, + to=to, + security_vexVersion=VEX_VERSION, + ) + )
For machine-readable entries, this should be linking to a proof of a fix, for example in
a relationship to the fix file.
Quoted 22 lines
meta/lib/oe/sbom30.py new file +5
-0+ + def new_vex_unpatched_relationship(self, from_, to): + self.add( + oe.spdx30.security_VexAffectedVulnAssessmentRelationship( + _id=self.new_spdxid("vex-affected", spdxid_hash(from_,*to)),
meta/lib/oe/sbom30.py new file +9
-0+ creationInfo=self.doc.creationInfo, + from_=from_, + relationshipType=oe.spdx30.RelationshipType.affects, + to=to, + security_vexVersion=VEX_VERSION, + ) + ) + + def new_vex_ignored_relationship(self, from_, to, *,
impact_statement):meta/lib/oe/sbom30.py new file +4
-0+ self.add( + oe.spdx30.security_VexNotAffectedVulnAssessmentRelationship( + _id=self.new_spdxid( + "vex-not-affected", spdxid_hash(impact_statement,
from_, *to)
meta/lib/oe/sbom30.py line 607
Quoted 9 lines
meta/lib/oe/sbom30.py new file +9
-0+ ), + creationInfo=self.doc.creationInfo, + from_=from_, + relationshipType=oe.spdx30.RelationshipType.doesNotAffect, + to=to, + security_vexVersion=VEX_VERSION, + security_impactStatement=impact_statement, + ) + )
Missing conversion of 'Ignored' to 'justificationType' which is necessary to allow machine processing of VEX statements. With 'impactStatement' only, which is freeform text, this is human readable only.
That's it in the first pass. There will be more when I can have a look in the output.
Regards,
Marta
Re: [OE-core][PATCH 5/6] classes/create-spdx-3.0: Add class
Quoted 4 lines · Marta Rybczynska
Hello,
This is my first round of the review, before applying the code.
Quoted 8 lines
On Mon, Jun 10, 2024 at 11:45 PM Joshua Watt via lists.openembedded.org <[email protected]> wrote:
| +SPDX_PROFILES ?= "core build software simpleLicensing security"
I would like to discuss this choice by default. People who do security won't need the 'build' profile (and this likely adds much data). On the other hand, people who do licensing won't need the 'security' profile.
It would be really nice to generate a basic SPDX without doing the build...
I think we should look at that later just to keep the scope reasonable; This is purposely trying to stick close to what SPDX 2.2 gives us.
Quoted 7 lines
Review
Quoted 4 lines
(quoted diff) +2
-0+ +SPDX_HASH_ALGORITHMS ?= "sha256 sha1"Any reason to still use sha1? It is obsolete. We could use sha3 instead if we want to have multiple ones.
I've dropped this in the latest version, in favor of only using sha256 to make my life easier :)
Quoted 69 lines
Quoted 54 lines
meta/classes/create-spdx-3.0.bbclass new file +50
-0+ + +python do_create_spdx() { + import oe.sbom30 + import oe.spdx30 + from pathlib import Path + from contextlib import contextmanager + import oe.cve_check + from datetime import datetime + + + def set_var_field(var, obj, name, package=None): + val = None + if package: + val = d.getVar("%s:%s" % (var, package)) + + if not val: + val = d.getVar(var) + + if val: + setattr(obj, name, val) + + deploydir = Path(d.getVar("SPDXDEPLOY")) + deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX")) + spdx_workdir = Path(d.getVar("SPDXWORK")) + include_sources = d.getVar("SPDX_INCLUDE_SOURCES") == "1" + pkg_arch = d.getVar("SSTATE_PKGARCH") + is_native = bb.data.inherits_class("native", d) or bb.data.inherits_class("cross", d) + + build_objset = oe.sbom30.ObjectSet.new_objset(d, d.getVar("PN")) + + build = build_objset.new_sub_build("recipe", "recipe") + build_objset.doc.rootElement.append(build) + + build_objset.set_is_native(is_native) + + for var in (d.getVar('SPDX_CUSTOM_ANNOTATION_VARS') or "").split(): + new_annotation( + d, + build_objset, + build, + "%s=%s" % (var, d.getVar(var)), + oe.spdx30.AnnotationType.other + ) + + build_inputs = set() + + # Add CVEs + cve_by_status = {} + cve_by_status["Patched"] = {cve: build_objset.new_cve_vuln(cve) for cve in oe.cve_check.get_patched_cves(d)}meta/classes/create-spdx-3.0.bbclass line 595
Quoted 10 lines
meta/classes/create-spdx-3.0.bbclass new file +10
-0+ + for cve in (d.getVarFlags("CVE_STATUS") or {}): + status, _, _ = oe.cve_check.decode_cve_status(d, cve) + # Already added above + if status == "Patched": + continue + + cve_by_status.setdefault(status, {})[cve] = build_objset.new_cve_vuln(cve) + + cpe_ids = oe.cve_check.get_cpe_ids(d.getVar("CVE_PRODUCT"), d.getVar("CVE_VERSION"))This will be adding only those from patches and everything with
CVE_STATUS. In some cases (extra-exclusions) vulnerabilities that have no real link with the given package. Also, the list isn't complete, does not contain most of the unpatched ones.
Quoted 9 lines
meta/classes/create-spdx-3.0.bbclass line 805
Quoted 4 lines
meta/classes/create-spdx-3.0.bbclass new file +4
-0+do_create_spdx[vardepsexclude] += "BB_NUMBER_THREADS" +# NOTE: depending on do_unpack is a hack that is necessary to get it's dependencies for archive the source +addtask do_create_spdx after do_populate_sysroot do_package do_packagedata do_unpack do_collect_spdx_deps do_package_write_ipk do_pacakge_write_deb do_package_write_rpm before do_populate_sdk do_build do_rm_work +This is really late... I fear that if we get that in, it will be too hard to make the generation mode modulable and output certain profiles at a different place than the whole build. With the same problem for the world build as we have today with 2.2.
Yep, but I think we need to focus on that after the initial implementation. It's a solvable problem, but it's going to to be tricky so it will take time, and I'd rather get SPDX 3 support out early.
Quoted 13 lines
meta/lib/oe/sbom30.py line 138
Quoted 10 lines
meta/lib/oe/sbom30.py new file +9
-0+def spdxid_hash(*items): + h = hashlib.md5() + for i in items: + if isinstance(i, oe.spdx30.Element): + h.update(i._id.encode("utf-8")) + else: + h.update(i.encode("utf-8")) + return h.hexdigest() +Can we avoid MD5?
It's not cryptographic, so it should not matter. I don't want the hash strings to be _terribly_ long, so unless you have some other short hashing algorithm, I'd rather stick with this.
Quoted 16 lines
Quoted 4 lines
meta/lib/oe/sbom30.py new file +3
-0+ def new_cve_vuln(self, cve): + v = oe.spdx30.security_Vulnerability() + v._id = self.new_spdxid("vulnerability", cve)meta/lib/oe/sbom30.py line 565
Quoted 10 lines
meta/lib/oe/sbom30.py new file +10
-0+ v.creationInfo = self.doc.creationInfo + + v.externalIdentifier.append( + oe.spdx30.ExternalIdentifier( + externalIdentifierType=oe.spdx30.ExternalIdentifierType.cve, + identifier=cve, + identifierLocator=[ + f"https://cve.mitre.org/cgi-bin/cvename.cgi?name={cve}", + f"https://www.cve.org/CVERecord?id={cve}", + ],
Quoted 1 line
Better to link to the JSON format https://cveawg.mitre.org/api/cve/{cve}
Will do, thanks. Should that be a third entry, or the only entry, or.... ?
Quoted 19 lines
Quoted 7 lines
meta/lib/oe/sbom30.py new file +5
-0+ ) + ) + return self.add(v) + + def new_vex_patched_relationship(self, from_, to):meta/lib/oe/sbom30.py line 580
Quoted 10 lines
meta/lib/oe/sbom30.py new file +10
-0+ self.add( + oe.spdx30.security_VexFixedVulnAssessmentRelationship( + _id=self.new_spdxid("vex-fixed", spdxid_hash(from_, *to)), + creationInfo=self.doc.creationInfo, + from_=from_, + relationshipType=oe.spdx30.RelationshipType.fixedIn, + to=to, + security_vexVersion=VEX_VERSION, + ) + )
Quoted 2 lines
For machine-readable entries, this should be linking to a proof of a fix, for example in a relationship to the fix file.
Yes, I think we can do that, but we need cve_check.py to be able to give us the file names (or hashes?) so we can correlate them.... I'll log a bug to do that later.
Quoted 24 lines
Quoted 24 lines
meta/lib/oe/sbom30.py new file +22
-0+ + def new_vex_unpatched_relationship(self, from_, to): + self.add( + oe.spdx30.security_VexAffectedVulnAssessmentRelationship( + _id=self.new_spdxid("vex-affected", spdxid_hash(from_, *to)), + creationInfo=self.doc.creationInfo, + from_=from_, + relationshipType=oe.spdx30.RelationshipType.affects, + to=to, + security_vexVersion=VEX_VERSION, + ) + ) + + def new_vex_ignored_relationship(self, from_, to, *, impact_statement): + self.add( + oe.spdx30.security_VexNotAffectedVulnAssessmentRelationship( + _id=self.new_spdxid( + "vex-not-affected", spdxid_hash(impact_statement, from_, *to) + ), + creationInfo=self.doc.creationInfo, + from_=from_, + relationshipType=oe.spdx30.RelationshipType.doesNotAffect,
Quoted 8 lines
meta/lib/oe/sbom30.py line 607
Quoted 5 lines
meta/lib/oe/sbom30.py new file +5
-0+ to=to, + security_vexVersion=VEX_VERSION, + security_impactStatement=impact_statement, + ) + )Missing conversion of 'Ignored' to 'justificationType' which is necessary to allow machine processing of VEX statements. With 'impactStatement' only, which is freeform text, this is human readable only.
Ya, I think I can add that based on the not-applicable-platform or not-applicable-config
Quoted 5 lines
That's it in the first pass. There will be more when I can have a look in the output.
Regards,
Marta
PATCH 6/6 classes-recipe/image_types: Add SPDX_IMAGE_PURPOSE to images
In reply to Joshua WattAdds the variable overrides to set the SPDX image purpose for various image types
--- meta/classes-recipe/image_types.bbclass | 2 ++ meta/classes-recipe/image_types_wic.bbclass | 1 + 2 files changed, 3 insertions(+)
meta/classes-recipe/image_types.bbclass +2 -0
@@ -146,6 +146,7 @@ IMAGE_CMD:vfat = "oe_mkvfatfs ${EXTRA_IMAGECMD}" IMAGE_CMD_TAR ?= "tar" # ignore return code 1 "file changed as we read it" as other tasks(e.g. do_image_wic) may be hardlinking rootfs IMAGE_CMD:tar = "${IMAGE_CMD_TAR} --sort=name --format=posix --numeric-owner -cf ${IMGDEPLOYDIR}/${IMAGE_NAME}.tar -C ${IMAGE_ROOTFS} . || [ $? -eq 1 ]" +SPDX_IMAGE_PURPOSE:tar = "archive" do_image_cpio[cleandirs] += "${WORKDIR}/cpio_append" IMAGE_CMD:cpio () {
@@ -167,6 +168,7 @@ IMAGE_CMD:cpio () { fi fi } +SPDX_IMAGE_PURPOSE:cpio = "archive" UBI_VOLNAME ?= "${MACHINE}-rootfs" UBI_VOLTYPE ?= "dynamic"
meta/classes-recipe/image_types_wic.bbclass +1 -0
@@ -91,6 +91,7 @@ IMAGE_CMD:wic () { mv "$build_wic/$(basename "${wks%.wks}")"*.${IMAGER} "$out.wic" } IMAGE_CMD:wic[vardepsexclude] = "WKS_FULL_PATH WKS_FILES TOPDIR" +SPDX_IMAGE_PURPOSE:wic = "diskImage" do_image_wic[cleandirs] = "${WORKDIR}/build-wic" PSEUDO_IGNORE_PATHS .= ",${WORKDIR}/build-wic"
Signature · 1 line
2.43.2
Quoted 7 lines · Joshua Watt via lists.openembedded.org
This patch series add support for SPDX 3.0. Currently it is not possible to have SPDX 2.2 and SPDX 3.0 enabled at the same time; users may choose to specifically use SPDX 3.0 instead of 2.2 by doing INHERIT += "create-spdx-3.0"
The default version is not changed yet, although that will likely happen soon
Does this series depend on other patches?
Cheers,
Richard
Quoted 11 lines · Joshua Watt via lists.openembedded.org
This patch series add support for SPDX 3.0. Currently it is not possible
to have SPDX 2.2 and SPDX 3.0 enabled at the same time; users may choose
to specifically use SPDX 3.0 instead of 2.2 by doing INHERIT += "create-spdx-3.0"The default version is not changed yet, although that will likely happen
soon
Quoted 7 lines
Joshua Watt (6):
classes-recipe/image: Add image file manifest classes/spdx-common: Move common SPDX to new class classes/spdx-common: Add SPDX version to path classes/spdx-common: Return empty list fromextract_licensesclasses/create-spdx-3.0: Add class classes-recipe/image_types: AddSPDX_IMAGE_PURPOSEto images
I set 3.0 to the default for testing and there were some interesting looking failures:
https://valkyrie.yoctoproject.org/#/builders/43/builds/52/steps/11/logs/stdio
I'm guessing:
https://valkyrie.yoctoproject.org/#/builders/37/builds/60
is due to a missing patch?
Other failures:
https://valkyrie.yoctoproject.org/#/builders/74/builds/44/steps/13/logs/stdio
https://valkyrie.yoctoproject.org/#/builders/93/builds/51/steps/11/logs/stdio
but I stopped the build at this point.
Cheers,
Richard
Quoted 25 lines · Richard Purdie
Quoted 11 lines · Joshua Watt via lists.openembedded.org
This patch series add support for SPDX 3.0. Currently it is not possible
to have SPDX 2.2 and SPDX 3.0 enabled at the same time; users may choose
to specifically use SPDX 3.0 instead of 2.2 by doing INHERIT += "create-spdx-3.0"The default version is not changed yet, although that will likely happen
soonQuoted 7 lines
Joshua Watt (6):
classes-recipe/image: Add image file manifest classes/spdx-common: Move common SPDX to new class classes/spdx-common: Add SPDX version to path classes/spdx-common: Return empty list fromextract_licensesclasses/create-spdx-3.0: Add class classes-recipe/image_types: AddSPDX_IMAGE_PURPOSEto imagesI set 3.0 to the default for testing and there were some interesting looking failures:
Quoted 7 lines
https://valkyrie.yoctoproject.org/#/builders/43/builds/52/steps/11/logs/stdio
I'm guessing:
https://valkyrie.yoctoproject.org/#/builders/37/builds/60
is due to a missing patch?
Hah, I don't think so, I just neglected to test SDK builds. My bad.
Quoted 5 lines
Other failures:
https://valkyrie.yoctoproject.org/#/builders/74/builds/44/steps/13/logs/stdio
https://valkyrie.yoctoproject.org/#/builders/93/builds/51/steps/11/logs/stdio
That one appears to be some corner case I need to figure out.
Quoted 2 lines
but I stopped the build at this point.
Thanks
Quoted 4 lines
Cheers,
Richard
v2 cover Add SPDX 3.0 support
In reply to Joshua WattThis patch series add support for SPDX 3.0 and sets it as the default. Currently it is not possible to have SPDX 2.2 and SPDX 3.0 enabled at the same time; users may choose to specifically use SPDX 3.0 instead of 2.2 by doing INHERIT += "create-spdx-3.0"
v2: Added tests and addressed feedback
Joshua Watt (7):
classes-recipe/image: Add image file manifest classes/create-spdx-3.0: Add class classes-recipe/image_types: Add SPDX_IMAGE_PURPOSE to images
selftest: spdx: Add SPDX 3.0 test cases
classes-recipe: nospdx: Add class classes/spdx-common: Move SPDX_SUPPLIER
Switch default spdx version to 3.0
meta/classes-recipe/image.bbclass | 58 + meta/classes-recipe/image_types.bbclass | 2 + meta/classes-recipe/image_types_wic.bbclass | 1 + meta/classes-recipe/nospdx.bbclass | 13 + meta/classes-recipe/packagegroup.bbclass | 2 + meta/classes/create-spdx-2.2.bbclass | 8 + meta/classes/create-spdx-3.0.bbclass | 1116 ++++ meta/classes/create-spdx-image-3.0.bbclass | 203 + meta/classes/create-spdx.bbclass | 2 +- meta/classes/spdx-common.bbclass | 32 +- meta/lib/oe/sbom30.py | 1118 ++++ meta/lib/oe/spdx30.py | 5996 ++++++++++++++++++ meta/lib/oeqa/selftest/cases/spdx.py | 119 +- meta/recipes-core/meta/build-sysroots.bb | 5 +- meta/recipes-core/meta/meta-world-pkgdata.bb | 3 +- 15 files changed, 8654 insertions(+), 24 deletions(-) create mode 100644 meta/classes-recipe/nospdx.bbclass create mode 100644 meta/classes/create-spdx-3.0.bbclass create mode 100644 meta/classes/create-spdx-image-3.0.bbclass create mode 100644 meta/lib/oe/sbom30.py create mode 100644 meta/lib/oe/spdx30.py
Signature · 1 line
2.43.2
PATCH v2 1/7 classes-recipe/image: Add image file manifest
Downstream tasks may want to know what image files were written by the do_image family of tasks (e.g. SPDX) so have each task write out a manifest file that describes the files it produced, then aggregate them in do_image_complete
--- meta/classes-recipe/image.bbclass | 58 +++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+)
meta/classes-recipe/image.bbclass +58 -0
@@ -88,6 +88,11 @@ PACKAGE_INSTALL_ATTEMPTONLY ?= "${FEATURE_INSTALL_OPTIONAL}" IMGDEPLOYDIR = "${WORKDIR}/deploy-${PN}-image-complete" +IMGMANIFESTDIR = "${WORKDIR}/image-task-manifest" + +IMAGE_FILE_MANIFEST_DIR = "${WORKDIR}/deploy-image-file-manifest" +IMAGE_FILE_MANIFEST = "${IMAGE_FILE_MANIFEST_DIR}/manifest.json" + # Images are generally built explicitly, do not need to be part of world. EXCLUDE_FROM_WORLD = "1"
@@ -277,14 +282,28 @@ fakeroot python do_image () { execute_pre_post_process(d, pre_process_cmds) } do_image[dirs] = "${TOPDIR}" +do_image[cleandirs] += "${IMGMANIFESTDIR}" addtask do_image after do_rootfs fakeroot python do_image_complete () { from oe.utils import execute_pre_post_process + from pathlib import Path + import json post_process_cmds = d.getVar("IMAGE_POSTPROCESS_COMMAND") execute_pre_post_process(d, post_process_cmds) + + image_manifest_dir = Path(d.getVar('IMGMANIFESTDIR')) + + data = [] + + for manifest_path in image_manifest_dir.glob("*.json"): + with manifest_path.open("r") as f: + data.extend(json.load(f)) + + with open(d.getVar("IMAGE_FILE_MANIFEST"), "w") as f: + json.dump(data, f) } do_image_complete[dirs] = "${TOPDIR}" SSTATETASKS += "do_image_complete"
@@ -292,6 +311,8 @@ SSTATE_SKIP_CREATION:task-image-complete = '1' do_image_complete[sstate-inputdirs] = "${IMGDEPLOYDIR}" do_image_complete[sstate-outputdirs] = "${DEPLOY_DIR_IMAGE}" do_image_complete[stamp-extra-info] = "${MACHINE_ARCH}" +do_image_complete[sstate-plaindirs] = "${IMAGE_FILE_MANIFEST_DIR}" +do_image_complete[dirs] += "${IMAGE_FILE_MANIFEST_DIR}" addtask do_image_complete after do_image before do_build python do_image_complete_setscene () { sstate_setscene(d)
@@ -507,12 +528,14 @@ python () { d.setVar(task, '\n'.join(cmds)) d.setVarFlag(task, 'func', '1') d.setVarFlag(task, 'fakeroot', '1') + d.setVarFlag(task, 'imagetype', t) d.appendVarFlag(task, 'prefuncs', ' ' + debug + ' set_image_size') d.prependVarFlag(task, 'postfuncs', 'create_symlinks ') d.appendVarFlag(task, 'subimages', ' ' + ' '.join(subimages)) d.appendVarFlag(task, 'vardeps', ' ' + ' '.join(vardeps)) d.appendVarFlag(task, 'vardepsexclude', ' DATETIME DATE ' + ' '.join(vardepsexclude)) + d.appendVarFlag(task, 'postfuncs', ' write_image_file_manifest') bb.debug(2, "Adding task %s before %s, after %s" % (task, 'do_image_complete', after)) bb.build.addtask(task, 'do_image_complete', after, d)
@@ -610,6 +633,41 @@ python create_symlinks() { bb.note("Skipping symlink, source does not exist: %s -> %s" % (dst, src)) } +python write_image_file_manifest() { + import json + from pathlib import Path + + taskname = d.getVar("BB_CURRENTTASK") + image_deploy_dir = Path(d.getVar('IMGDEPLOYDIR')) + image_manifest_dir = Path(d.getVar('IMGMANIFESTDIR')) + manifest_path = image_manifest_dir / ("do_" + d.getVar("BB_CURRENTTASK") + ".json") + + image_name = d.getVar("IMAGE_NAME") + image_basename = d.getVar("IMAGE_BASENAME") + machine = d.getVar("MACHINE") + + subimages = (d.getVarFlag("do_" + taskname, 'subimages', False) or "").split() + imagetype = d.getVarFlag("do_" + taskname, 'imagetype', False) + + data = { + "taskname": taskname, + "imagetype": imagetype, + "images": [] + } + + for type in subimages: + image_filename = image_name + "." + type + image_path = image_deploy_dir / image_filename + if not image_path.exists(): + continue + data["images"].append({ + "filename": image_filename, + }) + + with manifest_path.open("w") as f: + json.dump([data], f) +} + MULTILIBRE_ALLOW_REP += "${base_bindir} ${base_sbindir} ${bindir} ${sbindir} ${libexecdir} ${sysconfdir} ${nonarch_base_libdir}/udev /lib/modules/[^/]*/modules.*" MULTILIB_CHECK_FILE = "${WORKDIR}/multilib_check.py" MULTILIB_TEMP_ROOTFS = "${WORKDIR}/multilib"
Signature · 1 line
2.43.2
PATCH v2 2/7 classes/create-spdx-3.0: Add classs
In reply to Joshua WattAdds a class to generate SPDX 3.0 output and an image class that is used when generating images
--- meta/classes-recipe/packagegroup.bbclass | 2 + meta/classes/create-spdx-3.0.bbclass | 1116 ++++ meta/classes/create-spdx-image-3.0.bbclass | 203 + meta/classes/spdx-common.bbclass | 25 +- meta/lib/oe/sbom30.py | 1118 ++++ meta/lib/oe/spdx30.py | 5996 ++++++++++++++++++++ 6 files changed, 8459 insertions(+), 1 deletion(-) create mode 100644 meta/classes/create-spdx-3.0.bbclass create mode 100644 meta/classes/create-spdx-image-3.0.bbclass create mode 100644 meta/lib/oe/sbom30.py create mode 100644 meta/lib/oe/spdx30.py
meta/classes-recipe/packagegroup.bbclass +2 -0
@@ -56,6 +56,8 @@ deltask do_populate_sysroot do_create_runtime_spdx[deptask] = "do_create_spdx" do_create_runtime_spdx[rdeptask] = "" +do_create_package_spdx[deptask] = "do_create_spdx" +do_create_package_spdx[rdeptask] = "" INHIBIT_DEFAULT_DEPS = "1"
meta/classes/create-spdx-3.0.bbclass new file +1116 -0
1,116 lines not on this page · raw message
meta/classes/create-spdx-image-3.0.bbclass new file +203 -0
203 lines not on this page · raw message
meta/classes/spdx-common.bbclass +24 -1
@@ -16,13 +16,17 @@ SPDXDIR ??= "${WORKDIR}/spdx/${SPDX_VERSION}" SPDXDEPLOY = "${SPDXDIR}/deploy" SPDXWORK = "${SPDXDIR}/work" SPDXIMAGEWORK = "${SPDXDIR}/image-work" +SPDXROOTFSWORK = "${SPDXDIR}/rootfs-work" SPDXSDKWORK = "${SPDXDIR}/sdk-work" SPDXDEPS = "${SPDXDIR}/deps.json" +SPDXIMAGEDEPLOYDIR = "${SPDXDIR}/image-deploy" +SPDX_ROOTFS_PACKAGES = "${SPDXDIR}/rootfs-packages.json" SPDX_TOOL_NAME ??= "oe-spdx-creator" SPDX_TOOL_VERSION ??= "1.0" SPDXRUNTIMEDEPLOY = "${SPDXDIR}/runtime-deploy" +SPDXROOTFSDEPLOY = "${SPDXDIR}/rootfs-deploy" SPDX_INCLUDE_SOURCES ??= "0" SPDX_ARCHIVE_SOURCES ??= "0"
@@ -68,7 +72,7 @@ def get_json_indent(d): return 2 return None -python() { +def load_spdx_license_data(d): import json if d.getVar("SPDX_LICENSE_DATA"): return
@@ -78,6 +82,9 @@ python() { # Transform the license array to a dictionary data["licenses"] = {l["licenseId"]: l for l in data["licenses"]} d.setVar("SPDX_LICENSE_DATA", data) + +python() { + load_spdx_license_data(d) } def process_sources(d):
@@ -255,3 +262,19 @@ def spdx_get_src(d): spdx_get_src[vardepsexclude] += "STAGING_KERNEL_DIR" + +python spdx_collect_rootfs_packages() { + import json + from pathlib import Path + from oe.rootfs import image_list_installed_packages + + root_packages_file = Path(d.getVar("SPDX_ROOTFS_PACKAGES")) + + packages = image_list_installed_packages(d) + + root_packages_file.parent.mkdir(parents=True, exist_ok=True) + with root_packages_file.open("w") as f: + json.dump(packages, f) +} +ROOTFS_POSTUNINSTALL_COMMAND =+ "spdx_collect_rootfs_packages" +
meta/lib/oe/sbom30.py new file +1118 -0
1,118 lines not on this page · raw message
meta/lib/oe/spdx30.py new file +5996 -0
502 lines not on this page · raw message
The rest of this diff is not shown. View the raw message.
Signature · 1 line
2.43.2
Part of the diff is not shown here. View the raw message.
Patchtest results for [OE-core][PATCH v2 2/7] classes/create-spdx-3.0: Add classs
Thank you for your submission. Patchtest identified one or more issues with the patch. Please see the log below for more information:
---
Testing patch /home/patchtest/share/mboxes/v2-2-7-classes-create-spdx-3.0-Add-classs.patch
FAIL: test max line length: Patch line too long (current length 216, maximum is 200) (test_metadata.TestMetadata.test_max_line_length)
PASS: pretest pylint (test_python_pylint.PyLint.pretest_pylint)
PASS: test Signed-off-by presence (test_mbox.TestMbox.test_signed_off_by_presence)
PASS: test author valid (test_mbox.TestMbox.test_author_valid)
PASS: test commit message presence (test_mbox.TestMbox.test_commit_message_presence)
PASS: test mbox format (test_mbox.TestMbox.test_mbox_format)
PASS: test non-AUH upgrade (test_mbox.TestMbox.test_non_auh_upgrade)
PASS: test pylint (test_python_pylint.PyLint.test_pylint)
PASS: test shortlog format (test_mbox.TestMbox.test_shortlog_format)
PASS: test shortlog length (test_mbox.TestMbox.test_shortlog_length)
SKIP: pretest src uri left files: No modified recipes, skipping pretest (test_metadata.TestMetadata.pretest_src_uri_left_files)
SKIP: test CVE check ignore: No modified recipes or older target branch, skipping test (test_metadata.TestMetadata.test_cve_check_ignore)
SKIP: test CVE tag format: No new CVE patches introduced (test_patch.TestPatch.test_cve_tag_format)
SKIP: test Signed-off-by presence: No new CVE patches introduced (test_patch.TestPatch.test_signed_off_by_presence)
SKIP: test Upstream-Status presence: No new CVE patches introduced (test_patch.TestPatch.test_upstream_status_presence_format)
SKIP: test bugzilla entry format: No bug ID found (test_mbox.TestMbox.test_bugzilla_entry_format)
SKIP: test lic files chksum modified not mentioned: No modified recipes, skipping test (test_metadata.TestMetadata.test_lic_files_chksum_modified_not_mentioned)
SKIP: test lic files chksum presence: No added recipes, skipping test (test_metadata.TestMetadata.test_lic_files_chksum_presence)
SKIP: test license presence: No added recipes, skipping test (test_metadata.TestMetadata.test_license_presence)
SKIP: test series merge on head: Merge test is disabled for now (test_mbox.TestMbox.test_series_merge_on_head)
SKIP: test src uri left files: No modified recipes, skipping pretest (test_metadata.TestMetadata.test_src_uri_left_files)
SKIP: test summary presence: No added recipes, skipping test (test_metadata.TestMetadata.test_summary_presence)
SKIP: test target mailing list: Series merged, no reason to check other mailing lists (test_mbox.TestMbox.test_target_mailing_list)
---
Please address the issues identified and
submit a new revision of the patch, or alternatively, reply to this email with an explanation of why the patch should be accepted. If you believe these results are due to an error in patchtest, please submit a bug at https://bugzilla.yoctoproject.org/ (use the 'Patchtest' category under 'Yocto Project Subprojects'). For more information on specific failures, see: https://wiki.yoctoproject.org/wiki/Patchtest. Thank you!
PATCH v2 3/7 classes-recipe/image_types: Add SPDX_IMAGE_PURPOSE to images
In reply to Joshua WattAdds the variable overrides to set the SPDX image purpose for various image types
--- meta/classes-recipe/image_types.bbclass | 2 ++ meta/classes-recipe/image_types_wic.bbclass | 1 + 2 files changed, 3 insertions(+)
meta/classes-recipe/image_types.bbclass +2 -0
@@ -146,6 +146,7 @@ IMAGE_CMD:vfat = "oe_mkvfatfs ${EXTRA_IMAGECMD}" IMAGE_CMD_TAR ?= "tar" # ignore return code 1 "file changed as we read it" as other tasks(e.g. do_image_wic) may be hardlinking rootfs IMAGE_CMD:tar = "${IMAGE_CMD_TAR} --sort=name --format=posix --numeric-owner -cf ${IMGDEPLOYDIR}/${IMAGE_NAME}.tar -C ${IMAGE_ROOTFS} . || [ $? -eq 1 ]" +SPDX_IMAGE_PURPOSE:tar = "archive" do_image_cpio[cleandirs] += "${WORKDIR}/cpio_append" IMAGE_CMD:cpio () {
@@ -167,6 +168,7 @@ IMAGE_CMD:cpio () { fi fi } +SPDX_IMAGE_PURPOSE:cpio = "archive" UBI_VOLNAME ?= "${MACHINE}-rootfs" UBI_VOLTYPE ?= "dynamic"
meta/classes-recipe/image_types_wic.bbclass +1 -0
@@ -91,6 +91,7 @@ IMAGE_CMD:wic () { mv "$build_wic/$(basename "${wks%.wks}")"*.${IMAGER} "$out.wic" } IMAGE_CMD:wic[vardepsexclude] = "WKS_FULL_PATH WKS_FILES TOPDIR" +SPDX_IMAGE_PURPOSE:wic = "diskImage" do_image_wic[cleandirs] = "${WORKDIR}/build-wic" PSEUDO_IGNORE_PATHS .= ",${WORKDIR}/build-wic"
Signature · 1 line
2.43.2
PATCH v2 4/7 selftest: spdx: Add SPDX 3.0 test cases
In reply to Joshua WattAdds test cases for SPDX 3.0. Reworks the SPDX 2.2 test setup so it can also be run even if the default is SPDX 3.0
--- meta/lib/oeqa/selftest/cases/spdx.py | 119 +++++++++++++++++++++++++-- 1 file changed, 110 insertions(+), 9 deletions(-)
meta/lib/oeqa/selftest/cases/spdx.py +110 -9
@@ -6,21 +6,26 @@ import json import os +import textwrap +from pathlib import Path from oeqa.selftest.case import OESelftestTestCase -from oeqa.utils.commands import bitbake, get_bb_var, runCmd +from oeqa.utils.commands import bitbake, get_bb_var, get_bb_vars, runCmd -class SPDXCheck(OESelftestTestCase): +class SPDX22Check(OESelftestTestCase): @classmethod def setUpClass(cls): - super(SPDXCheck, cls).setUpClass() + super().setUpClass() bitbake("python3-spdx-tools-native") bitbake("-c addto_recipe_sysroot python3-spdx-tools-native") def check_recipe_spdx(self, high_level_dir, spdx_file, target_name): - config = """ -INHERIT += "create-spdx" -""" + config = textwrap.dedent( + """\ + INHERIT:remove = "create-spdx" + INHERIT += "create-spdx-2.2" + """ + ) self.write_config(config) deploy_dir = get_bb_var("DEPLOY_DIR")
@@ -29,7 +34,9 @@ INHERIT += "create-spdx" # qemux86-64 creates the directory qemux86_64 machine_dir = machine_var.replace("-", "_") - full_file_path = os.path.join(deploy_dir, "spdx", spdx_version, machine_dir, high_level_dir, spdx_file) + full_file_path = os.path.join( + deploy_dir, "spdx", spdx_version, machine_dir, high_level_dir, spdx_file + ) try: os.remove(full_file_path)
@@ -44,8 +51,13 @@ INHERIT += "create-spdx" self.assertNotEqual(report, None) self.assertNotEqual(report["SPDXID"], None) - python = os.path.join(get_bb_var('STAGING_BINDIR', 'python3-spdx-tools-native'), 'nativepython3') - validator = os.path.join(get_bb_var('STAGING_BINDIR', 'python3-spdx-tools-native'), 'pyspdxtools') + python = os.path.join( + get_bb_var("STAGING_BINDIR", "python3-spdx-tools-native"), + "nativepython3", + ) + validator = os.path.join( + get_bb_var("STAGING_BINDIR", "python3-spdx-tools-native"), "pyspdxtools" + ) result = runCmd("{} {} -i {}".format(python, validator, filename)) self.assertExists(full_file_path)
@@ -53,3 +65,92 @@ INHERIT += "create-spdx" def test_spdx_base_files(self): self.check_recipe_spdx("packages", "base-files.spdx.json", "base-files") + + +class SPDX3CheckBase(object): + """ + Base class for checking SPDX 3 based tests + """ + + def check_spdx_file(self, filename): + import oe.spdx30 + + self.assertExists(filename) + + # Read the file + objset = oe.spdx30.SHACLObjectSet() + with open(filename, "r") as f: + d = oe.spdx30.JSONLDDeserializer() + d.read(f, objset) + + return objset + + def check_recipe_spdx(self, target_name, spdx_path, *, task=None): + config = textwrap.dedent( + f"""\ + INHERIT:remove = "create-spdx" + INHERIT += "{self.SPDX_CLASS}" + """ + ) + self.write_config(config) + + if task: + bitbake(f"-c {task} {target_name}") + else: + bitbake(target_name) + + filename = spdx_path.format( + **get_bb_vars( + [ + "DEPLOY_DIR_IMAGE", + "DEPLOY_DIR_SPDX", + "MACHINE", + "MACHINE_ARCH", + "SDKMACHINE", + "SDK_DEPLOY", + "SPDX_VERSION", + "TOOLCHAIN_OUTPUTNAME", + ], + target_name, + ) + ) + + return self.check_spdx_file(filename) + + def check_objset_missing_ids(self, objset): + if objset.missing_ids: + self.assertTrue( + False, + "The following SPDXIDs are unresolved:\n " + + "\n ".join(objset.missing_ids), + ) + + +class SPDX30Check(SPDX3CheckBase, OESelftestTestCase): + SPDX_CLASS = "create-spdx-3.0" + SCHEMA_URL = "https://spdx.org/schema/3.0.0/spdx-json-schema.json" + + def test_base_files(self): + self.check_recipe_spdx( + "base-files", + "{DEPLOY_DIR_SPDX}/{MACHINE_ARCH}/packages/base-files.spdx.json", + ) + + def test_core_image_minimal(self): + objset = self.check_recipe_spdx( + "core-image-minimal", + "{DEPLOY_DIR_IMAGE}/core-image-minimal-{MACHINE}.rootfs.spdx.json", + ) + + # Document should be fully linked + self.check_objset_missing_ids(objset) + + def test_core_image_minimal_sdk(self): + objset = self.check_recipe_spdx( + "core-image-minimal", + "{SDK_DEPLOY}/{TOOLCHAIN_OUTPUTNAME}.spdx.json", + task="populate_sdk", + ) + + # Document should be fully linked + self.check_objset_missing_ids(objset)
Signature · 1 line
2.43.2
PATCH v2 5/7 classes-recipe: nospdx: Add class
In reply to Joshua WattAdds a class that allows recipes to opt out of generating SPDX
--- meta/classes-recipe/nospdx.bbclass | 13 +++++++++++++ meta/recipes-core/meta/build-sysroots.bb | 5 +---- meta/recipes-core/meta/meta-world-pkgdata.bb | 3 +-- 3 files changed, 15 insertions(+), 6 deletions(-) create mode 100644 meta/classes-recipe/nospdx.bbclass
meta/classes-recipe/nospdx.bbclass new file +13 -0
@@ -0,0 +1,13 @@ +# +# Copyright OpenEmbedded Contributors +# +# SPDX-License-Identifier: MIT +# + +deltask do_collect_spdx_deps +deltask do_create_spdx +deltask do_create_spdx_runtime +deltask do_create_package_spdx +deltask do_create_rootfs_spdx +deltask do_create_image_spdx +deltask do_create_image_sbom
meta/recipes-core/meta/build-sysroots.bb +1 -4
@@ -7,7 +7,7 @@ STANDALONE_SYSROOT_NATIVE = "${STAGING_DIR}/${BUILD_ARCH}" PACKAGE_ARCH = "${MACHINE_ARCH}" EXCLUDE_FROM_WORLD = "1" -inherit nopackages +inherit nopackages nospdx deltask fetch deltask unpack deltask patch
@@ -17,9 +17,6 @@ deltask configure deltask compile deltask install deltask populate_sysroot -deltask create_spdx -deltask collect_spdx_deps -deltask create_runtime_spdx deltask recipe_qa do_build_warn () {
meta/recipes-core/meta/meta-world-pkgdata.bb +1 -2
@@ -27,14 +27,13 @@ python do_collect_packagedata() { oe.copy_buildsystem.generate_locked_sigs(sigfile, d) } +inherit nospdx deltask do_fetch deltask do_unpack deltask do_patch deltask do_configure deltask do_compile deltask do_install -deltask do_create_spdx -deltask do_create_spdx_runtime do_prepare_recipe_sysroot[deptask] = ""
Signature · 1 line
2.43.2
PATCH v2 6/7 classes/spdx-common: Move SPDX_SUPPLIER
In reply to Joshua WattMove the SPDX_SUPPLIER variable to create-spdx-2.2 since it's format only has meaning in SPDX 2.2 (SPDX 3 uses SPDX_PACKAGE_SUPPLIER with a different format)
--- meta/classes/create-spdx-2.2.bbclass | 8 ++++++++ meta/classes/spdx-common.bbclass | 7 ------- 2 files changed, 8 insertions(+), 7 deletions(-)
meta/classes/create-spdx-2.2.bbclass +8 -0
@@ -8,6 +8,14 @@ inherit spdx-common SPDX_VERSION = "2.2" +SPDX_ORG ??= "OpenEmbedded ()" +SPDX_SUPPLIER ??= "Organization: ${SPDX_ORG}" +SPDX_SUPPLIER[doc] = "The SPDX PackageSupplier field for SPDX packages created from \ + this recipe. For SPDX documents create using this class during the build, this \ + is the contact information for the person or organization who is doing the \ + build." + + def get_namespace(d, name): import uuid namespace_uuid = uuid.uuid5(uuid.NAMESPACE_DNS, d.getVar("SPDX_UUID_NAMESPACE"))
meta/classes/spdx-common.bbclass +0 -7
@@ -40,13 +40,6 @@ SPDX_LICENSES ??= "${COREBASE}/meta/files/spdx-licenses.json" SPDX_CUSTOM_ANNOTATION_VARS ??= "" -SPDX_ORG ??= "OpenEmbedded ()" -SPDX_SUPPLIER ??= "Organization: ${SPDX_ORG}" -SPDX_SUPPLIER[doc] = "The SPDX PackageSupplier field for SPDX packages created from \ - this recipe. For SPDX documents create using this class during the build, this \ - is the contact information for the person or organization who is doing the \ - build." - def extract_licenses(filename): import re
Signature · 1 line
2.43.2
PATCH v2 7/7 Switch default spdx version to 3.0
In reply to Joshua WattChanges the default SPDX version to 3.0
--- meta/classes/create-spdx.bbclass | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
meta/classes/create-spdx.bbclass +1 -1
@@ -5,4 +5,4 @@ # # Include this class when you don't care what version of SPDX you get; it will # be updated to the latest stable version that is supported -inherit create-spdx-2.2 +inherit create-spdx-3.0
Signature · 1 line
2.43.2
Patchtest results for [OE-core][PATCH v2 7/7] Switch default spdx version to 3.0
Thank you for your submission. Patchtest identified one or more issues with the patch. Please see the log below for more information:
---
Testing patch /home/patchtest/share/mboxes/v2-7-7-Switch-default-spdx-version-to-3.0.patch
FAIL: test shortlog format: Commit shortlog (first line of commit message) should follow the format "<target>: <summary>" (test_mbox.TestMbox.test_shortlog_format)
PASS: test Signed-off-by presence (test_mbox.TestMbox.test_signed_off_by_presence)
PASS: test author valid (test_mbox.TestMbox.test_author_valid)
PASS: test commit message presence (test_mbox.TestMbox.test_commit_message_presence)
PASS: test mbox format (test_mbox.TestMbox.test_mbox_format)
PASS: test non-AUH upgrade (test_mbox.TestMbox.test_non_auh_upgrade)
PASS: test shortlog length (test_mbox.TestMbox.test_shortlog_length)
SKIP: pretest pylint: No python related patches, skipping test (test_python_pylint.PyLint.pretest_pylint)
SKIP: pretest src uri left files: No modified recipes, skipping pretest (test_metadata.TestMetadata.pretest_src_uri_left_files)
SKIP: test CVE tag format: No new CVE patches introduced (test_patch.TestPatch.test_cve_tag_format)
SKIP: test Signed-off-by presence: No new CVE patches introduced (test_patch.TestPatch.test_signed_off_by_presence)
SKIP: test Upstream-Status presence: No new CVE patches introduced (test_patch.TestPatch.test_upstream_status_presence_format)
SKIP: test bugzilla entry format: No bug ID found (test_mbox.TestMbox.test_bugzilla_entry_format)
SKIP: test pylint: No python related patches, skipping test (test_python_pylint.PyLint.test_pylint)
SKIP: test series merge on head: Merge test is disabled for now (test_mbox.TestMbox.test_series_merge_on_head)
SKIP: test target mailing list: Series merged, no reason to check other mailing lists (test_mbox.TestMbox.test_target_mailing_list)
---
Please address the issues identified and
submit a new revision of the patch, or alternatively, reply to this email with an explanation of why the patch should be accepted. If you believe these results are due to an error in patchtest, please submit a bug at https://bugzilla.yoctoproject.org/ (use the 'Patchtest' category under 'Yocto Project Subprojects'). For more information on specific failures, see: https://wiki.yoctoproject.org/wiki/Patchtest. Thank you!
Hey Joshua
https://errors.yoctoproject.org/Errors/Build/187139/
does this seem related to this changeset
Quoted 52 lines · Joshua Watt via lists.openembedded.org
This patch series add support for SPDX 3.0 and sets it as the default. Currently it is not possible to have SPDX 2.2 and SPDX 3.0 enabled at the same time; users may choose to specifically use SPDX 3.0 instead of 2.2 by doing INHERIT += "create-spdx-3.0"
v2: Added tests and addressed feedback
Joshua Watt (7):
classes-recipe/image: Add image file manifest classes/create-spdx-3.0: Add class classes-recipe/image_types: AddSPDX_IMAGE_PURPOSEto images
selftest: spdx: Add SPDX 3.0 test cases
classes-recipe: nospdx: Add class classes/spdx-common: MoveSPDX_SUPPLIER
Switch default spdx version to 3.0meta/classes-recipe/image.bbclass | 58 + meta/classes-recipe/image_types.bbclass | 2 + meta/classes-recipe/image_types_wic.bbclass | 1 + meta/classes-recipe/nospdx.bbclass | 13 + meta/classes-recipe/packagegroup.bbclass | 2 + meta/classes/create-spdx-2.2.bbclass | 8 + meta/classes/create-spdx-3.0.bbclass | 1116 ++++ meta/classes/create-spdx-image-3.0.bbclass | 203 + meta/classes/create-spdx.bbclass | 2 +- meta/classes/spdx-common.bbclass | 32 +- meta/lib/oe/sbom30.py | 1118 ++++ meta/lib/oe/spdx30.py | 5996 ++++++++++++++++++ meta/lib/oeqa/selftest/cases/spdx.py | 119 +- meta/recipes-core/meta/build-sysroots.bb | 5 +- meta/recipes-core/meta/meta-world-pkgdata.bb | 3 +- 15 files changed, 8654 insertions(+), 24 deletions(-) create mode 100644 meta/classes-recipe/nospdx.bbclass create mode 100644 meta/classes/create-spdx-3.0.bbclass create mode 100644 meta/classes/create-spdx-image-3.0.bbclass create mode 100644 meta/lib/oe/sbom30.py create mode 100644 meta/lib/oe/spdx30.py--
2.43.2-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#200929): https://lists.openembedded.org/g/openembedded-core/message/200929
Mute This Topic: https://lists.openembedded.org/mt/106770352/1997914
Group Owner: [email protected]
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [[email protected]]
-=-=-=-=-=-=-=-=-=-=-=-
Unfortunately, this still causes failures:
https://autobuilder.yoctoproject.org/typhoon/#/builders/83/builds/7060/steps/23/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/23/builds/9481/steps/14/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/20/builds/9574/steps/11/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/89/builds/9149/steps/12/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/82/builds/6555/steps/12/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/117/builds/4969/steps/12/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/65/builds/9220/steps/11/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/166/builds/518/steps/11/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/165/builds/514/steps/11/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/48/builds/9186/steps/12/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/37/builds/9166/steps/11/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/104/builds/8044/steps/11/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/103/builds/8038/steps/11/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/100/builds/6112/steps/12/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/64/builds/9159/steps/12/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/40/builds/9145/steps/11/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/81/builds/6733/steps/12/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/52/builds/8994/steps/12/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/108/builds/6122/steps/12/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/108/builds/6122/steps/12/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/58/builds/8923/steps/15/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/80/builds/6837/steps/14/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/79/builds/6881/steps/14/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/86/builds/6906/steps/14/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/87/builds/6892/steps/14/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/127/builds/3501/steps/15/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/76/builds/9099/steps/23/logs/stdio
Quoted 52 lines · Joshua Watt
This patch series add support for SPDX 3.0 and sets it as the default. Currently it is not possible to have SPDX 2.2 and SPDX 3.0 enabled at the same time; users may choose to specifically use SPDX 3.0 instead of 2.2 by doing INHERIT += "create-spdx-3.0"
v2: Added tests and addressed feedback
Joshua Watt (7):
classes-recipe/image: Add image file manifest classes/create-spdx-3.0: Add class classes-recipe/image_types: AddSPDX_IMAGE_PURPOSEto images
selftest: spdx: Add SPDX 3.0 test cases
classes-recipe: nospdx: Add class classes/spdx-common: MoveSPDX_SUPPLIER
Switch default spdx version to 3.0meta/classes-recipe/image.bbclass | 58 + meta/classes-recipe/image_types.bbclass | 2 + meta/classes-recipe/image_types_wic.bbclass | 1 + meta/classes-recipe/nospdx.bbclass | 13 + meta/classes-recipe/packagegroup.bbclass | 2 + meta/classes/create-spdx-2.2.bbclass | 8 + meta/classes/create-spdx-3.0.bbclass | 1116 ++++ meta/classes/create-spdx-image-3.0.bbclass | 203 + meta/classes/create-spdx.bbclass | 2 +- meta/classes/spdx-common.bbclass | 32 +- meta/lib/oe/sbom30.py | 1118 ++++ meta/lib/oe/spdx30.py | 5996 ++++++++++++++++++ meta/lib/oeqa/selftest/cases/spdx.py | 119 +- meta/recipes-core/meta/build-sysroots.bb | 5 +- meta/recipes-core/meta/meta-world-pkgdata.bb | 3 +- 15 files changed, 8654 insertions(+), 24 deletions(-) create mode 100644 meta/classes-recipe/nospdx.bbclass create mode 100644 meta/classes/create-spdx-3.0.bbclass create mode 100644 m: failures:eta/classes/create-spdx-image-3.0.bbclass create mode 100644 meta/lib/oe/sbom30.py create mode 100644 meta/lib/oe/spdx30.pySignature · 11 lines
2.43.2 -=-=-=-=-=-=-=-=-=-=-=- Links: You receive all messages sent to this group. View/Reply Online (#200929): https://lists.openembedded.org/g/openembedded-core/message/200929 Mute This Topic: https://lists.openembedded.org/mt/106770352/3617179 Group Owner: [email protected] Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [[email protected]] -=-=-=-=-=-=-=-=-=-=-=-
Signature · 3 lines
Alexandre Belloni, co-owner and COO, Bootlin Embedded Linux and Kernel engineering https://bootlin.com
Quoted 2 lines · Alexandre Belloni
Unfortunately, this still causes failures:
Thanks, I'll take a look
Quoted 93 lines
https://autobuilder.yoctoproject.org/typhoon/#/builders/83/builds/7060/steps/23/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/23/builds/9481/steps/14/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/20/builds/9574/steps/11/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/89/builds/9149/steps/12/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/82/builds/6555/steps/12/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/117/builds/4969/steps/12/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/65/builds/9220/steps/11/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/166/builds/518/steps/11/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/165/builds/514/steps/11/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/48/builds/9186/steps/12/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/37/builds/9166/steps/11/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/104/builds/8044/steps/11/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/103/builds/8038/steps/11/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/100/builds/6112/steps/12/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/64/builds/9159/steps/12/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/40/builds/9145/steps/11/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/81/builds/6733/steps/12/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/52/builds/8994/steps/12/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/108/builds/6122/steps/12/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/108/builds/6122/steps/12/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/58/builds/8923/steps/15/logs/stdiohttps://autobuilder.yoctoproject.org/typhoon/#/builders/80/builds/6837/steps/14/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/79/builds/6881/steps/14/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/86/builds/6906/steps/14/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/87/builds/6892/steps/14/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/127/builds/3501/steps/15/logs/stdiohttps://autobuilder.yoctoproject.org/typhoon/#/builders/76/builds/9099/steps/23/logs/stdio
Quoted 44 lines · Joshua Watt
This patch series add support for SPDX 3.0 and sets it as the default. Currently it is not possible to have SPDX 2.2 and SPDX 3.0 enabled at the same time; users may choose to specifically use SPDX 3.0 instead of 2.2 by doing INHERIT += "create-spdx-3.0"
v2: Added tests and addressed feedback
Joshua Watt (7):
classes-recipe/image: Add image file manifest classes/create-spdx-3.0: Add class classes-recipe/image_types: AddSPDX_IMAGE_PURPOSEto images
selftest: spdx: Add SPDX 3.0 test cases
classes-recipe: nospdx: Add class classes/spdx-common: MoveSPDX_SUPPLIER
Switch default spdx version to 3.0meta/classes-recipe/image.bbclass | 58 + meta/classes-recipe/image_types.bbclass | 2 + meta/classes-recipe/image_types_wic.bbclass | 1 + meta/classes-recipe/nospdx.bbclass | 13 + meta/classes-recipe/packagegroup.bbclass | 2 + meta/classes/create-spdx-2.2.bbclass | 8 + meta/classes/create-spdx-3.0.bbclass | 1116 ++++ meta/classes/create-spdx-image-3.0.bbclass | 203 + meta/classes/create-spdx.bbclass | 2 +- meta/classes/spdx-common.bbclass | 32 +- meta/lib/oe/sbom30.py | 1118 ++++ meta/lib/oe/spdx30.py | 5996 ++++++++++++++++++ meta/lib/oeqa/selftest/cases/spdx.py | 119 +- meta/recipes-core/meta/build-sysroots.bb | 5 +- meta/recipes-core/meta/meta-world-pkgdata.bb | 3 +- 15 files changed, 8654 insertions(+), 24 deletions(-) create mode 100644 meta/classes-recipe/nospdx.bbclass create mode 100644 meta/classes/create-spdx-3.0.bbclass create mode 100644 m: failures:eta/classes/create-spdx-image-3.0.bbclass create mode 100644 meta/lib/oe/sbom30.py create mode 100644 meta/lib/oe/spdx30.py--
2.43.2Quoted 8 lines
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#200929): https://lists.openembedded.org/g/openembedded-core/message/200929
Mute This Topic: https://lists.openembedded.org/mt/106770352/3617179
Group Owner: [email protected]
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [[email protected]]
-=-=-=-=-=-=-=-=-=-=-=---
Alexandre Belloni, co-owner and COO, Bootlin
Embedded Linux and Kernel engineering
https://bootlin.com
This jogged my memory on why I added the patch to allow tasks to opt out of running extend_recipe_sysroot. I'm sending this to get it written down before I forget, we can talk about it in the tech meeting on Tuesday if necessary. There are two build failure modes that having do_create_spdx run extend_recipe_sysroot causes.
The first failure mode is when a recipe explicitly does do_populate_sysroot[noexec] = "1" (an example here is glib-y2038-tests). In this case, when do_create_spdx tries to run extend_recipe_sysroot, it sees that the task depends on do_populate_sysroot, and this looks for the glibc-y2038-tests:populate_sysroot manifest file, which doesn't exist because the task was not run which results in a failure. Example here:
https://autobuilder.yoctoproject.org/typhoon/#/builders/81/builds/6733/steps/12/logs/stdio
The second failure mode is (I think) a bug in binutils-cross-testsuite. This recipe is odd because it doesn't actually inherit cross.bbclass, and thus isn't _actually_ a cross recipe, despite the name. This confuses find_sstate_manifest(), which assumes that if the recipe has "-cross-" in the name, the package arch must be ${BUILD_ARCH}, which is not the case here because it doesn't inherit cross.bbclass. When do_create_spdx tries to run extend_recipe_sysroot, it is looking for the binutils-cross-testsuite:populate_sysroot manifest file using the wrong architecture, and thus errors out. I _think_ this might be fixed by renaming the recipe so that it no longer contains the string "-cross-". Example here:
https://errors.yoctoproject.org/Errors/Details/787278/
do_create_spdx doesn't actually need the sysroot extended; it only actually depends on do_populate_sysroot do that it can inspect the sysroot produced by the current recipe to add the files to the SPDX output. IIUC, the automatic call to extend_recipe_sysroot when a task depends on do_populate_ssyroot is mostly a heuristic designed to ensure that tasks get the full recipe sysroot correctly without having to manually invoke the call everywhere. However, in the case of do_create_spdx() it is actually not necessary, thus I added the flag to disable that behavior on a per-task basis, since I thought that might be better than hard coding an exception for do_create_spdx.
The other solution would be to identify each individual recipe that is doing something weird and disable SPDX generation in that specific recipe (I've made a new class called nospdx.bbclass that makes this easy). I'm not sure how many recipes this would be, probably only a few, but it might be confusing to other layers why failures occur and what to do about it.... and again since the heuristic for when to call extend_recipe_sysroot is incorrect for do_create_spdx, I'd prefer to explore that route.
Quoted 95 lines · Alexandre Belloni
Unfortunately, this still causes failures:
https://autobuilder.yoctoproject.org/typhoon/#/builders/83/builds/7060/steps/23/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/23/builds/9481/steps/14/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/20/builds/9574/steps/11/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/89/builds/9149/steps/12/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/82/builds/6555/steps/12/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/117/builds/4969/steps/12/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/65/builds/9220/steps/11/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/166/builds/518/steps/11/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/165/builds/514/steps/11/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/48/builds/9186/steps/12/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/37/builds/9166/steps/11/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/104/builds/8044/steps/11/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/103/builds/8038/steps/11/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/100/builds/6112/steps/12/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/64/builds/9159/steps/12/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/40/builds/9145/steps/11/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/81/builds/6733/steps/12/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/52/builds/8994/steps/12/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/108/builds/6122/steps/12/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/108/builds/6122/steps/12/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/58/builds/8923/steps/15/logs/stdiohttps://autobuilder.yoctoproject.org/typhoon/#/builders/80/builds/6837/steps/14/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/79/builds/6881/steps/14/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/86/builds/6906/steps/14/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/87/builds/6892/steps/14/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/127/builds/3501/steps/15/logs/stdiohttps://autobuilder.yoctoproject.org/typhoon/#/builders/76/builds/9099/steps/23/logs/stdio
Quoted 44 lines · Joshua Watt
This patch series add support for SPDX 3.0 and sets it as the default. Currently it is not possible to have SPDX 2.2 and SPDX 3.0 enabled at the same time; users may choose to specifically use SPDX 3.0 instead of 2.2 by doing INHERIT += "create-spdx-3.0"
v2: Added tests and addressed feedback
Joshua Watt (7):
classes-recipe/image: Add image file manifest classes/create-spdx-3.0: Add class classes-recipe/image_types: AddSPDX_IMAGE_PURPOSEto images
selftest: spdx: Add SPDX 3.0 test cases
classes-recipe: nospdx: Add class classes/spdx-common: MoveSPDX_SUPPLIER
Switch default spdx version to 3.0meta/classes-recipe/image.bbclass | 58 + meta/classes-recipe/image_types.bbclass | 2 + meta/classes-recipe/image_types_wic.bbclass | 1 + meta/classes-recipe/nospdx.bbclass | 13 + meta/classes-recipe/packagegroup.bbclass | 2 + meta/classes/create-spdx-2.2.bbclass | 8 + meta/classes/create-spdx-3.0.bbclass | 1116 ++++ meta/classes/create-spdx-image-3.0.bbclass | 203 + meta/classes/create-spdx.bbclass | 2 +- meta/classes/spdx-common.bbclass | 32 +- meta/lib/oe/sbom30.py | 1118 ++++ meta/lib/oe/spdx30.py | 5996 ++++++++++++++++++ meta/lib/oeqa/selftest/cases/spdx.py | 119 +- meta/recipes-core/meta/build-sysroots.bb | 5 +- meta/recipes-core/meta/meta-world-pkgdata.bb | 3 +- 15 files changed, 8654 insertions(+), 24 deletions(-) create mode 100644 meta/classes-recipe/nospdx.bbclass create mode 100644 meta/classes/create-spdx-3.0.bbclass create mode 100644 m: failures:eta/classes/create-spdx-image-3.0.bbclass create mode 100644 meta/lib/oe/sbom30.py create mode 100644 meta/lib/oe/spdx30.py--
2.43.2Quoted 8 lines
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#200929): https://lists.openembedded.org/g/openembedded-core/message/200929
Mute This Topic: https://lists.openembedded.org/mt/106770352/3617179
Group Owner: [email protected]
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [[email protected]]
-=-=-=-=-=-=-=-=-=-=-=---
Alexandre Belloni, co-owner and COO, Bootlin
Embedded Linux and Kernel engineering
https://bootlin.com
v3 cover Add SPDX 3.0 support
In reply to Joshua WattThis patch series add support for SPDX 3.0 and sets it as the default. Currently it is not possible to have SPDX 2.2 and SPDX 3.0 enabled at the same time
v2: Added tests and addressed feedback
v3: Fixed several oe-selftest and build failures
Joshua Watt (10):
classes-recipe/image: Add image file manifest classes-recipe/baremetal-image: Add image file manifest classes/create-spdx-3.0: Add classes classes-global/staging: Exclude do_create_spdx from automatic sysroot
extension
binutils-cross-testsuite: Rename to binutils-testsuite classes-recipe/image_types: Add SPDX_IMAGE_PURPOSE to images
selftest: spdx: Add SPDX 3.0 test cases
classes-recipe: nospdx: Add class
classes/spdx-common: Move SPDX_SUPPLIER
Switch default spdx version to 3.0
meta/classes-global/mirrors.bbclass | 2 +-
meta/classes-global/staging.bbclass | 6 +-
meta/classes-recipe/baremetal-image.bbclass | 30 +-
meta/classes-recipe/image.bbclass | 58 +
meta/classes-recipe/image_types.bbclass | 2 +
meta/classes-recipe/image_types_wic.bbclass | 1 +
meta/classes-recipe/nospdx.bbclass | 13 +
meta/classes-recipe/packagegroup.bbclass | 2 +
meta/classes/create-spdx-2.2.bbclass | 8 +
meta/classes/create-spdx-3.0.bbclass | 1179 ++++
meta/classes/create-spdx-image-3.0.bbclass | 224 +
meta/classes/create-spdx.bbclass | 2 +-
meta/classes/spdx-common.bbclass | 13 +-
meta/conf/distro/include/maintainers.inc | 2 +-
meta/lib/oe/sbom30.py | 1122 +++
meta/lib/oe/spdx30.py | 5996 +++++++++++++++++
meta/lib/oeqa/selftest/cases/binutils.py | 2 +-
meta/lib/oeqa/selftest/cases/spdx.py | 119 +-
meta/recipes-core/meta/build-sysroots.bb | 5 +-
meta/recipes-core/meta/meta-world-pkgdata.bb | 3 +-
...ite_2.42.bb => binutils-testsuite_2.42.bb} | 3 +
21 files changed, 8762 insertions(+), 30 deletions(-)
create mode 100644 meta/classes-recipe/nospdx.bbclass
create mode 100644 meta/classes/create-spdx-3.0.bbclass
create mode 100644 meta/classes/create-spdx-image-3.0.bbclass
create mode 100644 meta/lib/oe/sbom30.py
create mode 100644 meta/lib/oe/spdx30.py
rename meta/recipes-devtools/binutils/{binutils-cross-testsuite_2.42.bb => binutils-testsuite_2.42.bb} (95%)Signature · 1 line
2.43.2
PATCH v3 1/10 classes-recipe/image: Add image file manifest
Downstream tasks may want to know what image files were written by the do_image family of tasks (e.g. SPDX) so have each task write out a manifest file that describes the files it produced, then aggregate them in do_image_complete
--- meta/classes-recipe/image.bbclass | 58 +++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+)
meta/classes-recipe/image.bbclass +58 -0
@@ -88,6 +88,11 @@ PACKAGE_INSTALL_ATTEMPTONLY ?= "${FEATURE_INSTALL_OPTIONAL}" IMGDEPLOYDIR = "${WORKDIR}/deploy-${PN}-image-complete" +IMGMANIFESTDIR = "${WORKDIR}/image-task-manifest" + +IMAGE_FILE_MANIFEST_DIR = "${WORKDIR}/deploy-image-file-manifest" +IMAGE_FILE_MANIFEST = "${IMAGE_FILE_MANIFEST_DIR}/manifest.json" + # Images are generally built explicitly, do not need to be part of world. EXCLUDE_FROM_WORLD = "1"
@@ -277,14 +282,28 @@ fakeroot python do_image () { execute_pre_post_process(d, pre_process_cmds) } do_image[dirs] = "${TOPDIR}" +do_image[cleandirs] += "${IMGMANIFESTDIR}" addtask do_image after do_rootfs fakeroot python do_image_complete () { from oe.utils import execute_pre_post_process + from pathlib import Path + import json post_process_cmds = d.getVar("IMAGE_POSTPROCESS_COMMAND") execute_pre_post_process(d, post_process_cmds) + + image_manifest_dir = Path(d.getVar('IMGMANIFESTDIR')) + + data = [] + + for manifest_path in image_manifest_dir.glob("*.json"): + with manifest_path.open("r") as f: + data.extend(json.load(f)) + + with open(d.getVar("IMAGE_FILE_MANIFEST"), "w") as f: + json.dump(data, f) } do_image_complete[dirs] = "${TOPDIR}" SSTATETASKS += "do_image_complete"
@@ -292,6 +311,8 @@ SSTATE_SKIP_CREATION:task-image-complete = '1' do_image_complete[sstate-inputdirs] = "${IMGDEPLOYDIR}" do_image_complete[sstate-outputdirs] = "${DEPLOY_DIR_IMAGE}" do_image_complete[stamp-extra-info] = "${MACHINE_ARCH}" +do_image_complete[sstate-plaindirs] += "${IMAGE_FILE_MANIFEST_DIR}" +do_image_complete[dirs] += "${IMAGE_FILE_MANIFEST_DIR}" addtask do_image_complete after do_image before do_build python do_image_complete_setscene () { sstate_setscene(d)
@@ -507,12 +528,14 @@ python () { d.setVar(task, '\n'.join(cmds)) d.setVarFlag(task, 'func', '1') d.setVarFlag(task, 'fakeroot', '1') + d.setVarFlag(task, 'imagetype', t) d.appendVarFlag(task, 'prefuncs', ' ' + debug + ' set_image_size') d.prependVarFlag(task, 'postfuncs', 'create_symlinks ') d.appendVarFlag(task, 'subimages', ' ' + ' '.join(subimages)) d.appendVarFlag(task, 'vardeps', ' ' + ' '.join(vardeps)) d.appendVarFlag(task, 'vardepsexclude', ' DATETIME DATE ' + ' '.join(vardepsexclude)) + d.appendVarFlag(task, 'postfuncs', ' write_image_file_manifest') bb.debug(2, "Adding task %s before %s, after %s" % (task, 'do_image_complete', after)) bb.build.addtask(task, 'do_image_complete', after, d)
@@ -610,6 +633,41 @@ python create_symlinks() { bb.note("Skipping symlink, source does not exist: %s -> %s" % (dst, src)) } +python write_image_file_manifest() { + import json + from pathlib import Path + + taskname = d.getVar("BB_CURRENTTASK") + image_deploy_dir = Path(d.getVar('IMGDEPLOYDIR')) + image_manifest_dir = Path(d.getVar('IMGMANIFESTDIR')) + manifest_path = image_manifest_dir / ("do_" + d.getVar("BB_CURRENTTASK") + ".json") + + image_name = d.getVar("IMAGE_NAME") + image_basename = d.getVar("IMAGE_BASENAME") + machine = d.getVar("MACHINE") + + subimages = (d.getVarFlag("do_" + taskname, 'subimages', False) or "").split() + imagetype = d.getVarFlag("do_" + taskname, 'imagetype', False) + + data = { + "taskname": taskname, + "imagetype": imagetype, + "images": [] + } + + for type in subimages: + image_filename = image_name + "." + type + image_path = image_deploy_dir / image_filename + if not image_path.exists(): + continue + data["images"].append({ + "filename": image_filename, + }) + + with manifest_path.open("w") as f: + json.dump([data], f) +} + MULTILIBRE_ALLOW_REP += "${base_bindir} ${base_sbindir} ${bindir} ${sbindir} ${libexecdir} ${sysconfdir} ${nonarch_base_libdir}/udev /lib/modules/[^/]*/modules.*" MULTILIB_CHECK_FILE = "${WORKDIR}/multilib_check.py" MULTILIB_TEMP_ROOTFS = "${WORKDIR}/multilib"
Signature · 1 line
2.43.2
PATCH v3 2/10 classes-recipe/baremetal-image: Add image file manifest
In reply to Joshua WattDownstream tasks may want to know what image files were written so write out a manifest in do_image_complete. The format of the manifest is the same as the one in image.bbclass
--- meta/classes-recipe/baremetal-image.bbclass | 30 +++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-)
meta/classes-recipe/baremetal-image.bbclass +28 -2
@@ -30,6 +30,9 @@ BAREMETAL_BINNAME ?= "hello_baremetal_${MACHINE}" IMAGE_LINK_NAME ?= "baremetal-helloworld-image-${MACHINE}" IMAGE_NAME_SUFFIX ?= "" +IMAGE_FILE_MANIFEST_DIR = "${WORKDIR}/deploy-image-file-manifest" +IMAGE_FILE_MANIFEST = "${IMAGE_FILE_MANIFEST_DIR}/manifest.json" + do_rootfs[dirs] = "${IMGDEPLOYDIR} ${DEPLOY_DIR_IMAGE}" do_image(){
@@ -37,8 +40,28 @@ do_image(){ install ${D}/${base_libdir}/firmware/${BAREMETAL_BINNAME}.elf ${IMGDEPLOYDIR}/${IMAGE_LINK_NAME}.elf } -do_image_complete(){ - : +python do_image_complete(){ + from pathlib import Path + import json + + data = { + "taskname": "do_image", + "imagetype": "baremetal-image", + "images": [] + } + + img_deploy_dir = Path(d.getVar("IMGDEPLOYDIR")) + + for child in img_deploy_dir.iterdir(): + if not child.is_file() or child.is_symlink(): + continue + + data["images"].append({ + "filename": child.name, + }) + + with open(d.getVar("IMAGE_FILE_MANIFEST"), "w") as f: + json.dump([data], f) } python do_rootfs(){
@@ -62,6 +85,7 @@ python do_rootfs(){ bb.utils.mkdirhier(sysconfdir) execute_pre_post_process(d, d.getVar('ROOTFS_POSTPROCESS_COMMAND')) + execute_pre_post_process(d, d.getVar("ROOTFS_POSTUNINSTALL_COMMAND")) }
@@ -72,6 +96,8 @@ SSTATE_SKIP_CREATION:task-image-complete = '1' do_image_complete[sstate-inputdirs] = "${IMGDEPLOYDIR}" do_image_complete[sstate-outputdirs] = "${DEPLOY_DIR_IMAGE}" do_image_complete[stamp-extra-info] = "${MACHINE_ARCH}" +do_image_complete[sstate-plaindirs] += "${IMAGE_FILE_MANIFEST_DIR}" +do_image_complete[dirs] += "${IMAGE_FILE_MANIFEST_DIR}" addtask do_image_complete after do_image before do_build python do_image_complete_setscene () {
Signature · 1 line
2.43.2
PATCH v3 3/10 classes/create-spdx-3.0: Add classes
In reply to Joshua WattAdds a class to generate SPDX 3.0 output and an image class that is used when generating images
--- meta/classes-recipe/packagegroup.bbclass | 2 + meta/classes/create-spdx-3.0.bbclass | 1179 ++++ meta/classes/create-spdx-image-3.0.bbclass | 224 + meta/classes/spdx-common.bbclass | 6 +- meta/lib/oe/sbom30.py | 1122 ++++ meta/lib/oe/spdx30.py | 5996 ++++++++++++++++++++ 6 files changed, 8528 insertions(+), 1 deletion(-) create mode 100644 meta/classes/create-spdx-3.0.bbclass create mode 100644 meta/classes/create-spdx-image-3.0.bbclass create mode 100644 meta/lib/oe/sbom30.py create mode 100644 meta/lib/oe/spdx30.py
meta/classes-recipe/packagegroup.bbclass +2 -0
@@ -56,6 +56,8 @@ deltask do_populate_sysroot do_create_runtime_spdx[deptask] = "do_create_spdx" do_create_runtime_spdx[rdeptask] = "" +do_create_package_spdx[deptask] = "do_create_spdx" +do_create_package_spdx[rdeptask] = "" INHIBIT_DEFAULT_DEPS = "1"
meta/classes/create-spdx-3.0.bbclass new file +1179 -0
1,179 lines not on this page · raw message
meta/classes/create-spdx-image-3.0.bbclass new file +224 -0
224 lines not on this page · raw message
meta/classes/spdx-common.bbclass +5 -1
@@ -17,6 +17,7 @@ SPDXDEPLOY = "${SPDXDIR}/deploy" SPDXWORK = "${SPDXDIR}/work" SPDXIMAGEWORK = "${SPDXDIR}/image-work" SPDXSDKWORK = "${SPDXDIR}/sdk-work" +SPDXSDKEXTWORK = "${SPDXDIR}/sdk-ext-work" SPDXDEPS = "${SPDXDIR}/deps.json" SPDX_TOOL_NAME ??= "oe-spdx-creator"
@@ -68,7 +69,7 @@ def get_json_indent(d): return 2 return None -python() { +def load_spdx_license_data(d): import json if d.getVar("SPDX_LICENSE_DATA"): return
@@ -78,6 +79,9 @@ python() { # Transform the license array to a dictionary data["licenses"] = {l["licenseId"]: l for l in data["licenses"]} d.setVar("SPDX_LICENSE_DATA", data) + +python() { + load_spdx_license_data(d) } def process_sources(d):
meta/lib/oe/sbom30.py new file +1122 -0
1,122 lines not on this page · raw message
meta/lib/oe/spdx30.py new file +5996 -0
443 lines not on this page · raw message
The rest of this diff is not shown. View the raw message.
Signature · 1 line
2.43.2
Part of the diff is not shown here. View the raw message.
PATCH v3 4/10 classes-global/staging: Exclude do_create_spdx from automatic sysroot extension
In reply to Joshua Wattdo_create_spdx is a outlier in that it doesn't need the RSS to be extended just because it depends on do_populate_sysroot. In fact, it only depends on do_populate_sysroot so it can see the actual recipes sysroot, and attempting to extend the sysroot can cause problems for some recipes (e.g. if a recipe does do_populate_sysroot[noexec] = "1")
As such, explicitly exclude do_create_spdx from extending the sysroot just because it depends on do_populate_sysroot.
--- meta/classes-global/staging.bbclass | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-)
meta/classes-global/staging.bbclass +5 -1
@@ -652,10 +652,14 @@ python do_prepare_recipe_sysroot () { addtask do_prepare_recipe_sysroot before do_configure after do_fetch python staging_taskhandler() { + EXCLUDED_TASK = ( + "do_prepare_recipe_sysroot", + "do_create_spdx", + ) bbtasks = e.tasklist for task in bbtasks: deps = d.getVarFlag(task, "depends") - if task != 'do_prepare_recipe_sysroot' and (task == "do_configure" or (deps and "populate_sysroot" in deps)): + if task not in EXCLUDED_TASKS and (task == "do_configure" or (deps and "populate_sysroot" in deps)): d.prependVarFlag(task, "prefuncs", "extend_recipe_sysroot ") } staging_taskhandler[eventmask] = "bb.event.RecipeTaskPreProcess"
Signature · 1 line
2.43.2
PATCH v3 5/10 binutils-cross-testsuite: Rename to binutils-testsuite
In reply to Joshua WattThis recipe needs to be renamed because the "-cross-" substring in the name triggers the cross architecture detection in sstate, but this recipe is not actually a cross recipe.
---
meta/classes-global/mirrors.bbclass | 2 +-
meta/conf/distro/include/maintainers.inc | 2 +-
meta/lib/oeqa/selftest/cases/binutils.py | 2 +-
...tils-cross-testsuite_2.42.bb => binutils-testsuite_2.42.bb} | 3 +++
4 files changed, 6 insertions(+), 3 deletions(-)
rename meta/recipes-devtools/binutils/{binutils-cross-testsuite_2.42.bb => binutils-testsuite_2.42.bb} (95%)meta/classes-global/mirrors.bbclass +1 -1
@@ -88,7 +88,7 @@ git://.*/.* git://HOST/git/PATH;protocol=https \ BB_GIT_SHALLOW:pn-binutils = "1" BB_GIT_SHALLOW:pn-binutils-cross-${TARGET_ARCH} = "1" BB_GIT_SHALLOW:pn-binutils-cross-canadian-${TRANSLATED_TARGET_ARCH} = "1" -BB_GIT_SHALLOW:pn-binutils-cross-testsuite = "1" +BB_GIT_SHALLOW:pn-binutils-testsuite = "1" BB_GIT_SHALLOW:pn-binutils-crosssdk-${SDK_SYS} = "1" BB_GIT_SHALLOW:pn-binutils-native = "1" BB_GIT_SHALLOW:pn-nativesdk-binutils = "1"
meta/conf/distro/include/maintainers.inc +1 -1
@@ -64,7 +64,7 @@ RECIPE_MAINTAINER:pn-bind = "Unassigned <[email protected]>" RECIPE_MAINTAINER:pn-binutils = "Khem Raj <[email protected]>" RECIPE_MAINTAINER:pn-binutils-cross-${TARGET_ARCH} = "Khem Raj <[email protected]>" RECIPE_MAINTAINER:pn-binutils-cross-canadian-${TRANSLATED_TARGET_ARCH} = "Khem Raj <[email protected]>" -RECIPE_MAINTAINER:pn-binutils-cross-testsuite = "Khem Raj <[email protected]>" +RECIPE_MAINTAINER:pn-binutils-testsuite = "Khem Raj <[email protected]>" RECIPE_MAINTAINER:pn-binutils-crosssdk-${SDK_SYS} = "Khem Raj <[email protected]>" RECIPE_MAINTAINER:pn-bison = "Chen Qi <[email protected]>" RECIPE_MAINTAINER:pn-blktool = "Yi Zhao <[email protected]>"
meta/lib/oeqa/selftest/cases/binutils.py +1 -1
@@ -33,7 +33,7 @@ class BinutilsCrossSelfTest(OESelftestTestCase, OEPTestResultTestCase): features.append('CHECK_TARGETS = "{0}"'.format(suite)) self.write_config("\n".join(features)) - recipe = "binutils-cross-testsuite" + recipe = "binutils-testsuite" bb_vars = get_bb_vars(["B", "TARGET_SYS", "T"], recipe) builddir, target_sys, tdir = bb_vars["B"], bb_vars["TARGET_SYS"], bb_vars["T"]
meta/recipes-devtools/binutils/binutils-testsuite_2.42.bb renamed from meta/recipes-devtools/binutils/binutils-cross-testsuite_2.42.bb +3 -0
@@ -1,3 +1,6 @@ +# NOTE: This recipe cannot have -cross- in the file name because it triggers +# the cross build detection in sstate which causes it to use the wrong +# architecture require binutils.inc require binutils-${PV}.inc
Signature · 1 line
2.43.2
PATCH v3 6/10 classes-recipe/image_types: Add SPDX_IMAGE_PURPOSE to images
In reply to Joshua WattAdds the variable overrides to set the SPDX image purpose for various image types
--- meta/classes-recipe/image_types.bbclass | 2 ++ meta/classes-recipe/image_types_wic.bbclass | 1 + 2 files changed, 3 insertions(+)
meta/classes-recipe/image_types.bbclass +2 -0
@@ -146,6 +146,7 @@ IMAGE_CMD:vfat = "oe_mkvfatfs ${EXTRA_IMAGECMD}" IMAGE_CMD_TAR ?= "tar" # ignore return code 1 "file changed as we read it" as other tasks(e.g. do_image_wic) may be hardlinking rootfs IMAGE_CMD:tar = "${IMAGE_CMD_TAR} --sort=name --format=posix --numeric-owner -cf ${IMGDEPLOYDIR}/${IMAGE_NAME}.tar -C ${IMAGE_ROOTFS} . || [ $? -eq 1 ]" +SPDX_IMAGE_PURPOSE:tar = "archive" do_image_cpio[cleandirs] += "${WORKDIR}/cpio_append" IMAGE_CMD:cpio () {
@@ -167,6 +168,7 @@ IMAGE_CMD:cpio () { fi fi } +SPDX_IMAGE_PURPOSE:cpio = "archive" UBI_VOLNAME ?= "${MACHINE}-rootfs" UBI_VOLTYPE ?= "dynamic"
meta/classes-recipe/image_types_wic.bbclass +1 -0
@@ -91,6 +91,7 @@ IMAGE_CMD:wic () { mv "$build_wic/$(basename "${wks%.wks}")"*.${IMAGER} "$out.wic" } IMAGE_CMD:wic[vardepsexclude] = "WKS_FULL_PATH WKS_FILES TOPDIR" +SPDX_IMAGE_PURPOSE:wic = "diskImage" do_image_wic[cleandirs] = "${WORKDIR}/build-wic" PSEUDO_IGNORE_PATHS .= ",${WORKDIR}/build-wic"
Signature · 1 line
2.43.2
PATCH v3 7/10 selftest: spdx: Add SPDX 3.0 test cases
In reply to Joshua WattAdds test cases for SPDX 3.0. Reworks the SPDX 2.2 test setup so it can also be run even if the default is SPDX 3.0
--- meta/lib/oeqa/selftest/cases/spdx.py | 119 +++++++++++++++++++++++++-- 1 file changed, 110 insertions(+), 9 deletions(-)
meta/lib/oeqa/selftest/cases/spdx.py +110 -9
@@ -6,21 +6,26 @@ import json import os +import textwrap +from pathlib import Path from oeqa.selftest.case import OESelftestTestCase -from oeqa.utils.commands import bitbake, get_bb_var, runCmd +from oeqa.utils.commands import bitbake, get_bb_var, get_bb_vars, runCmd -class SPDXCheck(OESelftestTestCase): +class SPDX22Check(OESelftestTestCase): @classmethod def setUpClass(cls): - super(SPDXCheck, cls).setUpClass() + super().setUpClass() bitbake("python3-spdx-tools-native") bitbake("-c addto_recipe_sysroot python3-spdx-tools-native") def check_recipe_spdx(self, high_level_dir, spdx_file, target_name): - config = """ -INHERIT += "create-spdx" -""" + config = textwrap.dedent( + """\ + INHERIT:remove = "create-spdx" + INHERIT += "create-spdx-2.2" + """ + ) self.write_config(config) deploy_dir = get_bb_var("DEPLOY_DIR")
@@ -29,7 +34,9 @@ INHERIT += "create-spdx" # qemux86-64 creates the directory qemux86_64 machine_dir = machine_var.replace("-", "_") - full_file_path = os.path.join(deploy_dir, "spdx", spdx_version, machine_dir, high_level_dir, spdx_file) + full_file_path = os.path.join( + deploy_dir, "spdx", spdx_version, machine_dir, high_level_dir, spdx_file + ) try: os.remove(full_file_path)
@@ -44,8 +51,13 @@ INHERIT += "create-spdx" self.assertNotEqual(report, None) self.assertNotEqual(report["SPDXID"], None) - python = os.path.join(get_bb_var('STAGING_BINDIR', 'python3-spdx-tools-native'), 'nativepython3') - validator = os.path.join(get_bb_var('STAGING_BINDIR', 'python3-spdx-tools-native'), 'pyspdxtools') + python = os.path.join( + get_bb_var("STAGING_BINDIR", "python3-spdx-tools-native"), + "nativepython3", + ) + validator = os.path.join( + get_bb_var("STAGING_BINDIR", "python3-spdx-tools-native"), "pyspdxtools" + ) result = runCmd("{} {} -i {}".format(python, validator, filename)) self.assertExists(full_file_path)
@@ -53,3 +65,92 @@ INHERIT += "create-spdx" def test_spdx_base_files(self): self.check_recipe_spdx("packages", "base-files.spdx.json", "base-files") + + +class SPDX3CheckBase(object): + """ + Base class for checking SPDX 3 based tests + """ + + def check_spdx_file(self, filename): + import oe.spdx30 + + self.assertExists(filename) + + # Read the file + objset = oe.spdx30.SHACLObjectSet() + with open(filename, "r") as f: + d = oe.spdx30.JSONLDDeserializer() + d.read(f, objset) + + return objset + + def check_recipe_spdx(self, target_name, spdx_path, *, task=None): + config = textwrap.dedent( + f"""\ + INHERIT:remove = "create-spdx" + INHERIT += "{self.SPDX_CLASS}" + """ + ) + self.write_config(config) + + if task: + bitbake(f"-c {task} {target_name}") + else: + bitbake(target_name) + + filename = spdx_path.format( + **get_bb_vars( + [ + "DEPLOY_DIR_IMAGE", + "DEPLOY_DIR_SPDX", + "MACHINE", + "MACHINE_ARCH", + "SDKMACHINE", + "SDK_DEPLOY", + "SPDX_VERSION", + "TOOLCHAIN_OUTPUTNAME", + ], + target_name, + ) + ) + + return self.check_spdx_file(filename) + + def check_objset_missing_ids(self, objset): + if objset.missing_ids: + self.assertTrue( + False, + "The following SPDXIDs are unresolved:\n " + + "\n ".join(objset.missing_ids), + ) + + +class SPDX30Check(SPDX3CheckBase, OESelftestTestCase): + SPDX_CLASS = "create-spdx-3.0" + SCHEMA_URL = "https://spdx.org/schema/3.0.0/spdx-json-schema.json" + + def test_base_files(self): + self.check_recipe_spdx( + "base-files", + "{DEPLOY_DIR_SPDX}/{MACHINE_ARCH}/packages/base-files.spdx.json", + ) + + def test_core_image_minimal(self): + objset = self.check_recipe_spdx( + "core-image-minimal", + "{DEPLOY_DIR_IMAGE}/core-image-minimal-{MACHINE}.rootfs.spdx.json", + ) + + # Document should be fully linked + self.check_objset_missing_ids(objset) + + def test_core_image_minimal_sdk(self): + objset = self.check_recipe_spdx( + "core-image-minimal", + "{SDK_DEPLOY}/{TOOLCHAIN_OUTPUTNAME}.spdx.json", + task="populate_sdk", + ) + + # Document should be fully linked + self.check_objset_missing_ids(objset)
Signature · 1 line
2.43.2
PATCH v3 8/10 classes-recipe: nospdx: Add class
In reply to Joshua WattAdds a class that allows recipes to opt out of generating SPDX
--- meta/classes-recipe/nospdx.bbclass | 13 +++++++++++++ meta/recipes-core/meta/build-sysroots.bb | 5 +---- meta/recipes-core/meta/meta-world-pkgdata.bb | 3 +-- 3 files changed, 15 insertions(+), 6 deletions(-) create mode 100644 meta/classes-recipe/nospdx.bbclass
meta/classes-recipe/nospdx.bbclass new file +13 -0
@@ -0,0 +1,13 @@ +# +# Copyright OpenEmbedded Contributors +# +# SPDX-License-Identifier: MIT +# + +deltask do_collect_spdx_deps +deltask do_create_spdx +deltask do_create_spdx_runtime +deltask do_create_package_spdx +deltask do_create_rootfs_spdx +deltask do_create_image_spdx +deltask do_create_image_sbom
meta/recipes-core/meta/build-sysroots.bb +1 -4
@@ -7,7 +7,7 @@ STANDALONE_SYSROOT_NATIVE = "${STAGING_DIR}/${BUILD_ARCH}" PACKAGE_ARCH = "${MACHINE_ARCH}" EXCLUDE_FROM_WORLD = "1" -inherit nopackages +inherit nopackages nospdx deltask fetch deltask unpack deltask patch
@@ -17,9 +17,6 @@ deltask configure deltask compile deltask install deltask populate_sysroot -deltask create_spdx -deltask collect_spdx_deps -deltask create_runtime_spdx deltask recipe_qa do_build_warn () {
meta/recipes-core/meta/meta-world-pkgdata.bb +1 -2
@@ -27,14 +27,13 @@ python do_collect_packagedata() { oe.copy_buildsystem.generate_locked_sigs(sigfile, d) } +inherit nospdx deltask do_fetch deltask do_unpack deltask do_patch deltask do_configure deltask do_compile deltask do_install -deltask do_create_spdx -deltask do_create_spdx_runtime do_prepare_recipe_sysroot[deptask] = ""
Signature · 1 line
2.43.2
PATCH v3 9/10 classes/spdx-common: Move SPDX_SUPPLIER
In reply to Joshua WattMove the SPDX_SUPPLIER variable to create-spdx-2.2 since it's format only has meaning in SPDX 2.2 (SPDX 3 uses SPDX_PACKAGE_SUPPLIER with a different format)
--- meta/classes/create-spdx-2.2.bbclass | 8 ++++++++ meta/classes/spdx-common.bbclass | 7 ------- 2 files changed, 8 insertions(+), 7 deletions(-)
meta/classes/create-spdx-2.2.bbclass +8 -0
@@ -8,6 +8,14 @@ inherit spdx-common SPDX_VERSION = "2.2" +SPDX_ORG ??= "OpenEmbedded ()" +SPDX_SUPPLIER ??= "Organization: ${SPDX_ORG}" +SPDX_SUPPLIER[doc] = "The SPDX PackageSupplier field for SPDX packages created from \ + this recipe. For SPDX documents create using this class during the build, this \ + is the contact information for the person or organization who is doing the \ + build." + + def get_namespace(d, name): import uuid namespace_uuid = uuid.uuid5(uuid.NAMESPACE_DNS, d.getVar("SPDX_UUID_NAMESPACE"))
meta/classes/spdx-common.bbclass +0 -7
@@ -37,13 +37,6 @@ SPDX_LICENSES ??= "${COREBASE}/meta/files/spdx-licenses.json" SPDX_CUSTOM_ANNOTATION_VARS ??= "" -SPDX_ORG ??= "OpenEmbedded ()" -SPDX_SUPPLIER ??= "Organization: ${SPDX_ORG}" -SPDX_SUPPLIER[doc] = "The SPDX PackageSupplier field for SPDX packages created from \ - this recipe. For SPDX documents create using this class during the build, this \ - is the contact information for the person or organization who is doing the \ - build." - def extract_licenses(filename): import re
Signature · 1 line
2.43.2
PATCH v3 10/10 Switch default spdx version to 3.0
In reply to Joshua WattChanges the default SPDX version to 3.0
--- meta/classes/create-spdx.bbclass | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
meta/classes/create-spdx.bbclass +1 -1
@@ -5,4 +5,4 @@ # # Include this class when you don't care what version of SPDX you get; it will # be updated to the latest stable version that is supported -inherit create-spdx-2.2 +inherit create-spdx-3.0
Signature · 1 line
2.43.2
v4 cover Add SPDX 3.0 support
In reply to Joshua WattThis patch series add support for SPDX 3.0 and sets it as the default. Currently it is not possible to have SPDX 2.2 and SPDX 3.0 enabled at the same time
v2: Added tests and addressed feedback
v3: Fixed several oe-selftest and build failures
v4: Fixed silly typo mistake in staging.bbclass
Joshua Watt (10):
classes-recipe/image: Add image file manifest classes-recipe/baremetal-image: Add image file manifest classes/create-spdx-3.0: Add classes classes-global/staging: Exclude do_create_spdx from automatic sysroot
extension
binutils-cross-testsuite: Rename to binutils-testsuite classes-recipe/image_types: Add SPDX_IMAGE_PURPOSE to images
selftest: spdx: Add SPDX 3.0 test cases
classes-recipe: nospdx: Add class
classes/spdx-common: Move SPDX_SUPPLIER
Switch default spdx version to 3.0
meta/classes-global/mirrors.bbclass | 2 +-
meta/classes-global/staging.bbclass | 9 +-
meta/classes-recipe/baremetal-image.bbclass | 30 +-
meta/classes-recipe/image.bbclass | 58 +
meta/classes-recipe/image_types.bbclass | 2 +
meta/classes-recipe/image_types_wic.bbclass | 1 +
meta/classes-recipe/nospdx.bbclass | 13 +
meta/classes-recipe/packagegroup.bbclass | 2 +
meta/classes/create-spdx-2.2.bbclass | 8 +
meta/classes/create-spdx-3.0.bbclass | 1179 ++++
meta/classes/create-spdx-image-3.0.bbclass | 224 +
meta/classes/create-spdx.bbclass | 2 +-
meta/classes/spdx-common.bbclass | 13 +-
meta/conf/distro/include/maintainers.inc | 2 +-
meta/lib/oe/sbom30.py | 1122 +++
meta/lib/oe/spdx30.py | 5996 +++++++++++++++++
meta/lib/oeqa/selftest/cases/binutils.py | 2 +-
meta/lib/oeqa/selftest/cases/spdx.py | 119 +-
meta/recipes-core/meta/build-sysroots.bb | 5 +-
meta/recipes-core/meta/meta-world-pkgdata.bb | 3 +-
...ite_2.42.bb => binutils-testsuite_2.42.bb} | 3 +
21 files changed, 8765 insertions(+), 30 deletions(-)
create mode 100644 meta/classes-recipe/nospdx.bbclass
create mode 100644 meta/classes/create-spdx-3.0.bbclass
create mode 100644 meta/classes/create-spdx-image-3.0.bbclass
create mode 100644 meta/lib/oe/sbom30.py
create mode 100644 meta/lib/oe/spdx30.py
rename meta/recipes-devtools/binutils/{binutils-cross-testsuite_2.42.bb => binutils-testsuite_2.42.bb} (95%)Signature · 1 line
2.43.2
PATCH v4 1/10 classes-recipe/image: Add image file manifest
Downstream tasks may want to know what image files were written by the do_image family of tasks (e.g. SPDX) so have each task write out a manifest file that describes the files it produced, then aggregate them in do_image_complete
--- meta/classes-recipe/image.bbclass | 58 +++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+)
meta/classes-recipe/image.bbclass +58 -0
@@ -88,6 +88,11 @@ PACKAGE_INSTALL_ATTEMPTONLY ?= "${FEATURE_INSTALL_OPTIONAL}" IMGDEPLOYDIR = "${WORKDIR}/deploy-${PN}-image-complete" +IMGMANIFESTDIR = "${WORKDIR}/image-task-manifest" + +IMAGE_FILE_MANIFEST_DIR = "${WORKDIR}/deploy-image-file-manifest" +IMAGE_FILE_MANIFEST = "${IMAGE_FILE_MANIFEST_DIR}/manifest.json" + # Images are generally built explicitly, do not need to be part of world. EXCLUDE_FROM_WORLD = "1"
@@ -277,14 +282,28 @@ fakeroot python do_image () { execute_pre_post_process(d, pre_process_cmds) } do_image[dirs] = "${TOPDIR}" +do_image[cleandirs] += "${IMGMANIFESTDIR}" addtask do_image after do_rootfs fakeroot python do_image_complete () { from oe.utils import execute_pre_post_process + from pathlib import Path + import json post_process_cmds = d.getVar("IMAGE_POSTPROCESS_COMMAND") execute_pre_post_process(d, post_process_cmds) + + image_manifest_dir = Path(d.getVar('IMGMANIFESTDIR')) + + data = [] + + for manifest_path in image_manifest_dir.glob("*.json"): + with manifest_path.open("r") as f: + data.extend(json.load(f)) + + with open(d.getVar("IMAGE_FILE_MANIFEST"), "w") as f: + json.dump(data, f) } do_image_complete[dirs] = "${TOPDIR}" SSTATETASKS += "do_image_complete"
@@ -292,6 +311,8 @@ SSTATE_SKIP_CREATION:task-image-complete = '1' do_image_complete[sstate-inputdirs] = "${IMGDEPLOYDIR}" do_image_complete[sstate-outputdirs] = "${DEPLOY_DIR_IMAGE}" do_image_complete[stamp-extra-info] = "${MACHINE_ARCH}" +do_image_complete[sstate-plaindirs] += "${IMAGE_FILE_MANIFEST_DIR}" +do_image_complete[dirs] += "${IMAGE_FILE_MANIFEST_DIR}" addtask do_image_complete after do_image before do_build python do_image_complete_setscene () { sstate_setscene(d)
@@ -507,12 +528,14 @@ python () { d.setVar(task, '\n'.join(cmds)) d.setVarFlag(task, 'func', '1') d.setVarFlag(task, 'fakeroot', '1') + d.setVarFlag(task, 'imagetype', t) d.appendVarFlag(task, 'prefuncs', ' ' + debug + ' set_image_size') d.prependVarFlag(task, 'postfuncs', 'create_symlinks ') d.appendVarFlag(task, 'subimages', ' ' + ' '.join(subimages)) d.appendVarFlag(task, 'vardeps', ' ' + ' '.join(vardeps)) d.appendVarFlag(task, 'vardepsexclude', ' DATETIME DATE ' + ' '.join(vardepsexclude)) + d.appendVarFlag(task, 'postfuncs', ' write_image_file_manifest') bb.debug(2, "Adding task %s before %s, after %s" % (task, 'do_image_complete', after)) bb.build.addtask(task, 'do_image_complete', after, d)
@@ -610,6 +633,41 @@ python create_symlinks() { bb.note("Skipping symlink, source does not exist: %s -> %s" % (dst, src)) } +python write_image_file_manifest() { + import json + from pathlib import Path + + taskname = d.getVar("BB_CURRENTTASK") + image_deploy_dir = Path(d.getVar('IMGDEPLOYDIR')) + image_manifest_dir = Path(d.getVar('IMGMANIFESTDIR')) + manifest_path = image_manifest_dir / ("do_" + d.getVar("BB_CURRENTTASK") + ".json") + + image_name = d.getVar("IMAGE_NAME") + image_basename = d.getVar("IMAGE_BASENAME") + machine = d.getVar("MACHINE") + + subimages = (d.getVarFlag("do_" + taskname, 'subimages', False) or "").split() + imagetype = d.getVarFlag("do_" + taskname, 'imagetype', False) + + data = { + "taskname": taskname, + "imagetype": imagetype, + "images": [] + } + + for type in subimages: + image_filename = image_name + "." + type + image_path = image_deploy_dir / image_filename + if not image_path.exists(): + continue + data["images"].append({ + "filename": image_filename, + }) + + with manifest_path.open("w") as f: + json.dump([data], f) +} + MULTILIBRE_ALLOW_REP += "${base_bindir} ${base_sbindir} ${bindir} ${sbindir} ${libexecdir} ${sysconfdir} ${nonarch_base_libdir}/udev /lib/modules/[^/]*/modules.*" MULTILIB_CHECK_FILE = "${WORKDIR}/multilib_check.py" MULTILIB_TEMP_ROOTFS = "${WORKDIR}/multilib"
Signature · 1 line
2.43.2
PATCH v4 2/10 classes-recipe/baremetal-image: Add image file manifest
In reply to Joshua WattDownstream tasks may want to know what image files were written so write out a manifest in do_image_complete. The format of the manifest is the same as the one in image.bbclass
--- meta/classes-recipe/baremetal-image.bbclass | 30 +++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-)
meta/classes-recipe/baremetal-image.bbclass +28 -2
@@ -30,6 +30,9 @@ BAREMETAL_BINNAME ?= "hello_baremetal_${MACHINE}" IMAGE_LINK_NAME ?= "baremetal-helloworld-image-${MACHINE}" IMAGE_NAME_SUFFIX ?= "" +IMAGE_FILE_MANIFEST_DIR = "${WORKDIR}/deploy-image-file-manifest" +IMAGE_FILE_MANIFEST = "${IMAGE_FILE_MANIFEST_DIR}/manifest.json" + do_rootfs[dirs] = "${IMGDEPLOYDIR} ${DEPLOY_DIR_IMAGE}" do_image(){
@@ -37,8 +40,28 @@ do_image(){ install ${D}/${base_libdir}/firmware/${BAREMETAL_BINNAME}.elf ${IMGDEPLOYDIR}/${IMAGE_LINK_NAME}.elf } -do_image_complete(){ - : +python do_image_complete(){ + from pathlib import Path + import json + + data = { + "taskname": "do_image", + "imagetype": "baremetal-image", + "images": [] + } + + img_deploy_dir = Path(d.getVar("IMGDEPLOYDIR")) + + for child in img_deploy_dir.iterdir(): + if not child.is_file() or child.is_symlink(): + continue + + data["images"].append({ + "filename": child.name, + }) + + with open(d.getVar("IMAGE_FILE_MANIFEST"), "w") as f: + json.dump([data], f) } python do_rootfs(){
@@ -62,6 +85,7 @@ python do_rootfs(){ bb.utils.mkdirhier(sysconfdir) execute_pre_post_process(d, d.getVar('ROOTFS_POSTPROCESS_COMMAND')) + execute_pre_post_process(d, d.getVar("ROOTFS_POSTUNINSTALL_COMMAND")) }
@@ -72,6 +96,8 @@ SSTATE_SKIP_CREATION:task-image-complete = '1' do_image_complete[sstate-inputdirs] = "${IMGDEPLOYDIR}" do_image_complete[sstate-outputdirs] = "${DEPLOY_DIR_IMAGE}" do_image_complete[stamp-extra-info] = "${MACHINE_ARCH}" +do_image_complete[sstate-plaindirs] += "${IMAGE_FILE_MANIFEST_DIR}" +do_image_complete[dirs] += "${IMAGE_FILE_MANIFEST_DIR}" addtask do_image_complete after do_image before do_build python do_image_complete_setscene () {
Signature · 1 line
2.43.2
Re: [OE-core][PATCH v4 02/10] classes-recipe/baremetal-image: Add image file manifest
There's already an IMAGE_MANIFEST file that's quite different from this? tmp/deploy/core-image-minimal-qemux86-64.rootfs.manifest base-files qemux86_64 3.0.14
base-passwd core2_64 3.6.3
busybox core2_64 1.36.1
So I find the name very confusing. Perhaps call this something completely different, like "result" or do-image-task-output or similar...
Regards
//Ernst
Den mån 24 juni 2024 kl 21:32 skrev Joshua Watt via lists.openembedded.org <[email protected]>:
Quoted 88 lines
Downstream tasks may want to know what image files were written so write out a manifest in
do_image_complete. The format of the manifest is the same as the one in image.bbclassSigned-off-by Joshua Watt <[email protected]>--- meta/classes-recipe/baremetal-image.bbclass | 30 +++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-)meta/classes-recipe/baremetal-image.bbclass +3
-0@@ -30,6 +30,9 @@ BAREMETAL_BINNAME ?= "hello_baremetal_${MACHINE}" IMAGE_LINK_NAME ?= "baremetal-helloworld-image-${MACHINE}" IMAGE_NAME_SUFFIX ?= "" +IMAGE_FILE_MANIFEST_DIR = "${WORKDIR}/deploy-image-file-manifest" +IMAGE_FILE_MANIFEST = "${IMAGE_FILE_MANIFEST_DIR}/manifest.json" + do_rootfs[dirs] = "${IMGDEPLOYDIR} ${DEPLOY_DIR_IMAGE}" do_image(){@@ -37,8 +40,28 @@ do_image(){ install ${D}/${base_libdir}/firmware/${BAREMETAL_BINNAME}.elf${IMGDEPLOYDIR}/${IMAGE_LINK_NAME}.elf
}meta/classes-recipe/baremetal-image.bbclass +25
-3-do_image_complete(){ - : +python do_image_complete(){ + from pathlib import Path + import json + + data = { + "taskname": "do_image", + "imagetype": "baremetal-image", + "images": [] + } + + img_deploy_dir = Path(d.getVar("IMGDEPLOYDIR")) + + for child in img_deploy_dir.iterdir(): + if not child.is_file() or child.is_symlink(): + continue + + data["images"].append({ + "filename": child.name, + }) + + with open(d.getVar("IMAGE_FILE_MANIFEST"), "w") as f: + json.dump([data], f) } python do_rootfs(){@@ -62,6 +85,7 @@ python do_rootfs(){ bb.utils.mkdirhier(sysconfdir) execute_pre_post_process(d, d.getVar('ROOTFS_POSTPROCESS_COMMAND')) + execute_pre_post_process(d, d.getVar("ROOTFS_POSTUNINSTALL_COMMAND")) }@@ -72,6 +96,8 @@ SSTATE_SKIP_CREATION:task-image-complete = '1' do_image_complete[sstate-inputdirs] = "${IMGDEPLOYDIR}" do_image_complete[sstate-outputdirs] = "${DEPLOY_DIR_IMAGE}" do_image_complete[stamp-extra-info] = "${MACHINE_ARCH}" +do_image_complete[sstate-plaindirs] += "${IMAGE_FILE_MANIFEST_DIR}" +do_image_complete[dirs] += "${IMAGE_FILE_MANIFEST_DIR}" addtask do_image_complete after do_image before do_build python do_image_complete_setscene () { --2.43.2
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group. View/Reply Online (#201104):
https://lists.openembedded.org/g/openembedded-core/message/201104
Mute This Topic: https://lists.openembedded.org/mt/106856870/4947266
Group Owner: [email protected]
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [ [email protected]]
-=-=-=-=-=-=-=-=-=-=-=-
PATCH v4 3/10 classes/create-spdx-3.0: Add classes
In reply to Joshua WattAdds a class to generate SPDX 3.0 output and an image class that is used when generating images
--- meta/classes-recipe/packagegroup.bbclass | 2 + meta/classes/create-spdx-3.0.bbclass | 1179 ++++ meta/classes/create-spdx-image-3.0.bbclass | 224 + meta/classes/spdx-common.bbclass | 6 +- meta/lib/oe/sbom30.py | 1122 ++++ meta/lib/oe/spdx30.py | 5996 ++++++++++++++++++++ 6 files changed, 8528 insertions(+), 1 deletion(-) create mode 100644 meta/classes/create-spdx-3.0.bbclass create mode 100644 meta/classes/create-spdx-image-3.0.bbclass create mode 100644 meta/lib/oe/sbom30.py create mode 100644 meta/lib/oe/spdx30.py
meta/classes-recipe/packagegroup.bbclass +2 -0
@@ -56,6 +56,8 @@ deltask do_populate_sysroot do_create_runtime_spdx[deptask] = "do_create_spdx" do_create_runtime_spdx[rdeptask] = "" +do_create_package_spdx[deptask] = "do_create_spdx" +do_create_package_spdx[rdeptask] = "" INHIBIT_DEFAULT_DEPS = "1"
meta/classes/create-spdx-3.0.bbclass new file +1179 -0
1,179 lines not on this page · raw message
meta/classes/create-spdx-image-3.0.bbclass new file +224 -0
224 lines not on this page · raw message
meta/classes/spdx-common.bbclass +5 -1
@@ -17,6 +17,7 @@ SPDXDEPLOY = "${SPDXDIR}/deploy" SPDXWORK = "${SPDXDIR}/work" SPDXIMAGEWORK = "${SPDXDIR}/image-work" SPDXSDKWORK = "${SPDXDIR}/sdk-work" +SPDXSDKEXTWORK = "${SPDXDIR}/sdk-ext-work" SPDXDEPS = "${SPDXDIR}/deps.json" SPDX_TOOL_NAME ??= "oe-spdx-creator"
@@ -68,7 +69,7 @@ def get_json_indent(d): return 2 return None -python() { +def load_spdx_license_data(d): import json if d.getVar("SPDX_LICENSE_DATA"): return
@@ -78,6 +79,9 @@ python() { # Transform the license array to a dictionary data["licenses"] = {l["licenseId"]: l for l in data["licenses"]} d.setVar("SPDX_LICENSE_DATA", data) + +python() { + load_spdx_license_data(d) } def process_sources(d):
meta/lib/oe/sbom30.py new file +1122 -0
1,122 lines not on this page · raw message
meta/lib/oe/spdx30.py new file +5996 -0
443 lines not on this page · raw message
The rest of this diff is not shown. View the raw message.
Signature · 1 line
2.43.2
Part of the diff is not shown here. View the raw message.
Re: [OE-core][PATCH v4 03/10] classes/create-spdx-3.0: Add classes
Quoted 52 lines · Joshua Watt via lists.openembedded.org
meta/classes/create-spdx-3.0.bbclass new file +52
-0+python spdx30_build_started_handler () { + import oe.spdx30 + import oe.sbom30 + import os + from pathlib import Path + + # Create a copy of the datastore. Set PN to "bitbake" so that SPDX IDs can + # be generated + d = e.data.createCopy() + d.setVar("PN", "bitbake") + d.setVar("BB_TASKHASH", "bitbake") + load_spdx_license_data(d) + + deploy_dir_spdx = Path(e.data.getVar("DEPLOY_DIR_SPDX")) + + nonce = os.urandom(16).hex() + + objset = oe.sbom30.ObjectSet.new_objset(d, "bitbake", False) + + build = objset.add_root(oe.spdx30.build_Build( + _id=objset.new_spdxid(nonce, include_unihash=False), + creationInfo=objset.doc.creationInfo, + build_buildType=oe.sbom30.SPDX_BUILD_TYPE, + build_buildStartTime=oe.sbom30.spdx_now() + )) + + host_import_key = d.getVar("SPDX_BUILD_HOST") + if host_import_key: + objset.new_scoped_relationship( + [build], + oe.spdx30.RelationshipType.hasHost, + oe.spdx30.LifecycleScopeType.build, + [objset.new_import("SPDX_BUILD_HOST")], + ) + + invoked_by = objset.new_agent("SPDX_INVOKED_BY") + if invoked_by: + invoked_by_spdx = objset.new_scoped_relationship( + [build], + oe.spdx30.RelationshipType.invokedBy, + oe.spdx30.LifecycleScopeType.build, + [invoked_by], + ) + + on_behalf_of = objset.new_agent("SPDX_ON_BEHALF_OF") + if on_behalf_of: + objset.new_scoped_relationship( + [on_behalf_of], + oe.spdx30.RelationshipType.delegatedTo, + oe.spdx30.LifecycleScopeType.build, + invoked_by_spdx, + )
meta/classes/create-spdx-3.0.bbclass line 1178
Quoted 10 lines
meta/classes/create-spdx-3.0.bbclass new file +10
-0+ + for obj in objset.foreach_type(oe.spdx30.Element): + obj.extension.append(oe.sbom30.OELinkExtension(link_spdx_id=False)) + obj.extension.append(oe.sbom30.OEIdAliasExtension()) + + oe.sbom30.write_jsonld_doc(d, objset, deploy_dir_spdx / "bitbake.spdx.json") +} + +addhandler spdx30_build_started_handler +spdx30_build_started_handler[eventmask] = "bb.event.ConfigParsed"
Reading through, this caught my eye. Do we really need to do that at ConfigParsed? I suspect you'll hit this a lot more than you expect since each new execution thread could parse the config and trigger this. Would BuildStarted not be more appropriate?
I'm also worried about determinism here. I didn't look into what data this is writing out but it worries me...
Cheers,
Richard
Re: [OE-core][PATCH v4 03/10] classes/create-spdx-3.0: Add classes
In reply to Joshua WattComments inline below
Quoted 91 lines · Joshua Watt
Adds a class to generate SPDX 3.0 output and an image class that is used when generating images
Signed-off-by Joshua Watt <[email protected]>--- meta/classes-recipe/packagegroup.bbclass | 2 + meta/classes/create-spdx-3.0.bbclass | 1179 ++++ meta/classes/create-spdx-image-3.0.bbclass | 224 + meta/classes/spdx-common.bbclass | 6 +- meta/lib/oe/sbom30.py | 1122 ++++ meta/lib/oe/spdx30.py | 5996 ++++++++++++++++++++ 6 files changed, 8528 insertions(+), 1 deletion(-)create mode 100644 meta/classes/create-spdx-3.0.bbclass create mode 100644 meta/classes/create-spdx-image-3.0.bbclass create mode 100644
meta/lib/oe/sbom30.py
create mode 100644meta/lib/oe/spdx30.pymeta/classes-recipe/packagegroup.bbclass +2
-0@@ -56,6 +56,8 @@ deltask do_populate_sysroot do_create_runtime_spdx[deptask] = "do_create_spdx" do_create_runtime_spdx[rdeptask] = "" +do_create_package_spdx[deptask] = "do_create_spdx" +do_create_package_spdx[rdeptask] = "" INHIBIT_DEFAULT_DEPS = "1"meta/classes/create-spdx-3.0.bbclass new file +55
-0@@ -0,0 +1,1179 @@ +# +# Copyright OpenEmbedded Contributors +# +# SPDX-License-Identifier: GPL-2.0-only +# + +inherit spdx-common + +SPDX_VERSION = "3.0.0" + +# The list of SPDX profiles generated documents will conform to +SPDX_PROFILES ?= "core build software simpleLicensing security" + +SPDX_INCLUDE_BUILD_VARIABLES ??= "0" +SPDX_INCLUDE_BUILD_VARIABLES[doc] = "If set to '1', the bitbake variables for a \ + recipe will be included in the Build object" + +SPDX_IMPORTS ??= "" +SPDX_IMPORTS[doc] = "SPDX_IMPORTS is the base variable that describes how to \ + reference external SPDX ids. Each import is defined as a key in this \ + variable with a suffix to describe to as a suffix to look up more \ + information about the import. Each key can have the following variables: \ + SPDX_IMPORTS_<key>_spdxid: The Fully qualified SPDX ID of the object \ + SPDX_IMPORTS_<key>_uri: The URI where the SPDX Document that contains \ + the external object can be found. Optional but recommended \ + SPDX_IMPORTS_<key>_hash_<hash>: The Checksum of the SPDX Document that \ + contains the External ID. <hash> must be one the valid SPDX hashing \ + algorithms, as described by the HashAlgorithm vocabulary in the\ + SPDX 3 spec. Optional but recommended \ +" + +# Agents +# Bitbake variables can be used to describe an SPDX Agent that may be used +# during the build. An Agent is specified using a set of variables which all +# start with some common base name: +# +# <BASE>_name: The name of the Agent (required) +# <BASE>_type: The type of Agent. Must be one of "person", "organization", +# "software", or "agent" (the default if not specified) +# <BASE>_comment: The comment for the Agent (optional) +# <BASE>_id_<ID>: And External Identifier for the Agent. <ID> must be a valid +# ExternalIdentifierType from the SPDX 3 spec. Commonly, an E-mail address +# can be specified with <BASE>_id_email +# +# Alternatively, an Agent can be an external reference by referencing a key +# in SPDX_IMPORTS like so: +# +# <BASE>_import = "<key>" +# +# Finally, the same agent described by another set of agent variables can be +# referenced by specifying the basename of the variable that should be +# referenced: +# +# SPDX_PACKAGE_SUPPLIER_ref = "SPDX_AUTHORS_openembedded" +
meta/classes/create-spdx-3.0.bbclass line 65
Quoted 10 lines
meta/classes/create-spdx-3.0.bbclass new file +10
-0+SPDX_AUTHORS ??= "openembedded" +SPDX_AUTHORS[doc] = "A space separated list of the document authors. Each item \ + is used to name a base variable like SPDX_AUTHORS_<AUTHOR> that \ + describes the author." + +SPDX_AUTHORS_openembedded_name = "OpenEmbedded" +SPDX_AUTHORS_openembedded_type = "organization" + +SPDX_BUILD_HOST[doc] = "The base variable name to describe the build host on \ + which a build is running. Must be an SPDX_IMPORTS key"
Is there any sort of documentation or external reference for the variable above (as well as the SPDX_ below) that explains what the SPDX standard is expecting to be put in there?
I.e. machine name, host type, an arbitrary string that means something to the agent construction the SPDX, etc. I.e. how would I know what is valid in these various things?
This then leads to a second question, deterministic behavior/reproducibility. I believe the purpose of this is reproducible builds, but we should have a more deterministic approach in the Yocto Project where we provide (and/or check for host capabilities) to help allow this to be a more generic, many different host process.
Which leads to a third question, when a build uses sstate-cache, each host could be different then the build host that actually combines the builds into an image (SBOM). Is this a concern?
Quoted 8,515 lines
meta/classes/create-spdx-3.0.bbclass new file +1113
-0+SPDX_INVOKED_BY[doc] = "The base variable name to describe the Agent that \ + invoked the build, which builds will link to if specified." + +SPDX_ON_BEHALF_OF[doc] = "The base variable name to describe the Agent on who's \ + behalf the invoking Agent (SPDX_INVOKED_BY) is running the build." + +SPDX_PACKAGE_SUPPLIER[doc] = "The base variable name to describe the Agent who is supplying \ + artifacts produced by the build" + +SPDX_PACKAGE_ADDITIONAL_PURPOSE ?= "" +SPDX_PACKAGE_ADDITIONAL_PURPOSE[doc] = "The list of additional purposes to assign to \ + the generated packages for a recipe. The primary purpose is always `install`. \ + Packages overrides are allowed to override the additional purposes for \ + individual packages." + +SPDX_IMAGE_PURPOSE ?= "filesystemImage" +SPDX_IMAGE_PURPOSE[doc] = "The list of purposes to assign to the generated images. \ + The first listed item will be the Primary Purpose and all additional items will \ + be added as additional purposes" + +SPDX_SDK_PURPOSE ?= "install" +SPDX_SDK_PURPOSE[doc] = "The list of purposes to assign to the generate SDK installer. \ + The first listed item will be the Primary Purpose and all additional items will \ + be added as additional purposes" + +SPDX_INCLUDE_VEX ??= "current" +SPDX_INCLUDE_VEX[doc] = "Controls what VEX information is in the output. Set to \ + 'none' to disable all VEX data. Set to 'current' to only include VEX data \ + for vulnerabilities not already fixed in the upstream source code \ + (recommended). Set to 'all' to get all known historical vulnerabilities, \ + including those already fixed upstream (warning: This can be large and \ + slow)." + +IMAGE_CLASSES:append = " create-spdx-image-3.0" + +def add_license_expression(d, objset, license_expression): + from pathlib import Path + import oe.spdx30 + import oe.sbom30 + + license_data = d.getVar("SPDX_LICENSE_DATA") + simple_license_text = {} + license_text_map = {} + license_ref_idx = 0 + + def add_license_text(name): + nonlocal objset + nonlocal simple_license_text + + if name in simple_license_text: + return simple_license_text[name] + + lic = objset.find_filter( + oe.spdx30.simplelicensing_SimpleLicensingText, + name=name, + ) + + if lic is not None: + simple_license_text[name] = lic + return lic + + lic = objset.add(oe.spdx30.simplelicensing_SimpleLicensingText( + _id=objset.new_spdxid("license-text", name), + creationInfo=objset.doc.creationInfo, + name=name, + )) + simple_license_text[name] = lic + + if name == "PD": + lic.simplelicensing_licenseText = "Software released to the public domain" + return lic + + # Seach for the license in COMMON_LICENSE_DIR and LICENSE_PATH + for directory in [d.getVar('COMMON_LICENSE_DIR')] + (d.getVar('LICENSE_PATH') or '').split(): + try: + with (Path(directory) / name).open(errors="replace") as f: + lic.simplelicensing_licenseText = f.read() + return lic + + except FileNotFoundError: + pass + + # If it's not SPDX or PD, then NO_GENERIC_LICENSE must be set + filename = d.getVarFlag('NO_GENERIC_LICENSE', name) + if filename: + filename = d.expand("${S}/" + filename) + with open(filename, errors="replace") as f: + lic.simplelicensing_licenseText = f.read() + return lic + else: + bb.fatal("Cannot find any text for license %s" % name) + + def convert(l): + nonlocal license_text_map + nonlocal license_ref_idx + + if l == "(" or l == ")": + return l + + if l == "&": + return "AND" + + if l == "|": + return "OR" + + if l == "CLOSED": + return "NONE" + + spdx_license = d.getVarFlag("SPDXLICENSEMAP", l) or l + if spdx_license in license_data["licenses"]: + return spdx_license + + spdx_license = "LicenseRef-" + l + if spdx_license not in license_text_map: + license_text_map[spdx_license] = add_license_text(l)._id + + return spdx_license + + lic_split = license_expression.replace("(", " ( ").replace(")", " ) ").replace("|", " | ").replace("&", " & ").split() + spdx_license_expression = ' '.join(convert(l) for l in lic_split) + + return objset.new_license_expression(spdx_license_expression, license_text_map) + + +def add_package_files(d, objset, topdir, get_spdxid, get_purposes, *, archive=None, ignore_dirs=[], ignore_top_level_dirs=[]): + from pathlib import Path + import oe.spdx30 + import oe.sbom30 + + source_date_epoch = d.getVar("SOURCE_DATE_EPOCH") + if source_date_epoch: + source_date_epoch = int(source_date_epoch) + + spdx_files = set() + + file_counter = 1 + for subdir, dirs, files in os.walk(topdir): + dirs[:] = [d for d in dirs if d not in ignore_dirs] + if subdir == str(topdir): + dirs[:] = [d for d in dirs if d not in ignore_top_level_dirs] + + for file in files: + filepath = Path(subdir) / file + if filepath.is_symlink() or not filepath.is_file(): + continue + + bb.debug(1, "Adding file %s to %s" % (filepath, objset.doc._id)) + + filename = str(filepath.relative_to(topdir)) + file_purposes = get_purposes(filepath) + + spdx_file = objset.new_file( + get_spdxid(file_counter), + filename, + filepath, + purposes=file_purposes, + ) + spdx_files.add(spdx_file) + + if oe.spdx30.software_SoftwarePurpose.source in file_purposes: + objset.scan_declared_licenses(spdx_file, filepath) + + if archive is not None: + with filepath.open("rb") as f: + info = archive.gettarinfo(fileobj=f) + info.name = filename + info.uid = 0 + info.gid = 0 + info.uname = "root" + info.gname = "root" + + if source_date_epoch is not None and info.mtime > source_date_epoch: + info.mtime = source_date_epoch + + archive.addfile(info, f) + + file_counter += 1 + + return spdx_files + + +def get_package_sources_from_debug(d, package, package_files, sources, source_hash_cache): + from pathlib import Path + import oe.packagedata + + def file_path_match(file_path, pkg_file): + if file_path.lstrip("/") == pkg_file.name.lstrip("/"): + return True + + for e in pkg_file.extension: + if isinstance(e, oe.sbom30.OEFileNameAliasExtension): + for a in e.aliases: + if file_path.lstrip("/") == a.lstrip("/"): + return True + + return False + + debug_search_paths = [ + Path(d.getVar('PKGD')), + Path(d.getVar('STAGING_DIR_TARGET')), + Path(d.getVar('STAGING_DIR_NATIVE')), + Path(d.getVar('STAGING_KERNEL_DIR')), + ] + + pkg_data = oe.packagedata.read_subpkgdata_extended(package, d) + + if pkg_data is None: + return + + dep_source_files = set() + + for file_path, file_data in pkg_data["files_info"].items(): + if not "debugsrc" in file_data: + continue + + if not any(file_path_match(file_path, pkg_file) for pkg_file in package_files): + bb.fatal("No package file found for %s in %s; SPDX found: %s" % (str(file_path), package, + " ".join(p.name for p in package_files))) + continue + + for debugsrc in file_data["debugsrc"]: + for search in debug_search_paths: + if debugsrc.startswith("/usr/src/kernel"): + debugsrc_path = search / debugsrc.replace('/usr/src/kernel/', '') + else: + debugsrc_path = search / debugsrc.lstrip("/") + + if debugsrc_path in source_hash_cache: + file_sha256 = source_hash_cache[debugsrc_path] + if file_sha256 is None: + continue + else: + if not debugsrc_path.exists(): + source_hash_cache[debugsrc_path] = None + continue + + file_sha256 = bb.utils.sha256_file(debugsrc_path) + source_hash_cache[debugsrc_path] = file_sha256 + + if file_sha256 in sources: + source_file = sources[file_sha256] + dep_source_files.add(source_file) + else: + bb.debug(1, "Debug source %s with SHA256 %s not found in any dependency" % (str(debugsrc_path), file_sha256)) + break + else: + bb.debug(1, "Debug source %s not found" % debugsrc) + + return dep_source_files + +get_package_sources_from_debug[vardepsexclude] += "STAGING_KERNEL_DIR" + +def collect_dep_objsets(d, build): + import json + from pathlib import Path + import oe.sbom30 + import oe.spdx30 + + deps = get_spdx_deps(d) + + dep_objsets = [] + dep_builds = set() + + dep_build_spdxids = set() + for dep_pn, _, in_taskhash in deps: + bb.debug(1, "Fetching SPDX for dependency %s" % (dep_pn)) + dep_build, dep_objset = oe.sbom30.find_root_obj_in_jsonld(d, "recipes", dep_pn, oe.spdx30.build_Build) + # If the dependency is part of the taskhash, return it to be linked + # against. Otherwise, it cannot be linked against because this recipe + # will not rebuilt if dependency changes + if in_taskhash: + dep_objsets.append(dep_objset) + + # The build _can_ be linked against (by alias) + dep_builds.add(dep_build) + + return dep_objsets, dep_builds + +collect_dep_objsets[vardepsexclude] = "SSTATE_ARCHS" + +def collect_dep_sources(dep_objsets): + import oe.spdx30 + import oe.sbom30 + + sources = {} + for objset in dep_objsets: + # Don't collect sources from native recipes as they + # match non-native sources also. + if objset.is_native(): + continue + + bb.debug(1, "Fetching Sources for dependency %s" % (objset.doc.name)) + + dep_build = objset.find_root(oe.spdx30.build_Build) + if not dep_build: + bb.fatal("Unable to find a build") + + for e in objset.foreach_type(oe.spdx30.Relationship): + if dep_build is not e.from_: + continue + + if e.relationshipType != oe.spdx30.RelationshipType.hasInputs: + continue + + for to in e.to: + if not isinstance(to, oe.spdx30.software_File): + continue + + if to.software_primaryPurpose != oe.spdx30.software_SoftwarePurpose.source: + continue + + for v in to.verifiedUsing: + if v.algorithm == oe.spdx30.HashAlgorithm.sha256: + sources[v.hashValue] = to + break + else: + bb.fatal("No SHA256 found for %s in %s" % (to.name, objset.doc.name)) + + return sources + +def add_download_files(d, objset): + import oe.patch + import oe.spdx30 + import os + + inputs = set() + + urls = d.getVar("SRC_URI").split() + fetch = bb.fetch2.Fetch(urls, d) + + for download_idx, src_uri in enumerate(urls): + fd = fetch.ud[src_uri] + + for name in fd.names: + file_name = os.path.basename(fetch.localpath(src_uri)) + if oe.patch.patch_path(src_uri, fetch, '', expand=False): + primary_purpose = oe.spdx30.software_SoftwarePurpose.patch + else: + primary_purpose = oe.spdx30.software_SoftwarePurpose.source + + if fd.type == "file": + if os.path.isdir(fd.localpath): + walk_idx = 1 + for root, dirs, files in os.walk(fd.localpath): + for f in files: + f_path = os.path.join(root, f) + if os.path.islink(f_path): + # TODO: SPDX doesn't support symlinks yet + continue + + file = objset.new_file( + objset.new_spdxid("source", str(download_idx + 1), str(walk_idx)), + os.path.join(file_name, os.path.relpath(f_path, fd.localpath)), + f_path, + purposes=[primary_purpose], + ) + + inputs.add(file) + walk_idx += 1 + + else: + file = objset.new_file( + objset.new_spdxid("source", str(download_idx + 1)), + file_name, + fd.localpath, + purposes=[primary_purpose], + ) + inputs.add(file) + + else: + uri = fd.type + proto = getattr(fd, "proto", None) + if proto is not None: + uri = uri + "+" + proto + uri = uri + "://" + fd.host + fd.path + + if fd.method.supports_srcrev(): + uri = uri + "@" + fd.revisions[name] + + dl = objset.add(oe.spdx30.software_Package( + _id=objset.new_spdxid("source", str(download_idx + 1)), + creationInfo=objset.doc.creationInfo, + name=file_name, + software_primaryPurpose=primary_purpose, + software_downloadLocation=uri, + )) + + if fd.method.supports_checksum(fd): + # TODO Need something better than hard coding this + for checksum_id in ["sha256", "sha1"]: + expected_checksum = getattr(fd, "%s_expected" % checksum_id, None) + if expected_checksum is None: + continue + + dl.verifiedUsing.append( + oe.spdx30.Hash( + algorithm=getattr(oe.spdx30.HashAlgorithm, checksum_id), + hashValue=expected_checksum, + ) + ) + + inputs.add(dl) + + return inputs + + +def collect_build_package_inputs(d, objset, build, packages): + providers = collect_package_providers(d) + + build_deps = set() + + for name in sorted(packages.keys()): + if name not in providers: + bb.fatal("Unable to find SPDX provider for '%s'" % name) + + pkg_name, pkg_hashfn = providers[name] + + # Copy all of the package SPDX files into the Sbom elements + pkg_spdx, _ = oe.sbom30.find_root_obj_in_jsonld( + d, + "packages", + pkg_name, + oe.spdx30.software_Package, + software_primaryPurpose=oe.spdx30.software_SoftwarePurpose.install, + ) + build_deps.add(pkg_spdx._id) + + if build_deps: + objset.new_scoped_relationship( + [build], + oe.spdx30.RelationshipType.hasInputs, + oe.spdx30.LifecycleScopeType.build, + sorted(list(build_deps)), + ) + +def set_purposes(d, element, *var_names, force_purposes=[]): + purposes = force_purposes[:] + + for var_name in var_names: + val = d.getVar(var_name) + if val: + purposes.extend(val.split()) + break + + if not purposes: + bb.warn("No SPDX purposes found in %s" % " ".join(var_names)) + return + + element.software_primaryPurpose = getattr(oe.spdx30.software_SoftwarePurpose, purposes[0]) + element.software_additionalPurpose = [getattr(oe.spdx30.software_SoftwarePurpose, p) for p in purposes[1:]] + + +python do_create_spdx() { + import oe.sbom30 + import oe.spdx30 + from pathlib import Path + from contextlib import contextmanager + import oe.cve_check + from datetime import datetime + + def set_var_field(var, obj, name, package=None): + val = None + if package: + val = d.getVar("%s:%s" % (var, package)) + + if not val: + val = d.getVar(var) + + if val: + setattr(obj, name, val) + + deploydir = Path(d.getVar("SPDXDEPLOY")) + deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX")) + spdx_workdir = Path(d.getVar("SPDXWORK")) + include_sources = d.getVar("SPDX_INCLUDE_SOURCES") == "1" + pkg_arch = d.getVar("SSTATE_PKGARCH") + is_native = bb.data.inherits_class("native", d) or bb.data.inherits_class("cross", d) + include_vex = d.getVar("SPDX_INCLUDE_VEX") + if not include_vex in ("none", "current", "all"): + bb.fatal("SPDX_INCLUDE_VEX must be one of 'none', 'current', 'all'") + + build_objset = oe.sbom30.ObjectSet.new_objset(d, d.getVar("PN")) + + build = build_objset.new_sub_build("recipe", "recipe") + build_objset.doc.rootElement.append(build) + + build_objset.set_is_native(is_native) + + for var in (d.getVar('SPDX_CUSTOM_ANNOTATION_VARS') or "").split(): + new_annotation( + d, + build_objset, + build, + "%s=%s" % (var, d.getVar(var)), + oe.spdx30.AnnotationType.other + ) + + build_inputs = set() + + # Add CVEs + cve_by_status = {} + if include_vex != "none": + for cve in (d.getVarFlags("CVE_STATUS") or {}): + status, detail, description = oe.cve_check.decode_cve_status(d, cve) + + # If this CVE is fixed upstream, skip it unless all CVEs are + # specified. + if include_vex != "all" and detail in ("fixed-version", "cpe-stable-backport"): + bb.debug(1, "Skipping %s since it is already fixed upstream" % cve) + continue + + cve_by_status.setdefault(status, {})[cve] = ( + build_objset.new_cve_vuln(cve), + detail, + description, + ) + + cpe_ids = oe.cve_check.get_cpe_ids(d.getVar("CVE_PRODUCT"), d.getVar("CVE_VERSION")) + + source_files = add_download_files(d, build_objset) + build_inputs |= source_files + + recipe_spdx_license = add_license_expression(d, build_objset, d.getVar("LICENSE")) + build_objset.new_relationship( + source_files, + oe.spdx30.RelationshipType.hasConcludedLicense, + [recipe_spdx_license], + ) + + if process_sources(d) and include_sources: + bb.debug(1, "Adding source files to SPDX") + spdx_get_src(d) + + build_inputs |= add_package_files( + d, + build_objset, + spdx_workdir, + lambda file_counter: build_objset.new_spdxid("sourcefile", str(file_counter)), + lambda filepath: [oe.spdx30.software_SoftwarePurpose.source], + ignore_dirs=[".git"], + ignore_top_level_dirs=["temp"], + archive=None, + ) + + + dep_objsets, dep_builds = collect_dep_objsets(d, build) + if dep_builds: + build_objset.new_scoped_relationship( + [build], + oe.spdx30.RelationshipType.dependsOn, + oe.spdx30.LifecycleScopeType.build, + sorted(oe.sbom30.get_element_link_id(b) for b in dep_builds), + ) + + debug_source_ids = set() + source_hash_cache = {} + + # Write out the package SPDX data now. It is not complete as we cannot + # write the runtime data, so write it to a staging area and a later task + # will write out the final collection + + # TODO: Handle native recipe output + if not is_native: + bb.debug(1, "Collecting Dependency sources files") + sources = collect_dep_sources(dep_objsets) + + bb.build.exec_func("read_subpackage_metadata", d) + + pkgdest = Path(d.getVar("PKGDEST")) + for package in d.getVar("PACKAGES").split(): + if not oe.packagedata.packaged(package, d): + continue + + pkg_name = d.getVar("PKG:%s" % package) or package + + bb.debug(1, "Creating SPDX for package %s" % pkg_name) + + pkg_objset = oe.sbom30.ObjectSet.new_objset(d, pkg_name) + + spdx_package = pkg_objset.add_root(oe.spdx30.software_Package( + _id=pkg_objset.new_spdxid("package", pkg_name), + creationInfo=pkg_objset.doc.creationInfo, + name=pkg_name, + software_packageVersion=d.getVar("PV"), + builtTime=oe.sbom30.spdx_now(), + )) + + set_purposes( + d, + spdx_package, + "SPDX_PACKAGE_ADDITIONAL_PURPOSE:%s" % package, + "SPDX_PACKAGE_ADDITIONAL_PURPOSE", + force_purposes=["install"], + ) + + + supplier = build_objset.new_agent("SPDX_PACKAGE_SUPPLIER") + if supplier is not None: + spdx_package.supplier = supplier if isinstance(supplier, str) else supplier._id + + set_var_field("HOMEPAGE", spdx_package, "software_homePage", package=package) + set_var_field("SUMMARY", spdx_package, "summary", package=package) + set_var_field("DESCRIPTION", spdx_package, "description", package=package) + + pkg_objset.new_scoped_relationship( + [build._id], + oe.spdx30.RelationshipType.hasOutputs, + oe.spdx30.LifecycleScopeType.build, + [spdx_package], + ) + + for cpe_id in cpe_ids: + spdx_package.externalIdentifier.append( + oe.spdx30.ExternalIdentifier( + externalIdentifierType=oe.spdx30.ExternalIdentifierType.cpe23, + identifier=cpe_id, + )) + + # TODO: Generate a file for each actual IPK/DEB/RPM/TGZ file + # generated and link it to the package + #spdx_package_file = pkg_objset.add(oe.spdx30.software_File( + # _id=pkg_objset.new_spdxid("distribution", pkg_name), + # creationInfo=pkg_objset.doc.creationInfo, + # name=pkg_name, + # builtTime=oe.sbom30.spdx_now(), + # software_primaryPurpose=spdx_package.software_primaryPurpose, + # software_additionalPurpose=spdx_package.software_additionalPurpose, + #)) + + ## TODO add hashes + #pkg_objset.new_relationship( + # [spdx_package], + # oe.spdx30.RelationshipType.hasDistributionArtifact, + # [spdx_package_file], + #) + + # NOTE: licenses live in the recipe collection and are referenced + # by ID in the package collection(s). This helps reduce duplication + # (since a lot of packages will have the same license), and also + # prevents duplicate license SPDX IDs in the packages + package_license = d.getVar("LICENSE:%s" % package) + if package_license and package_license != d.getVar("LICENSE"): + package_spdx_license = add_license_expression(d, build_objset, package_license) + else: + package_spdx_license = recipe_spdx_license + + pkg_objset.new_relationship( + [spdx_package], + oe.spdx30.RelationshipType.hasConcludedLicense, + [package_spdx_license._id], + ) + + # NOTE: CVE Elements live in the recipe collection + all_cves = set() + for status, cves in cve_by_status.items(): + for cve, items in cves.items(): + spdx_cve, detail, description = items + + all_cves.add(spdx_cve._id) + + if status == "Patched": + pkg_objset.new_vex_patched_relationship([spdx_cve._id], [spdx_package]) + elif status == "Unpatched": + pkg_objset.new_vex_unpatched_relationship([spdx_cve._id], [spdx_package]) + elif status == "Ignored": + spdx_vex = pkg_objset.new_vex_ignored_relationship( + [spdx_cve._id], + [spdx_package], + impact_statement=description, + ) + + if detail in ("ignored", "cpe-incorrect", "disputed", "upstream-wontfix"): + # VEX doesn't have justifications for this + pass + elif detail in ("not-applicable-config", "not-applicable-platform"): + for v in spdx_vex: + v.security_justificationType = oe.spdx30.security_VexJustificationType.vulnerableCodeNotPresent + else: + bb.fatal(f"Unknown detail '{detail}' for ignored {cve}") + else: + bb.fatal(f"Unknown CVE status {status}") + + if all_cves: + pkg_objset.new_relationship( + [spdx_package], + oe.spdx30.RelationshipType.hasAssociatedVulnerability, + sorted(list(all_cves)), + ) + + bb.debug(1, "Adding package files to SPDX for package %s" % pkg_name) + package_files = add_package_files( + d, + pkg_objset, + pkgdest / package, + lambda file_counter: pkg_objset.new_spdxid("package", pkg_name, "file", str(file_counter)), + # TODO: Can we know the purpose here? + lambda filepath: [], + ignore_top_level_dirs=['CONTROL', 'DEBIAN'], + archive=None, + ) + + if package_files: + pkg_objset.new_relationship( + [spdx_package], + oe.spdx30.RelationshipType.contains, + sorted(list(package_files)), + ) + + if include_sources: + debug_sources = get_package_sources_from_debug(d, package, package_files, sources, source_hash_cache) + debug_source_ids |= set(oe.sbom30.get_element_link_id(d) for d in debug_sources) + + oe.sbom30.write_recipe_jsonld_doc(d, pkg_objset, "packages-staging", deploydir, create_spdx_id_links=False) + + if include_sources: + bb.debug(1, "Adding sysroot files to SPDX") + sysroot_files = add_package_files( + d, + build_objset, + d.expand("${COMPONENTS_DIR}/${PACKAGE_ARCH}/${PN}"), + lambda file_counter: build_objset.new_spdxid("sysroot", str(file_counter)), + lambda filepath: [], + archive=None, + ) + + if sysroot_files: + build_objset.new_scoped_relationship( + [build], + oe.spdx30.RelationshipType.hasOutputs, + oe.spdx30.LifecycleScopeType.build, + sorted(list(sysroot_files)), + ) + + if build_inputs or debug_source_ids: + build_objset.new_scoped_relationship( + [build], + oe.spdx30.RelationshipType.hasInputs, + oe.spdx30.LifecycleScopeType.build, + sorted(list(build_inputs)) + sorted(list(debug_source_ids)), + ) + + oe.sbom30.write_recipe_jsonld_doc(d, build_objset, "recipes", deploydir) +} +do_create_spdx[vardepsexclude] += "BB_NUMBER_THREADS" +addtask do_create_spdx after \ + do_collect_spdx_deps \ + do_populate_sysroot do_package do_packagedata \ + ${create_spdx_source_deps(d)} \ + do_package_write_ipk do_package_write_deb do_package_write_rpm do_package_write_tar \ + before do_populate_sdk do_populate_sdk_ext do_build do_rm_work + +def create_spdx_source_deps(d): + deps = [] + if d.getVar("SPDX_INCLUDE_SOURCES") == "1": + deps.extend([ + # do_unpack is a hack for now; we only need it to get the + # dependencies do_unpack already has so we can extract the source + # ourselves + "do_unpack", + # For kernel source code + "do_shared_workdir", + ]) + return " ".join(deps) + +SSTATETASKS += "do_create_spdx" +do_create_spdx[sstate-inputdirs] = "${SPDXDEPLOY}" +do_create_spdx[sstate-outputdirs] = "${DEPLOY_DIR_SPDX}" + +python do_create_spdx_setscene () { + sstate_setscene(d) +} +addtask do_create_spdx_setscene + +do_create_spdx[dirs] = "${SPDXWORK}" +do_create_spdx[cleandirs] = "${SPDXDEPLOY} ${SPDXWORK}" +do_create_spdx[depends] += "${PATCHDEPENDENCY}" + +python do_create_package_spdx() { + import oe.sbom30 + import oe.spdx30 + import oe.packagedata + from pathlib import Path + + deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX")) + deploydir = Path(d.getVar("SPDXRUNTIMEDEPLOY")) + is_native = bb.data.inherits_class("native", d) or bb.data.inherits_class("cross", d) + + providers = collect_package_providers(d) + pkg_arch = d.getVar("SSTATE_PKGARCH") + + if not is_native: + bb.build.exec_func("read_subpackage_metadata", d) + + dep_package_cache = {} + + # Any element common to all packages that need to be referenced by ID + # should be written into this objset set + common_objset = oe.sbom30.ObjectSet.new_objset(d, "%s-package-common" % d.getVar("PN")) + + pkgdest = Path(d.getVar("PKGDEST")) + for package in d.getVar("PACKAGES").split(): + localdata = bb.data.createCopy(d) + pkg_name = d.getVar("PKG:%s" % package) or package + localdata.setVar("PKG", pkg_name) + localdata.setVar('OVERRIDES', d.getVar("OVERRIDES", False) + ":" + package) + + if not oe.packagedata.packaged(package, localdata): + continue + + spdx_package, pkg_objset = oe.sbom30.load_obj_in_jsonld( + d, + pkg_arch, + "packages-staging", + pkg_name, + oe.spdx30.software_Package, + software_primaryPurpose=oe.spdx30.software_SoftwarePurpose.install, + ) + + # We will write out a new collection, so link it to the new + # creation info in the common package data. The old creation info + # should still exist and be referenced by all the existing elements + # in the package + pkg_objset.creationInfo = pkg_objset.copy_creation_info(common_objset.doc.creationInfo) + + runtime_spdx_deps = set() + + deps = bb.utils.explode_dep_versions2(localdata.getVar("RDEPENDS") or "") + seen_deps = set() + for dep, _ in deps.items(): + if dep in seen_deps: + continue + + if dep not in providers: + continue + + (dep, _) = providers[dep] + + if not oe.packagedata.packaged(dep, localdata): + continue + + dep_pkg_data = oe.packagedata.read_subpkgdata_dict(dep, d) + dep_pkg = dep_pkg_data["PKG"] + + if dep in dep_package_cache: + dep_spdx_package = dep_package_cache[dep] + else: + bb.debug(1, "Searching for %s" % dep_pkg) + dep_spdx_package, _ = oe.sbom30.find_root_obj_in_jsonld( + d, + "packages-staging", + dep_pkg, + oe.spdx30.software_Package, + software_primaryPurpose=oe.spdx30.software_SoftwarePurpose.install, + ) + dep_package_cache[dep] = dep_spdx_package + + runtime_spdx_deps.add(dep_spdx_package) + seen_deps.add(dep) + + if runtime_spdx_deps: + pkg_objset.new_scoped_relationship( + [spdx_package], + oe.spdx30.RelationshipType.dependsOn, + oe.spdx30.LifecycleScopeType.runtime, + [oe.sbom30.get_element_link_id(dep) for dep in runtime_spdx_deps], + ) + + oe.sbom30.write_recipe_jsonld_doc(d, pkg_objset, "packages", deploydir) + + oe.sbom30.write_recipe_jsonld_doc(d, common_objset, "common-package", deploydir) +} + +do_create_package_spdx[vardepsexclude] += "OVERRIDES SSTATE_ARCHS" + +addtask do_create_package_spdx after do_create_spdx before do_build do_rm_work +SSTATETASKS += "do_create_package_spdx" +do_create_package_spdx[sstate-inputdirs] = "${SPDXRUNTIMEDEPLOY}" +do_create_package_spdx[sstate-outputdirs] = "${DEPLOY_DIR_SPDX}" + +python do_create_package_spdx_setscene () { + sstate_setscene(d) +} +addtask do_create_package_spdx_setscene + +do_create_package_spdx[dirs] = "${SPDXRUNTIMEDEPLOY}" +do_create_package_spdx[cleandirs] = "${SPDXRUNTIMEDEPLOY}" +do_create_package_spdx[rdeptask] = "do_create_spdx" + +do_populate_sdk[recrdeptask] += "do_create_spdx do_create_package_spdx" +do_populate_sdk[cleandirs] += "${SPDXSDKWORK}" +do_populate_sdk[postfuncs] += "sdk_create_sbom" +POPULATE_SDK_POST_HOST_COMMAND:append:task-populate-sdk = " sdk_host_create_spdx" +POPULATE_SDK_POST_TARGET_COMMAND:append:task-populate-sdk = " sdk_target_create_spdx" + +do_populate_sdk_ext[recrdeptask] += "do_create_spdx do_create_package_spdx" +do_populate_sdk_ext[cleandirs] += "${SPDXSDKEXTWORK}" +do_populate_sdk_ext[postfuncs] += "sdk_ext_create_sbom" +POPULATE_SDK_POST_HOST_COMMAND:append:task-populate-sdk-ext = " sdk_ext_host_create_spdx" +POPULATE_SDK_POST_TARGET_COMMAND:append:task-populate-sdk-ext = " sdk_ext_target_create_spdx" + +python sdk_host_create_spdx() { + from pathlib import Path + spdx_work_dir = Path(d.getVar('SPDXSDKWORK')) + + sdk_create_spdx(d, "host", spdx_work_dir, d.getVar("TOOLCHAIN_OUTPUTNAME")) +} + +python sdk_target_create_spdx() { + from pathlib import Path + spdx_work_dir = Path(d.getVar('SPDXSDKWORK')) + + sdk_create_spdx(d, "target", spdx_work_dir, d.getVar("TOOLCHAIN_OUTPUTNAME")) +} + +python sdk_ext_host_create_spdx() { + from pathlib import Path + spdx_work_dir = Path(d.getVar('SPDXSDKEXTWORK')) + + # TODO: This doesn't seem to work + sdk_create_spdx(d, "host", spdx_work_dir, d.getVar("TOOLCHAINEXT_OUTPUTNAME")) +} + +python sdk_ext_target_create_spdx() { + from pathlib import Path + spdx_work_dir = Path(d.getVar('SPDXSDKEXTWORK')) + + # TODO: This doesn't seem to work + sdk_create_spdx(d, "target", spdx_work_dir, d.getVar("TOOLCHAINEXT_OUTPUTNAME")) +} + +def sdk_create_spdx(d, sdk_type, spdx_work_dir, toolchain_outputname): + from pathlib import Path + from oe.sdk import sdk_list_installed_packages + import oe.spdx30 + import oe.sbom30 + from datetime import datetime + + sdk_name = toolchain_outputname + "-" + sdk_type + sdk_packages = sdk_list_installed_packages(d, sdk_type == "target") + + objset = oe.sbom30.ObjectSet.new_objset(d, sdk_name) + + sdk_rootfs = objset.add_root(oe.spdx30.software_Package( + _id=objset.new_spdxid("sdk-rootfs", sdk_name), + creationInfo=objset.doc.creationInfo, + name=sdk_name, + software_primaryPurpose=oe.spdx30.software_SoftwarePurpose.archive, + builtTime=oe.sbom30.spdx_now(), + )) + + sdk_build = objset.add_root(objset.new_sub_build("sdk-rootfs", "sdk-rootfs")) + sdk_build.build_buildEndTime = oe.sbom30.spdx_now() + + objset.new_scoped_relationship( + [sdk_build], + oe.spdx30.RelationshipType.hasOutputs, + oe.spdx30.LifecycleScopeType.build, + [sdk_rootfs], + ) + + collect_build_package_inputs(d, objset, sdk_build, sdk_packages) + + objset.add_aliases() + oe.sbom30.write_jsonld_doc(d, objset, spdx_work_dir / "sdk-rootfs.spdx.json") + +python sdk_create_sbom() { + from pathlib import Path + sdk_deploydir = Path(d.getVar("SDKDEPLOYDIR")) + spdx_work_dir = Path(d.getVar('SPDXSDKWORK')) + + create_sdk_sbom(d, sdk_deploydir, spdx_work_dir, d.getVar("TOOLCHAIN_OUTPUTNAME")) +} + +python sdk_ext_create_sbom() { + from pathlib import Path + sdk_deploydir = Path(d.getVar("SDKEXTDEPLOYDIR")) + spdx_work_dir = Path(d.getVar('SPDXSDKEXTWORK')) + + create_sdk_sbom(d, sdk_deploydir, spdx_work_dir, d.getVar("TOOLCHAINEXT_OUTPUTNAME")) +} + +def create_sdk_sbom(d, sdk_deploydir, spdx_work_dir, toolchain_outputname): + import oe.spdx30 + import oe.sbom30 + from pathlib import Path + from datetime import datetime + + now = oe.sbom30.spdx_now() + + # Load the document written earlier + rootfs_objset = oe.sbom30.load_jsonld(d, spdx_work_dir / "sdk-rootfs.spdx.json", required=True) + + # Create a new build for the SDK installer + sdk_build = rootfs_objset.new_sub_build("sdk-populate", "sdk-populate") + sdk_build.build_buildEndTime = oe.sbom30.spdx_now() + + rootfs = rootfs_objset.find_root(oe.spdx30.software_Package) + if rootfs is None: + bb.fatal("Unable to find rootfs artifact") + + rootfs_objset.new_scoped_relationship( + [sdk_build], + oe.spdx30.RelationshipType.hasInputs, + oe.spdx30.LifecycleScopeType.build, + [rootfs] + ) + + files = set() + root_files = [] + + # NOTE: os.walk() doesn't return symlinks + for dirpath, dirnames, filenames in os.walk(sdk_deploydir): + for fn in filenames: + fpath = Path(dirpath) / fn + if not fpath.is_file() or fpath.is_symlink(): + continue + + relpath = str(fpath.relative_to(sdk_deploydir)) + + f = rootfs_objset.new_file( + rootfs_objset.new_spdxid("sdk-installer", relpath), + relpath, + fpath, + ) + f.builtTime = now + + if fn.endswith(".manifest"): + f.software_primaryPurpose = oe.spdx30.software_SoftwarePurpose.manifest + elif fn.endswith(".testdata.json"): + f.software_primaryPurpose = oe.spdx30.software_SoftwarePurpose.configuration + else: + set_purposes(d, f, "SPDX_SDK_PURPOSE") + root_files.append(f) + + files.add(f) + + if files: + rootfs_objset.new_scoped_relationship( + [sdk_build], + oe.spdx30.RelationshipType.hasOutputs, + oe.spdx30.LifecycleScopeType.build, + files, + ) + else: + bb.warn(f"No SDK output files found in {sdk_deploydir}") + + objset, sbom = oe.sbom30.create_sbom(d, toolchain_outputname, sorted(list(files)), [rootfs_objset]) + + oe.sbom30.write_jsonld_doc(d, objset, sdk_deploydir / (toolchain_outputname + ".spdx.json")) + +python spdx30_build_started_handler () { + import oe.spdx30 + import oe.sbom30 + import os + from pathlib import Path + + # Create a copy of the datastore. Set PN to "bitbake" so that SPDX IDs can + # be generated + d = e.data.createCopy() + d.setVar("PN", "bitbake") + d.setVar("BB_TASKHASH", "bitbake") + load_spdx_license_data(d) + + deploy_dir_spdx = Path(e.data.getVar("DEPLOY_DIR_SPDX")) + + nonce = os.urandom(16).hex() + + objset = oe.sbom30.ObjectSet.new_objset(d, "bitbake", False) + + build = objset.add_root(oe.spdx30.build_Build( + _id=objset.new_spdxid(nonce, include_unihash=False), + creationInfo=objset.doc.creationInfo, + build_buildType=oe.sbom30.SPDX_BUILD_TYPE, + build_buildStartTime=oe.sbom30.spdx_now() + )) + + host_import_key = d.getVar("SPDX_BUILD_HOST") + if host_import_key: + objset.new_scoped_relationship( + [build], + oe.spdx30.RelationshipType.hasHost, + oe.spdx30.LifecycleScopeType.build, + [objset.new_import("SPDX_BUILD_HOST")], + ) + + invoked_by = objset.new_agent("SPDX_INVOKED_BY") + if invoked_by: + invoked_by_spdx = objset.new_scoped_relationship( + [build], + oe.spdx30.RelationshipType.invokedBy, + oe.spdx30.LifecycleScopeType.build, + [invoked_by], + ) + + on_behalf_of = objset.new_agent("SPDX_ON_BEHALF_OF") + if on_behalf_of: + objset.new_scoped_relationship( + [on_behalf_of], + oe.spdx30.RelationshipType.delegatedTo, + oe.spdx30.LifecycleScopeType.build, + invoked_by_spdx, + ) + + for obj in objset.foreach_type(oe.spdx30.Element): + obj.extension.append(oe.sbom30.OELinkExtension(link_spdx_id=False)) + obj.extension.append(oe.sbom30.OEIdAliasExtension()) + + oe.sbom30.write_jsonld_doc(d, objset, deploy_dir_spdx / "bitbake.spdx.json") +} + +addhandler spdx30_build_started_handler +spdx30_build_started_handler[eventmask] = "bb.event.ConfigParsed" +meta/classes/create-spdx-image-3.0.bbclass new file +224
-0@@ -0,0 +1,224 @@ +# +# Copyright OpenEmbedded Contributors +# +# SPDX-License-Identifier: GPL-2.0-only +# +# SPDX image tasks + +SPDX_ROOTFS_PACKAGES = "${SPDXDIR}/rootfs-packages.json" +SPDXIMAGEDEPLOYDIR = "${SPDXDIR}/image-deploy" +SPDXROOTFSDEPLOY = "${SPDXDIR}/rootfs-deploy" + +python spdx_collect_rootfs_packages() { + import json + from pathlib import Path + from oe.rootfs import image_list_installed_packages + + root_packages_file = Path(d.getVar("SPDX_ROOTFS_PACKAGES")) + + packages = image_list_installed_packages(d) + if not packages: + packages = {} + + root_packages_file.parent.mkdir(parents=True, exist_ok=True) + with root_packages_file.open("w") as f: + json.dump(packages, f) +} +ROOTFS_POSTUNINSTALL_COMMAND =+ "spdx_collect_rootfs_packages" + +python do_create_rootfs_spdx() { + import json + from pathlib import Path + import oe.spdx30 + import oe.sbom30 + from datetime import datetime + + deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX")) + deploydir = Path(d.getVar("SPDXROOTFSDEPLOY")) + root_packages_file = Path(d.getVar("SPDX_ROOTFS_PACKAGES")) + image_basename = d.getVar("IMAGE_BASENAME") + machine = d.getVar("MACHINE") + + with root_packages_file.open("r") as f: + packages = json.load(f) + + objset = oe.sbom30.ObjectSet.new_objset(d, "%s-%s" % (image_basename, machine)) + + rootfs = objset.add_root(oe.spdx30.software_Package( + _id=objset.new_spdxid("rootfs", image_basename), + creationInfo=objset.doc.creationInfo, + name=image_basename, + software_primaryPurpose=oe.spdx30.software_SoftwarePurpose.archive, + builtTime=oe.sbom30.spdx_now() + )) + + rootfs_build = objset.add_root(objset.new_sub_build("rootfs", "rootfs")) + rootfs_build.build_buildEndTime = oe.sbom30.spdx_now() + + objset.new_scoped_relationship( + [rootfs_build], + oe.spdx30.RelationshipType.hasOutputs, + oe.spdx30.LifecycleScopeType.build, + [rootfs], + ) + + collect_build_package_inputs(d, objset, rootfs_build, packages) + + oe.sbom30.write_recipe_jsonld_doc(d, objset, "rootfs", deploydir) +} +addtask do_create_rootfs_spdx after do_rootfs before do_image +SSTATETASKS += "do_create_rootfs_spdx" +do_create_rootfs_spdx[sstate-inputdirs] = "${SPDXROOTFSDEPLOY}" +do_create_rootfs_spdx[sstate-outputdirs] = "${DEPLOY_DIR_SPDX}" +do_create_rootfs_spdx[recrdeptask] += "do_create_spdx do_create_package_spdx" +do_create_rootfs_spdx[cleandirs] += "${SPDXROOTFSDEPLOY}" + +python do_create_rootfs_spdx_setscene() { + sstate_setscene(d) +} +addtask do_create_rootfs_spdx_setscene + +python do_create_image_spdx() { + import oe.spdx30 + import oe.sbom30 + import json + from pathlib import Path + + image_deploy_dir = Path(d.getVar('IMGDEPLOYDIR')) + manifest_path = Path(d.getVar("IMAGE_FILE_MANIFEST")) + spdx_work_dir = Path(d.getVar('SPDXIMAGEWORK')) + + image_basename = d.getVar('IMAGE_BASENAME') + machine = d.getVar("MACHINE") + + objset = oe.sbom30.ObjectSet.new_objset(d, "%s-%s" % (image_basename, machine)) + + with manifest_path.open("r") as f: + manifest = json.load(f) + + builds = [] + for task in manifest: + imagetype = task["imagetype"] + taskname = task["taskname"] + + image_build = objset.add_root(objset.new_sub_build(taskname, "image/%s" % imagetype)) + image_build.build_buildEndTime = oe.sbom30.spdx_now() + builds.append(image_build) + + artifacts = [] + + for image in task["images"]: + image_filename = image["filename"] + image_path = image_deploy_dir / image_filename + a = objset.add_root(oe.spdx30.software_File( + _id=objset.new_spdxid("image", image_filename), + creationInfo=objset.doc.creationInfo, + name=image_filename, + builtTime=oe.sbom30.spdx_now(), + verifiedUsing=[ + oe.spdx30.Hash( + algorithm=oe.spdx30.HashAlgorithm.sha256, + hashValue=bb.utils.sha256_file(image_path), + ) + ] + )) + set_purposes(d, a, "SPDX_IMAGE_PURPOSE:%s" % imagetype, "SPDX_IMAGE_PURPOSE") + + artifacts.append(a) + + if artifacts: + objset.new_scoped_relationship( + [image_build], + oe.spdx30.RelationshipType.hasOutputs, + oe.spdx30.LifecycleScopeType.build, + artifacts, + ) + + if builds: + rootfs_image, _ = oe.sbom30.find_root_obj_in_jsonld( + d, + "rootfs", + "%s-%s" % (image_basename, machine), + oe.spdx30.software_Package, + # TODO: Should use a purpose to filter here? + ) + objset.new_scoped_relationship( + builds, + oe.spdx30.RelationshipType.hasInputs, + oe.spdx30.LifecycleScopeType.build, + [rootfs_image._id], + ) + + objset.add_aliases() + objset.link() + oe.sbom30.write_recipe_jsonld_doc(d, objset, "image", spdx_work_dir) +} +addtask do_create_image_spdx after do_image_complete do_create_rootfs_spdx before do_build +SSTATETASKS += "do_create_image_spdx" +SSTATE_SKIP_CREATION:task-combine-image-type-spdx = "1" +do_create_image_spdx[sstate-inputdirs] = "${SPDXIMAGEWORK}" +do_create_image_spdx[sstate-outputdirs] = "${DEPLOY_DIR_SPDX}" +do_create_image_spdx[cleandirs] = "${SPDXIMAGEWORK}" +do_create_image_spdx[dirs] = "${SPDXIMAGEWORK}" + +python do_create_image_spdx_setscene() { + sstate_setscene(d) +} +addtask do_create_image_spdx_setscene + + +python do_create_image_sbom() { + import os + from pathlib import Path + import oe.spdx30 + import oe.sbom30 + + image_name = d.getVar("IMAGE_NAME") + image_basename = d.getVar("IMAGE_BASENAME") + image_link_name = d.getVar("IMAGE_LINK_NAME") + imgdeploydir = Path(d.getVar("SPDXIMAGEDEPLOYDIR")) + machine = d.getVar("MACHINE") + + spdx_path = imgdeploydir / (image_name + ".spdx.json") + + root_elements = [] + + # TODO: Do we need to add the rootfs or are the image files sufficient? + rootfs_image, _ = oe.sbom30.find_root_obj_in_jsonld( + d, + "rootfs", + "%s-%s" % (image_basename, machine), + oe.spdx30.software_Package, + # TODO: Should use a purpose here? + ) + root_elements.append(rootfs_image._id) + + image_objset, _ = oe.sbom30.find_jsonld(d, "image", "%s-%s" % (image_basename, machine), required=True) + for o in image_objset.foreach_root(oe.spdx30.software_File): + root_elements.append(o._id) + + objset, sbom = oe.sbom30.create_sbom(d, image_name, root_elements) + + oe.sbom30.write_jsonld_doc(d, objset, spdx_path) + + def make_image_link(target_path, suffix): + if image_link_name: + link = imgdeploydir / (image_link_name + suffix) + if link != target_path: + link.symlink_to(os.path.relpath(target_path, link.parent)) + + make_image_link(spdx_path, ".spdx.json") +} +addtask do_create_image_sbom after do_create_rootfs_spdx do_create_image_spdx before do_build +SSTATETASKS += "do_create_image_sbom" +SSTATE_SKIP_CREATION:task-create-image-sbom = "1" +do_create_image_sbom[sstate-inputdirs] = "${SPDXIMAGEDEPLOYDIR}" +do_create_image_sbom[sstate-outputdirs] = "${DEPLOY_DIR_IMAGE}" +do_create_image_sbom[stamp-extra-info] = "${MACHINE_ARCH}" +do_create_image_sbom[cleandirs] = "${SPDXIMAGEDEPLOYDIR}" +do_create_image_sbom[recrdeptask] += "do_create_spdx do_create_package_spdx" + +python do_create_image_sbom_setscene() { + sstate_setscene(d) +} +addtask do_create_image_sbom_setscenemeta/classes/spdx-common.bbclass +5
-1@@ -17,6 +17,7 @@ SPDXDEPLOY = "${SPDXDIR}/deploy" SPDXWORK = "${SPDXDIR}/work" SPDXIMAGEWORK = "${SPDXDIR}/image-work" SPDXSDKWORK = "${SPDXDIR}/sdk-work" +SPDXSDKEXTWORK = "${SPDXDIR}/sdk-ext-work" SPDXDEPS = "${SPDXDIR}/deps.json" SPDX_TOOL_NAME ??= "oe-spdx-creator"@@ -68,7 +69,7 @@ def get_json_indent(d): return 2 return None -python() { +def load_spdx_license_data(d): import json if d.getVar("SPDX_LICENSE_DATA"): return@@ -78,6 +79,9 @@ python() { # Transform the license array to a dictionary data["licenses"] = {l["licenseId"]: l for l in data["licenses"]} d.setVar("SPDX_LICENSE_DATA", data) + +python() { + load_spdx_license_data(d) } def process_sources(d):meta/lib/oe/sbom30.py new file +1122
-0@@ -0,0 +1,1122 @@ +# +# Copyright OpenEmbedded Contributors +# +# SPDX-License-Identifier: GPL-2.0-only +# + +from pathlib import Path + +import oe.spdx30 +import bb +import re +import hashlib +import uuid +import os +from datetime import datetime, timezone + +OE_SPDX_BASE = "https://rdf.openembedded.org/spdx/3.0/" + +VEX_VERSION = "1.0.0" + +SPDX_BUILD_TYPE = "http://openembedded.org/bitbake" + + [email protected](OE_SPDX_BASE + "link-extension") +class OELinkExtension(oe.spdx30.extension_Extension): + """ + This custom extension controls if an Element creates a symlink based on + its SPDX ID in the deploy directory. Some elements may not be able to be + linked because they are duplicated in multiple documents (e.g. the bitbake + Build Element). Those elements can add this extension and set link_spdx_id + to False + + It is in internal extension that should be removed when writing out a final + SBoM + """ + + CLOSED = True + INTERNAL = True + + @classmethod + def _register_props(cls): + super()._register_props() + cls._add_property( + "link_spdx_id", + oe.spdx30.BooleanProp(), + OE_SPDX_BASE + "link-spdx-id", + min_count=1, + max_count=1, + ) + + # The symlinks written to the deploy directory are based on the hash of + # the SPDX ID. While this makes it easy to look them up, it can be + # difficult to trace a Element to the hashed symlink name. As a + # debugging aid, this property is set to the basename of the symlink + # when the symlink is created to make it easier to trace + cls._add_property( + "link_name", + oe.spdx30.StringProp(), + OE_SPDX_BASE + "link-name", + max_count=1, + ) + + [email protected](OE_SPDX_BASE + "id-alias") +class OEIdAliasExtension(oe.spdx30.extension_Extension): + """ + This extension allows an Element to provide an internal alias for the SPDX + ID. Since SPDX requires unique URIs for each SPDX ID, most of the objects + created have a unique UUID namespace and the unihash of the task encoded in + their SPDX ID. However, this causes a problem for referencing documents + across recipes, since the taskhash of a dependency may not factor into the + taskhash of the current task and thus the current task won't rebuild and + see the new SPDX ID when the dependency changes (e.g. ABI safe recipes and + tasks). + + To help work around this, this extension provides a non-unique alias for an + Element by which it can be referenced from other tasks/recipes. When a + final SBoM is created, references to these aliases will be replaced with + the actual unique SPDX ID. + + Most Elements will automatically get an alias created when they are written + out if they do not already have one. To suppress the creation of an alias, + add an extension with a blank `alias` property. + + + It is in internal extension that should be removed when writing out a final + SBoM + """ + + CLOSED = True + INTERNAL = True + + @classmethod + def _register_props(cls): + super()._register_props() + cls._add_property( + "alias", + oe.spdx30.StringProp(), + OE_SPDX_BASE + "alias", + max_count=1, + ) + + cls._add_property( + "link_name", + oe.spdx30.StringProp(), + OE_SPDX_BASE + "link-name", + max_count=1, + ) + + [email protected](OE_SPDX_BASE + "file-name-alias") +class OEFileNameAliasExtension(oe.spdx30.extension_Extension): + CLOSED = True + INTERNAL = True + + @classmethod + def _register_props(cls): + super()._register_props() + cls._add_property( + "aliases", + oe.spdx30.ListProp(oe.spdx30.StringProp()), + OE_SPDX_BASE + "filename-alias", + ) + + [email protected](OE_SPDX_BASE + "license-scanned") +class OELicenseScannedExtension(oe.spdx30.extension_Extension): + """ + The presence of this extension means the file has already been scanned for + license information + """ + + CLOSED = True + INTERNAL = True + + [email protected](OE_SPDX_BASE + "document-extension") +class OEDocumentExtension(oe.spdx30.extension_Extension): + """ + This extension is added to a SpdxDocument to indicate various useful bits + of information about its contents + """ + + CLOSED = True + + @classmethod + def _register_props(cls): + super()._register_props() + cls._add_property( + "is_native", + oe.spdx30.BooleanProp(), + OE_SPDX_BASE + "is-native", + max_count=1, + ) + + +def spdxid_hash(*items): + h = hashlib.md5() + for i in items: + if isinstance(i, oe.spdx30.Element): + h.update(i._id.encode("utf-8")) + else: + h.update(i.encode("utf-8")) + return h.hexdigest() + + +def spdx_now(): + return datetime.now(timezone.utc) + + +def get_element_link_id(e): + """ + Get the string ID which should be used to link to an Element. If the + element has an alias, that will be preferred, otherwise its SPDX ID will be + used. + """ + ext = get_alias(e) + if ext is not None and ext.alias: + return ext.alias + return e._id + + +def set_alias(obj, alias): + for ext in obj.extension: + if not isinstance(ext, OEIdAliasExtension): + continue + ext.alias = alias + return ext + + ext = OEIdAliasExtension(alias=alias) + obj.extension.append(ext) + return ext + + +def get_alias(obj): + for ext in obj.extension: + if not isinstance(ext, OEIdAliasExtension): + continue + return ext + + return None + + +def extract_licenses(filename): + lic_regex = re.compile( + rb"^\W*SPDX-License-Identifier:\s*([ \w\d.()+-]+?)(?:\s+\W*)?$", re.MULTILINE + ) + + try: + with open(filename, "rb") as f: + size = min(15000, os.stat(filename).st_size) + txt = f.read(size) + licenses = re.findall(lic_regex, txt) + if licenses: + ascii_licenses = [lic.decode("ascii") for lic in licenses] + return ascii_licenses + except Exception as e: + bb.warn(f"Exception reading {filename}: {e}") + return [] + + +def to_list(l): + if isinstance(l, set): + l = sorted(list(l)) + + if not isinstance(l, (list, tuple)): + raise TypeError("Must be a list or tuple. Got %s" % type(l)) + + return l + + +class ObjectSet(oe.spdx30.SHACLObjectSet): + def __init__(self, d): + super().__init__() + self.d = d + + def create_index(self): + self.by_sha256_hash = {} + super().create_index() + + def add_index(self, obj): + # Check that all elements are given an ID before being inserted + if isinstance(obj, oe.spdx30.Element): + if not obj._id: + raise ValueError("Element missing ID") + for ext in obj.extension: + if not isinstance(ext, OEIdAliasExtension): + continue + if ext.alias: + self.obj_by_id[ext.alias] = obj + + for v in obj.verifiedUsing: + if not isinstance(v, oe.spdx30.Hash): + continue + + if v.algorithm == oe.spdx30.HashAlgorithm.sha256: + continue + + self.by_sha256_hash.setdefault(v.hashValue, set()).add(obj) + + super().add_index(obj) + if isinstance(obj, oe.spdx30.SpdxDocument): + self.doc = obj + + def __filter_obj(self, obj, attr_filter): + return all(getattr(obj, k) == v for k, v in attr_filter.items()) + + def foreach_filter(self, typ, *, match_subclass=True, **attr_filter): + for obj in self.foreach_type(typ, match_subclass=match_subclass): + if self.__filter_obj(obj, attr_filter): + yield obj + + def find_filter(self, typ, *, match_subclass=True, **attr_filter): + for obj in self.foreach_filter( + typ, match_subclass=match_subclass, **attr_filter + ): + return obj + return None + + def foreach_root(self, typ, **attr_filter): + for obj in self.doc.rootElement: + if not isinstance(obj, typ): + continue + + if self.__filter_obj(obj, attr_filter): + yield obj + + def find_root(self, typ, **attr_filter): + for obj in self.foreach_root(typ, **attr_filter): + return obj + return None + + def add_root(self, obj): + self.add(obj) + self.doc.rootElement.append(obj) + return obj + + def is_native(self): + for e in self.doc.extension: + if not isinstance(e, oe.sbom30.OEDocumentExtension): + continue + + if e.is_native is not None: + return e.is_native + + return False + + def set_is_native(self, is_native): + for e in self.doc.extension: + if not isinstance(e, oe.sbom30.OEDocumentExtension): + continue + + e.is_native = is_native + return + + if is_native: + self.doc.extension.append(oe.sbom30.OEDocumentExtension(is_native=True)) + + def add_aliases(self): + for o in self.foreach_type(oe.spdx30.Element): + if not o._id or o._id.startswith("_:"): + continue + + alias_ext = get_alias(o) + if alias_ext is None: + unihash = self.d.getVar("BB_UNIHASH") + namespace = self.get_namespace() + if unihash not in o._id: + bb.warn(f"Unihash {unihash} not found in {o._id}") + elif namespace not in o._id: + bb.warn(f"Namespace {namespace} not found in {o._id}") + else: + alias_ext = set_alias( + o, + o._id.replace(unihash, "UNIHASH").replace( + namespace, self.d.getVar("PN") + ), + ) + + def remove_internal_extensions(self): + def remove(o): + o.extension = [e for e in o.extension if not getattr(e, "INTERNAL", False)] + + for o in self.foreach_type(oe.spdx30.Element): + remove(o) + + if self.doc: + remove(self.doc) + + def get_namespace(self): + namespace_uuid = uuid.uuid5( + uuid.NAMESPACE_DNS, self.d.getVar("SPDX_UUID_NAMESPACE") + ) + pn = self.d.getVar("PN") + return "%s/%s-%s" % ( + self.d.getVar("SPDX_NAMESPACE_PREFIX"), + pn, + str(uuid.uuid5(namespace_uuid, pn)), + ) + + def new_spdxid(self, *suffix, include_unihash=True): + items = [self.get_namespace()] + if include_unihash: + unihash = self.d.getVar("BB_UNIHASH") + items.append(unihash) + items.extend(re.sub(r"[^a-zA-Z0-9_-]", "_", s) for s in suffix) + return "/".join(items) + + def new_import(self, key): + base = f"SPDX_IMPORTS_{key}" + spdxid = self.d.getVar(f"{base}_spdxid") + if not spdxid: + bb.fatal(f"{key} is not a valid SPDX_IMPORTS key") + + for i in self.docs.imports: + if i.externalSpdxId == spdxid: + # Already imported + return spdxid + + m = oe.spdx30.ExternalMap(externalSpdxId=spdxid) + + uri = self.d.getVar(f"{base}_uri") + if uri: + m.locationHint = uri + + for pyname, algorithm in oe.spdx30.HashAlgorithm.NAMED_INDIVIDUALS.items(): + value = self.d.getVar(f"{base}_hash_{pyname}") + if value: + m.verifiedUsing.append( + oe.spdx30.Hash( + algorithm=algorithm, + hashValue=value, + ) + ) + + self.doc.imports.append(m) + return spdxid + + def new_agent(self, varname, *, creation_info=None): + ref_varname = self.d.getVar(f"{varname}_ref") + if ref_varname: + if ref_varname == varname: + bb.fatal(f"{varname} cannot reference itself") + return new_agent(varname, creation_info=creation_info) + + import_key = self.d.getVar(f"{varname}_import") + if import_key: + return self.new_import(import_key) + + name = self.d.getVar(f"{varname}_name") + if not name: + return None + + spdxid = self.new_spdxid("agent", name) + agent = self.find_by_id(spdxid) + if agent is not None: + return agent + + agent_type = self.d.getVar("%s_type" % varname) + if agent_type == "person": + agent = oe.spdx30.Person() + elif agent_type == "software": + agent = oe.spdx30.SoftwareAgent() + elif agent_type == "organization": + agent = oe.spdx30.Organization() + elif not agent_type or agent_type == "agent": + agent = oe.spdx30.Agent() + else: + bb.fatal("Unknown agent type '%s' in %s_type" % (agent_type, varname)) + + agent._id = spdxid + agent.creationInfo = creation_info or self.doc.creationInfo + agent.name = name + + comment = self.d.getVar("%s_comment" % varname) + if comment: + agent.comment = comment + + for ( + pyname, + idtype, + ) in oe.spdx30.ExternalIdentifierType.NAMED_INDIVIDUALS.items(): + value = self.d.getVar("%s_id_%s" % (varname, pyname)) + if value: + agent.externalIdentifier.append( + oe.spdx30.ExternalIdentifier( + externalIdentifierType=idtype, + identifier=value, + ) + ) + + return self.add(agent) + + def new_creation_info(self): + creation_info = oe.spdx30.CreationInfo() + + name = "%s %s" % ( + self.d.getVar("SPDX_TOOL_NAME"), + self.d.getVar("SPDX_TOOL_VERSION"), + ) + tool = self.add( + oe.spdx30.Tool( + _id=self.new_spdxid("tool", name), + creationInfo=creation_info, + name=name, + ) + ) + + authors = [] + for a in self.d.getVar("SPDX_AUTHORS").split(): + varname = "SPDX_AUTHORS_%s" % a + author = self.new_agent(varname, creation_info=creation_info) + + if not author: + bb.fatal("Unable to find or create author %s" % a) + + authors.append(author) + + creation_info.created = spdx_now() + creation_info.specVersion = self.d.getVar("SPDX_VERSION") + creation_info.createdBy = authors + creation_info.createdUsing = [tool] + + return creation_info + + def copy_creation_info(self, copy): + c = oe.spdx30.CreationInfo( + created=spdx_now(), + specVersion=self.d.getVar("SPDX_VERSION"), + ) + + for author in copy.createdBy: + if isinstance(author, str): + c.createdBy.append(author) + else: + c.createdBy.append(author._id) + + for tool in copy.createdUsing: + if isinstance(tool, str): + c.createdUsing.append(tool) + else: + c.createdUsing.append(tool._id) + + return c + + def new_annotation(self, subject, comment, typ): + return self.add( + oe.spdx30.Annotation( + _id=self.new_spdxid("annotation", spdxid_hash(comment, typ)), + creationInfo=self.doc.creationInfo, + annotationType=typ, + subject=subject, + statement=comment, + ) + ) + + def _new_relationship( + self, + cls, + from_, + typ, + to, + *, + spdxid_name="relationship", + **props, + ): + from_ = to_list(from_) + to = to_list(to) + + if not from_: + return [] + + if not to: + # TODO: Switch to the code constant once SPDX 3.0.1 is released + to = ["https://spdx.org/rdf/3.0.0/terms/Core/NoneElement"] + + ret = [] + + for f in from_: + hash_args = [typ, f] + for k in sorted(props.keys()): + hash_args.append(props[k]) + hash_args.extend(to) + + relationship = self.add( + cls( + _id=self.new_spdxid(spdxid_name, spdxid_hash(*hash_args)), + creationInfo=self.doc.creationInfo, + from_=f, + relationshipType=typ, + to=to, + **props, + ) + ) + ret.append(relationship) + + return ret + + def new_relationship(self, from_, typ, to): + return self._new_relationship(oe.spdx30.Relationship, from_, typ, to) + + def new_scoped_relationship(self, from_, typ, scope, to): + return self._new_relationship( + oe.spdx30.LifecycleScopedRelationship, + from_, + typ, + to, + scope=scope, + ) + + def new_license_expression(self, license_expression, license_text_map={}): + license_list_version = self.d.getVar("SPDX_LICENSE_DATA")["licenseListVersion"] + # SPDX 3 requires that the license list version be a semver + # MAJOR.MINOR.MICRO, but the actual license version is only MAJOR.MINOR + # (as of this writing). As such, manually append a .0 micro version if + # its missing to keep SPDX happy + if license_list_version.count(".") < 3: + license_list_version += ".0" + + spdxid = [ + "license", + license_list_version, + re.sub(r"[^a-zA-Z0-9_-]", "_", license_expression), + ] + + license_text = ( + (k, license_text_map[k]) for k in sorted(license_text_map.keys()) + ) + + if not license_text: + lic = self.find_filter( + oe.spdx30.simplelicensing_LicenseExpression, + simplelicensing_licenseExpression=license_expression, + simplelicensing_licenseListVersion=license_list_version, + ) + if lic is not None: + return lic + else: + spdxid.append(spdxid_hash(*(v for _, v in license_text))) + lic = self.find_by_id(self.new_spdxid(*spdxid)) + if lic is not None: + return lic + + lic = self.add( + oe.spdx30.simplelicensing_LicenseExpression( + _id=self.new_spdxid(*spdxid), + creationInfo=self.doc.creationInfo, + simplelicensing_licenseExpression=license_expression, + simplelicensing_licenseListVersion=license_list_version, + ) + ) + + for key, value in license_text: + lic.simplelicensing_customIdToUri.append( + oe.spdx30.DictionaryEntry(key=key, value=value) + ) + + return lic + + def scan_declared_licenses(self, spdx_file, filepath): + for e in spdx_file.extension: + if isinstance(e, OELicenseScannedExtension): + return + + file_licenses = set() + for extracted_lic in extract_licenses(filepath): + file_licenses.add(self.new_license_expression(extracted_lic)) + + self.new_relationship( + [spdx_file], + oe.spdx30.RelationshipType.hasDeclaredLicense, + file_licenses, + ) + spdx_file.extension.append(OELicenseScannedExtension()) + + def new_file(self, _id, name, path, *, purposes=[]): + sha256_hash = bb.utils.sha256_file(path) + + for f in self.by_sha256_hash.get(sha256_hash, []): + if not isinstance(oe.spdx30.software_File): + continue + + if purposes: + new_primary = purposes[0] + new_additional = [] + + if f.software_primaryPurpose: + new_additional.append(f.software_primaryPurpose) + new_additional.extend(f.software_additionalPurpose) + + new_additional = sorted( + list(set(p for p in new_additional if p != new_primary)) + ) + + f.software_primaryPurpose = new_primary + f.software_additionalPurpose = new_additional + + if f.name != name: + for e in f.extension: + if isinstance(e, OEFileNameAliasExtension): + e.aliases.append(name) + break + else: + f.extension.append(OEFileNameAliasExtension(aliases=[name])) + + return f + + spdx_file = oe.spdx30.software_File( + _id=_id, + creationInfo=self.doc.creationInfo, + name=name, + ) + if purposes: + spdx_file.software_primaryPurpose = purposes[0] + spdx_file.software_additionalPurpose = purposes[1:] + + spdx_file.verifiedUsing.append( + oe.spdx30.Hash( + algorithm=oe.spdx30.HashAlgorithm.sha256, + hashValue=sha256_hash, + ) + ) + + return self.add(spdx_file) + + def new_cve_vuln(self, cve): + v = oe.spdx30.security_Vulnerability() + v._id = self.new_spdxid("vulnerability", cve) + v.creationInfo = self.doc.creationInfo + + v.externalIdentifier.append( + oe.spdx30.ExternalIdentifier( + externalIdentifierType=oe.spdx30.ExternalIdentifierType.cve, + identifier=cve, + identifierLocator=[ + f"https://cveawg.mitre.org/api/cve/{cve}", + f"https://www.cve.org/CVERecord?id={cve}", + ], + ) + ) + return self.add(v) + + def new_vex_patched_relationship(self, from_, to): + return self._new_relationship( + oe.spdx30.security_VexFixedVulnAssessmentRelationship, + from_, + oe.spdx30.RelationshipType.fixedIn, + to, + spdxid_name="vex-fixed", + security_vexVersion=VEX_VERSION, + ) + + def new_vex_unpatched_relationship(self, from_, to): + return self._new_relationship( + oe.spdx30.security_VexAffectedVulnAssessmentRelationship, + from_, + oe.spdx30.RelationshipType.affects, + to, + spdxid_name="vex-affected", + security_vexVersion=VEX_VERSION, + ) + + def new_vex_ignored_relationship(self, from_, to, *, impact_statement): + return self._new_relationship( + oe.spdx30.security_VexNotAffectedVulnAssessmentRelationship, + from_, + oe.spdx30.RelationshipType.doesNotAffect, + to, + spdxid_name="vex-not-affected", + security_vexVersion=VEX_VERSION, + security_impactStatement=impact_statement, + ) + + def import_bitbake_build(self): + def find_bitbake_build(objset): + return objset.find_filter( + oe.spdx30.build_Build, + build_buildType=SPDX_BUILD_TYPE, + ) + + build = find_bitbake_build(self) + if build: + return build + + deploy_dir_spdx = Path(self.d.getVar("DEPLOY_DIR_SPDX")) + bb_objset = load_jsonld( + self.d, deploy_dir_spdx / "bitbake.spdx.json", required=True + ) + + build = find_bitbake_build(bb_objset) + if build is None: + bb.fatal(f"No build found in {deploy_dir_spdx}") + + self.doc.imports.extend(bb_objset.doc.imports) + self.update(bb_objset.objects) + + return build + + def new_sub_build(self, name, typ): + build = self.add( + oe.spdx30.build_Build( + _id=self.new_spdxid("build", name), + creationInfo=self.doc.creationInfo, + name="%s-%s" % (self.d.getVar("PN"), name), + build_buildType="%s/%s" % (SPDX_BUILD_TYPE, typ), + ) + ) + + bitbake_build = self.import_bitbake_build() + + self.new_relationship( + [bitbake_build], + oe.spdx30.RelationshipType.ancestorOf, + [build], + ) + + if self.d.getVar("SPDX_INCLUDE_BUILD_VARIABLES") == "1": + for varname in sorted(self.d.keys()): + if varname.startswith("__"): + continue + + value = self.d.getVar(varname, expand=False) + + # TODO: Deal with non-string values + if not isinstance(value, str): + continue + + build.parameters.append( + oe.spdx30.DictionaryEntry(key=varname, value=value) + ) + + return build + + def new_archive(self, archive_name): + return self.add( + oe.spdx30.software_File( + _id=self.new_spdxid("archive", str(archive_name)), + creationInfo=self.doc.creationInfo, + name=str(archive_name), + software_primaryPurpose=oe.spdx30.software_SoftwarePurpose.archive, + ) + ) + + @classmethod + def new_objset(cls, d, name, copy_from_bitbake_doc=True): + objset = cls(d) + + document = oe.spdx30.SpdxDocument( + _id=objset.new_spdxid("document", name), + name=name, + ) + document.extension.append(OEIdAliasExtension()) + document.extension.append(OELinkExtension(link_spdx_id=False)) + objset.doc = document + + if copy_from_bitbake_doc: + build = objset.import_bitbake_build() + document.creationInfo = objset.copy_creation_info(build.creationInfo) + else: + document.creationInfo = objset.new_creation_info() + + return objset + + def expand_collection(self, *, add_objectsets=[]): + """ + Expands a collection to pull in all missing elements + + Returns the set of ids that could not be found to link into the document + """ + missing_spdxids = set() + imports = {e.externalSpdxId: e for e in self.doc.imports} + + def merge_doc(other): + nonlocal imports + + for e in other.doc.imports: + if not e.externalSpdxId in imports: + imports[e.externalSpdxId] = e + + self.objects |= other.objects + + for o in add_objectsets: + merge_doc(o) + + needed_spdxids = self.link() + provided_spdxids = set(self.obj_by_id.keys()) + + while True: + import_spdxids = set(imports.keys()) + searching_spdxids = ( + needed_spdxids - provided_spdxids - missing_spdxids - import_spdxids + ) + if not searching_spdxids: + break + + spdxid = searching_spdxids.pop() + bb.debug( + 1, + f"Searching for {spdxid}. Remaining: {len(searching_spdxids)}, Total: {len(provided_spdxids)}, Missing: {len(missing_spdxids)}, Imports: {len(import_spdxids)}", + ) + dep_objset, dep_path = find_by_spdxid(self.d, spdxid) + + if dep_objset: + dep_provided = set(dep_objset.obj_by_id.keys()) + if spdxid not in dep_provided: + bb.fatal(f"{spdxid} not found in {dep_path}") + provided_spdxids |= dep_provided + needed_spdxids |= dep_objset.missing_ids + merge_doc(dep_objset) + else: + missing_spdxids.add(spdxid) + + bb.debug(1, "Linking...") + missing = self.link() + if missing != missing_spdxids: + bb.fatal( + f"Linked document doesn't match missing SPDX ID list. Got: {missing}\nExpected: {missing_spdxids}" + ) + + self.doc.imports = sorted(imports.values(), key=lambda e: e.externalSpdxId) + + return missing_spdxids + + +def load_jsonld(d, path, required=False): + deserializer = oe.spdx30.JSONLDDeserializer() + objset = ObjectSet(d) + try: + with path.open("rb") as f: + deserializer.read(f, objset) + except FileNotFoundError: + if required: + bb.fatal("No SPDX document named %s found" % path) + return None + + if not objset.doc: + bb.fatal("SPDX Document %s has no SPDXDocument element" % path) + return None + + objset.objects.remove(objset.doc) + return objset + + +def jsonld_arch_path(d, arch, subdir, name, deploydir=None): + if deploydir is None: + deploydir = Path(d.getVar("DEPLOY_DIR_SPDX")) + return deploydir / arch / subdir / (name + ".spdx.json") + + +def jsonld_hash_path(_id): + h = hashlib.sha256(_id.encode("utf-8")).hexdigest() + + return Path("by-spdxid-hash") / h[:2], h + + +def load_jsonld_by_arch(d, arch, subdir, name, *, required=False): + path = jsonld_arch_path(d, arch, subdir, name) + objset = load_jsonld(d, path, required=required) + if objset is not None: + return (objset, path) + return (None, None) + + +def find_jsonld(d, subdir, name, *, required=False): + package_archs = d.getVar("SSTATE_ARCHS").split() + package_archs.reverse() + + for arch in package_archs: + objset, path = load_jsonld_by_arch(d, arch, subdir, name) + if objset is not None: + return (objset, path) + + if required: + bb.fatal("Could not find a %s SPDX document named %s" % (subdir, name)) + + return (None, None) + + +def write_jsonld_doc(d, objset, dest): + if not isinstance(objset, ObjectSet): + bb.fatal("Only an ObjsetSet can be serialized") + return + + if not objset.doc: + bb.fatal("ObjectSet is missing a SpdxDocument") + return + + objset.doc.rootElement = sorted(list(set(objset.doc.rootElement))) + objset.doc.profileConformance = sorted( + list( + getattr(oe.spdx30.ProfileIdentifierType, p) + for p in d.getVar("SPDX_PROFILES").split() + ) + ) + + dest.parent.mkdir(exist_ok=True, parents=True) + + if d.getVar("SPDX_PRETTY") == "1": + serializer = oe.spdx30.JSONLDSerializer( + indent=2, + ) + else: + serializer = oe.spdx30.JSONLDInlineSerializer() + + objset.objects.add(objset.doc) + with dest.open("wb") as f: + serializer.write(objset, f, force_at_graph=True) + objset.objects.remove(objset.doc) + + +def write_recipe_jsonld_doc( + d, + objset, + subdir, + deploydir, + *, + create_spdx_id_links=True, +): + pkg_arch = d.getVar("SSTATE_PKGARCH") + + dest = jsonld_arch_path(d, pkg_arch, subdir, objset.doc.name, deploydir=deploydir) + + def link_id(_id): + hash_path = jsonld_hash_path(_id) + + link_name = jsonld_arch_path( + d, + pkg_arch, + *hash_path, + deploydir=deploydir, + ) + try: + link_name.parent.mkdir(exist_ok=True, parents=True) + link_name.symlink_to(os.path.relpath(dest, link_name.parent)) + except: + target = link_name.readlink() + bb.warn( + f"Unable to link {_id} in {dest} as {link_name}. Already points to {target}" + ) + raise + + return hash_path[-1] + + objset.add_aliases() + + try: + if create_spdx_id_links: + for o in objset.foreach_type(oe.spdx30.Element): + if not o._id or o._id.startswith("_:"): + continue + + ext = None + for e in o.extension: + if not isinstance(e, OELinkExtension): + continue + + ext = e + break + + if ext is None: + ext = OELinkExtension(link_spdx_id=True) + o.extension.append(ext) + + if ext.link_spdx_id: + ext.link_name = link_id(o._id) + + alias_ext = get_alias(o) + if alias_ext is not None and alias_ext.alias: + alias_ext.link_name = link_id(alias_ext.alias) + + finally: + # It is really helpful for debugging if the JSON document is written + # out, so always do that even if there is an error making the links + write_jsonld_doc(d, objset, dest) + + +def find_root_obj_in_jsonld(d, subdir, fn_name, obj_type, **attr_filter): + objset, fn = find_jsonld(d, subdir, fn_name, required=True) + + spdx_obj = objset.find_root(obj_type, **attr_filter) + if not spdx_obj: + bb.fatal("No root %s found in %s" % (obj_type.__name__, fn)) + + return spdx_obj, objset + + +def load_obj_in_jsonld(d, arch, subdir, fn_name, obj_type, **attr_filter): + objset, fn = load_jsonld_by_arch(d, arch, subdir, fn_name, required=True) + + spdx_obj = objset.find_filter(obj_type, **attr_filter) + if not spdx_obj: + bb.fatal("No %s found in %s" % (obj_type.__name__, fn)) + + return spdx_obj, objset + + +def find_by_spdxid(d, spdxid, *, required=False): + return find_jsonld(d, *jsonld_hash_path(spdxid), required=required) + + +def create_sbom(d, name, root_elements, add_objectsets=[]): + objset = ObjectSet.new_objset(d, name) + + sbom = objset.add( + oe.spdx30.software_Sbom( + _id=objset.new_spdxid("sbom", name), + name=name, + creationInfo=objset.doc.creationInfo, + software_sbomType=[oe.spdx30.software_SbomType.build], + rootElement=root_elements, + ) + ) + + missing_spdxids = objset.expand_collection(add_objectsets=add_objectsets) + if missing_spdxids: + bb.warn( + "The following SPDX IDs were unable to be resolved:\n " + + "\n ".join(sorted(list(missing_spdxids))) + ) + + # Filter out internal extensions from final SBoMs + objset.remove_internal_extensions() + + # SBoM should be the only root element of the document + objset.doc.rootElement = [sbom] + + # De-duplicate licenses + unique = set() + dedup = {} + for lic in objset.foreach_type(oe.spdx30.simplelicensing_LicenseExpression): + for u in unique: + if ( + u.simplelicensing_licenseExpression + == lic.simplelicensing_licenseExpression + and u.simplelicensing_licenseListVersion + == lic.simplelicensing_licenseListVersion + ): + dedup[lic] = u + break + else: + unique.add(lic) + + if dedup: + for rel in objset.foreach_filter( + oe.spdx30.Relationship, + relationshipType=oe.spdx30.RelationshipType.hasDeclaredLicense, + ): + rel.to = [dedup.get(to, to) for to in rel.to] + + for rel in objset.foreach_filter( + oe.spdx30.Relationship, + relationshipType=oe.spdx30.RelationshipType.hasConcludedLicense, + ): + rel.to = [dedup.get(to, to) for to in rel.to] + + for k, v in dedup.items(): + bb.debug(1, f"Removing duplicate License {k._id} -> {v._id}") + objset.objects.remove(k) + + objset.create_index() + + return objset, sbommeta/lib/oe/spdx30.py new file +5996
-1@@ -0,0 +1,5996 @@ +#! /usr/bin/env python3 +# +# Generated Python bindings from a SHACL model +# +# This file was automatically generated by shacl2code. DO NOT MANUALLY MODIFY IT +# +# SPDX-License-Identifier: MIT + +import functools +import hashlib +import json +import re +import sys +import threading +import time +from contextlib import contextmanager +from datetime import datetime, timezone, timedelta +from enum import Enum +from abc import ABC, abstractmethod + + +def check_type(obj, types): + if not isinstance(obj, types): + if isinstance(types, (list, tuple)): + raise TypeError( + f"Value must be one of type: {', '.join(t.__name__ for t in types)}. Got {type(obj)}" + ) + raise TypeError(f"Value must be of type {types.__name__}. Got {type(obj)}") + + +class Property(ABC): + """ + A generic SHACL object property. The different types will derive from this + class + """ + + def __init__(self, *, pattern=None): + self.pattern = pattern + + def init(self): + return None + + def validate(self, value): + check_type(value, self.VALID_TYPES) + if self.pattern is not None and not re.search( + self.pattern, self.to_string(value) + ): + raise ValueError( + f"Value is not correctly formatted. Got '{self.to_string(value)}'" + ) + + def set(self, value): + return value + + def check_min_count(self, value, min_count): + return min_count == 1 + + def check_max_count(self, value, max_count): + return max_count == 1 + + def elide(self, value): + return value is None + + def walk(self, value, callback, path): + callback(value, path) + + def iter_objects(self, value, recursive, visited): + return [] + + def link_prop(self, value, objectset, missing, visited): + return value + + def to_string(self, value): + return str(value) + + @abstractmethod + def encode(self, encoder, value, state): + pass + + @abstractmethod + def decode(self, decoder, *, objectset=None): + pass + + +class StringProp(Property): + """ + A scalar string property for an SHACL object + """ + + VALID_TYPES = str + + def set(self, value): + return str(value) + + def encode(self, encoder, value, state): + encoder.write_string(value) + + def decode(self, decoder, *, objectset=None): + return decoder.read_string() + + +class AnyURIProp(StringProp): + def encode(self, encoder, value, state): + encoder.write_iri(value) + + def decode(self, decoder, *, objectset=None): + return decoder.read_iri() + + +class DateTimeProp(Property): + """ + A Date/Time Object with optional timezone + """ + + VALID_TYPES = datetime + UTC_FORMAT_STR = "%Y-%m-%dT%H:%M:%SZ" + REGEX = r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(Z|[+-]\d{2}:\d{2})?$" + + def set(self, value): + return self._normalize(value) + + def encode(self, encoder, value, state): + encoder.write_datetime(self.to_string(value)) + + def decode(self, decoder, *, objectset=None): + s = decoder.read_datetime() + if s is None: + return None + v = self.from_string(s) + return self._normalize(v) + + def _normalize(self, value): + if value.utcoffset() is None: + value = value.astimezone() + offset = value.utcoffset() + if offset % timedelta(minutes=1): + offset = offset - (offset % timedelta(minutes=1)) + value = value.replace(tzinfo=timezone(offset)) + value = value.replace(microsecond=0) + return value + + def to_string(self, value): + value = self._normalize(value) + if value.tzinfo == timezone.utc: + return value.strftime(self.UTC_FORMAT_STR) + return value.isoformat() + + def from_string(self, value): + if not re.match(self.REGEX, value): + raise ValueError(f"'{value}' is not a correctly formatted datetime") + if "Z" in value: + d = datetime( + *(time.strptime(value, self.UTC_FORMAT_STR)[0:6]), + tzinfo=timezone.utc, + ) + else: + d = datetime.fromisoformat(value) + + return self._normalize(d) + + +class DateTimeStampProp(DateTimeProp): + """ + A Date/Time Object with required timestamp + """ + + REGEX = r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(Z|[+-]\d{2}:\d{2})$" + + +class IntegerProp(Property): + VALID_TYPES = int + + def set(self, value): + return int(value) + + def encode(self, encoder, value, state): + encoder.write_integer(value) + + def decode(self, decoder, *, objectset=None): + return decoder.read_integer() + + +class PositiveIntegerProp(IntegerProp): + def validate(self, value): + super().validate(value) + if value < 1: + raise ValueError(f"Value must be >=1. Got {value}") + + +class NonNegativeIntegerProp(IntegerProp): + def validate(self, value): + super().validate(value) + if value < 0: + raise ValueError(f"Value must be >= 0. Got {value}") + + +class BooleanProp(Property): + VALID_TYPES = bool + + def set(self, value): + return bool(value) + + def encode(self, encoder, value, state): + encoder.write_bool(value) + + def decode(self, decoder, *, objectset=None): + return decoder.read_bool() + + +class FloatProp(Property): + VALID_TYPES = (float, int) + + def set(self, value): + return float(value) + + def encode(self, encoder, value, state): + encoder.write_float(value) + + def decode(self, decoder, *, objectset=None): + return decoder.read_float() + + +class ObjectProp(Property): + """ + A scalar SHACL object property of a SHACL object + """ + + def __init__(self, cls, required): + super().__init__() + self.cls = cls + self.required = required + + def init(self): + if self.required and not self.cls.IS_ABSTRACT: + return self.cls() + return None + + def validate(self, value): + check_type(value, (self.cls, str)) + + def walk(self, value, callback, path): + if value is None: + return + + if not isinstance(value, str): + value.walk(callback, path) + else: + callback(value, path) + + def iter_objects(self, value, recursive, visited): + if value is None or isinstance(value, str): + return + + if value not in visited: + visited.add(value) + yield value + + if recursive: + for c in value.iter_objects(recursive=True, visited=visited): + yield c + + def encode(self, encoder, value, state): + if value is None: + raise ValueError("Object cannot be None") + + if isinstance(value, str): + value = _NI_ENCODE_CONTEXT.get(value, value) + encoder.write_iri(value) + return + + return value.encode(encoder, state) + + def decode(self, decoder, *, objectset=None): + iri = decoder.read_iri() + if iri is None: + return self.cls.decode(decoder, objectset=objectset) + + iri = _NI_DECODE_CONTEXT.get(iri, iri) + + if objectset is None: + return iri + + obj = objectset.find_by_id(iri) + if obj is None: + return iri + + self.validate(obj) + return obj + + def link_prop(self, value, objectset, missing, visited): + if value is None: + return value + + if isinstance(value, str): + o = objectset.find_by_id(value) + if o is not None: + self.validate(o) + return o + + if missing is not None: + missing.add(value) + + return value + + # De-duplicate IDs + if value._id: + value = objectset.find_by_id(value._id, value) + self.validate(value) + + value.link_helper(objectset, missing, visited) + return value + + +class ListProxy(object): + def __init__(self, prop, data=None): + if data is None: + self.__data = [] + else: + self.__data = data + self.__prop = prop + + def append(self, value): + self.__prop.validate(value) + self.__data.append(self.__prop.set(value)) + + def insert(self, idx, value): + self.__prop.validate(value) + self.__data.insert(idx, self.__prop.set(value)) + + def extend(self, items): + for i in items: + self.append(i) + + def sort(self, *args, **kwargs): + self.__data.sort(*args, **kwargs) + + def __getitem__(self, key): + return self.__data[key] + + def __setitem__(self, key, value): + if isinstance(key, slice): + for v in value: + self.__prop.validate(v) + self.__data[key] = [self.__prop.set(v) for v in value] + else: + self.__prop.validate(value) + self.__data[key] = self.__prop.set(value) + + def __delitem__(self, key): + del self.__data[key] + + def __contains__(self, item): + return item in self.__data + + def __iter__(self): + return iter(self.__data) + + def __len__(self): + return len(self.__data) + + def __str__(self): + return str(self.__data) + + def __repr__(self): + return repr(self.__data) + + def __eq__(self, other): + if isinstance(other, ListProxy): + return self.__data == other.__data + + return self.__data == other + + +class ListProp(Property): + """ + A list of SHACL properties + """ + + VALID_TYPES = (list, ListProxy) + + def __init__(self, prop): + super().__init__() + self.prop = prop + + def init(self): + return ListProxy(self.prop) + + def validate(self, value): + super().validate(value) + + for i in value: + self.prop.validate(i) + + def set(self, value): + if isinstance(value, ListProxy): + return value + + return ListProxy(self.prop, [self.prop.set(d) for d in value]) + + def check_min_count(self, value, min_count): + check_type(value, ListProxy) + return len(value) >= min_count + + def check_max_count(self, value, max_count): + check_type(value, ListProxy) + return len(value) <= max_count + + def elide(self, value): + check_type(value, ListProxy) + return len(value) == 0 + + def walk(self, value, callback, path): + callback(value, path) + for idx, v in enumerate(value): + self.prop.walk(v, callback, path + [f"[{idx}]"]) + + def iter_objects(self, value, recursive, visited): + for v in value: + for c in self.prop.iter_objects(v, recursive, visited): + yield c + + def link_prop(self, value, objectset, missing, visited): + if isinstance(value, ListProxy): + data = [self.prop.link_prop(v, objectset, missing, visited) for v in value] + else: + data = [self.prop.link_prop(v, objectset, missing, visited) for v in value] + + return ListProxy(self.prop, data=data) + + def encode(self, encoder, value, state): + check_type(value, ListProxy) + + with encoder.write_list() as list_s: + for v in value: + with list_s.write_list_item() as item_s: + self.prop.encode(item_s, v, state) + + def decode(self, decoder, *, objectset=None): + data = [] + for val_d in decoder.read_list(): + v = self.prop.decode(val_d, objectset=objectset) + self.prop.validate(v) + data.append(v) +Lines of this file are not shown. View the raw message.
The rest of this diff is not shown. View the raw message.
Links: You receive all messages sent to this group.
View/Reply Online (#201106): https://lists.openembedded.org/g/openembedded-core/message/201106
Mute This Topic: https://lists.openembedded.org/mt/106856878/3616948
Group Owner: [email protected]
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [[email protected]]
-=-=-=-=-=-=-=-=-=-=-=-
Part of the diff is not shown here. View the raw message.
Re: [OE-core][PATCH v4 03/10] classes/create-spdx-3.0: Add classes
Quoted 4 lines · Mark Hatle
Comments inline below
On 6/24/24 2:10 PM, Joshua Watt wrote:
-- snip --
Quoted 7 lines
meta/classes/create-spdx-3.0.bbclass line 65
Quoted 3 lines
meta/classes/create-spdx-3.0.bbclass new file +3
-0+ +SPDX_BUILD_HOST[doc] = "The base variable name to describe the build host on \ + which a build is running. Must be an SPDX_IMPORTS key"Is there any sort of documentation or external reference for the variable above (as well as the
SPDX_below) that explains what the SPDX standard is expecting to be put in there?
Not specifically for this, but for the SPDX 3.0 spec in general, the web docs are pretty comprehensive:
https://spdx.github.io/spdx-spec/v3.0/ . Although, be aware the navigation sidebar is really annoying ATM, but that's supposed to get fixed soon.
For a starter on how SPDX documents are written, see:
https://github.com/spdx/spdx-spec/blob/development/v3.0.1/docs/annexes/getting-started.md
It's a little tricky to encode the SPDX 3 structured data in bitbake variables; this is what I could come up with so far but if you have suggestions on improvements, let me know.
Specifically for this variable, it's referencing a "key" in SPDX_IMPORTS. SPDX_IMPORTS in turn is encoding entries in the "imports" property of
https://spdx.github.io/spdx-spec/v3.0/model/Core/Classes/SpdxDocument/
. The indirection is necessary so that you can tell users where the SPDX ID lives, since it's external to this document. It's pretty much impossible for us to validate that the SPDX ID you put in is real, short of downloading the referenced document, parsing it, and seeing if the SPDX ID is present.
For a walk through of how to cross-link SPDX 3 documents, look at:
https://github.com/spdx/spdx-spec/blob/development/v3.0.1/docs/annexes/cross-reference.md
Quoted 4 lines
I.e. machine name, host type, an arbitrary string that means something to the agent construction the SPDX, etc. I.e. how would I know what is valid in these various things?
There are pretty good tools to validate SPDX documents (offline
even!). https://github.com/spdx/spdx-3-model/blob/main/serialization/json_ld/validation.md
gives an overview on how to do this. It still won't do the validation
that the external SPDX ID is valid for the same reasons as above, but
it's pretty good otherwise.
Quoted 6 lines
This then leads to a second question, deterministic behavior/reproducibility. I believe the purpose of this is reproducible builds, but we should have a more deterministic approach in the Yocto Project where we provide (and/or check for host capabilities) to help allow this to be a more generic, many different host process.
I've attempted to make this process as deterministic as SPDX 3 allows. As an example, SPDX IDs are generated by hashing deterministic data (where as random SPDX IDs would be decidedly simpler!). However, there are parts of SPDX 3 that are simply not deterministic for various reasons (please read to the end of this section). They generally fall under a few categories:
The first category is probably best classified as "not very useful if deterministic" and omitted by default; SPDX_BUILD_HOST would be one such examples, since if you are going to set this to the same value all the time and not reference the _actual_ host, you may as well not include it at all. For these, I've set no default value (there isn't one that would make sense anyway), so their omission keeps things deterministic. However users do actually want this, it will necessarily result in non-deterministic builds unless you always do your builds on the same host. I think it might be helpful to annotate in the doc string which variables will introduce such non-determinism, so I'll do that.
The second category is "not very useful if deterministic", but are included in the output by this patch. Examples of this would be build timestamps and the bitbake parent build tracking (which basically tracks the invocation of bitbake itself as the "parent" build, so you can tell which tasks ran in the same invocation). These are useful pieces of information, and consumers do actually care about these things, so if push comes to shove we could add a flag to enable them, but I'm also leery of having too many configuration options for SPDX.
The last category are the require non-deterministic fields in SPDX 3. The primary offender here is the SPDX creation info "created"
datestamp: https://spdx.github.io/spdx-spec/v3.0/model/Core/Classes/CreationInfo/
. This is a mandatory field that is the timestamp of when the SPDX data itself was created, and every SPDX object you create links to this so you can track exactly when each object in a merged document was created. I did attempt to make the argument to SPDX that it was mandatory non-determinism, but it is very important to the SPDX community (for reasons I've not fully understood), and they _really_ want it to be the actual document creation date, not SOURCE_DATE_EPOCH or similar, so I really am not sure what to do about that one. I was more or less told to ignore these fields when calculating if output is "deterministic", which is a little annoying, but not an argument I could win.
I'm a little stuck between the SPDX side and the Yocto side on the determinism front. It's easy to say "SPDX must be deterministic", and "it's awful if non-deterministic" if your looking at it from just the point of view of you want to spit out some data, but equally, it's easy to say "determinism doesn't really matter" and "the non-deterministic data is important information" when you are consuming the data. This particular patch series errs on the side of making the data the most useful to the end consumers, in part because I really want Yocto to generate the most comprehensive and useful output it can; we are a pretty early adopter of SPDX 3, so being able to provide the most useful data we can early can means consumers writing downstream tools can use us as a reference which drastically improves their compatibility with our output. Yocto has a phenomenal supply chain story to tell, and I really want to tell it to the fullest extent that we can in our SPDX data. I can't reconcile that with "everything must be deterministic" though, so..... ?
Quoted 4 lines
Which leads to a third question, when a build uses sstate-cache, each host could be different then the build host that actually combines the builds into an image (SBOM). Is this a concern?
That is very much on purpose so you can track where your sstate came from, who built it, etc, but also why we don't set any of these by default (see above). This is pretty important from a supply chain tracking perspective, and part of the comprehensive story we can tell about the supply chain (IMHO anyway).
Currently, we are only tracking the actual do_create_spdx() task as the "build" (which isn't clear in the generated SPDX, but I'll make change to fix that), so you have to be a little bit careful about some of the conclusions you draw from that. Once we get the base SPDX 3 support in place, I want to look at having other sstate tasks generate SPDX fragements when they run which would allow us to trace those tasks more precisely. I don't really want to solve that now though as it is going to be quite a bit more complex to solve in a satisfactory manner.
Quoted 1 line
-- snip --
Re: [OE-core][PATCH v4 03/10] classes/create-spdx-3.0: Add classes
Quoted 88 lines · Joshua Watt
Quoted 4 lines · Mark Hatle
Comments inline below
On 6/24/24 2:10 PM, Joshua Watt wrote:
-- snip --
Quoted 7 lines
meta/classes/create-spdx-3.0.bbclass line 65
Quoted 3 lines
meta/classes/create-spdx-3.0.bbclass new file +3
-0+ +SPDX_BUILD_HOST[doc] = "The base variable name to describe the build host on \ + which a build is running. Must be an SPDX_IMPORTS key"Is there any sort of documentation or external reference for the variable above (as well as the
SPDX_below) that explains what the SPDX standard is expecting to be put in there?Not specifically for this, but for the SPDX 3.0 spec in general, the web docs are pretty comprehensive:
https://spdx.github.io/spdx-spec/v3.0/ . Although, be aware the navigation sidebar is really annoying ATM, but that's supposed to get fixed soon.For a starter on how SPDX documents are written, see:
https://github.com/spdx/spdx-spec/blob/development/v3.0.1/docs/annexes/getting-started.mdIt's a little tricky to encode the SPDX 3 structured data in bitbake variables; this is what I could come up with so far but if you have suggestions on improvements, let me know.
Specifically for this variable, it's referencing a "key" in
SPDX_IMPORTS.SPDX_IMPORTSin turn is encoding entries in the "imports" property of
https://spdx.github.io/spdx-spec/v3.0/model/Core/Classes/SpdxDocument/
. The indirection is necessary so that you can tell users where the SPDX ID lives, since it's external to this document. It's pretty much impossible for us to validate that the SPDX ID you put in is real, short of downloading the referenced document, parsing it, and seeing if the SPDX ID is present.For a walk through of how to cross-link SPDX 3 documents, look at:
https://github.com/spdx/spdx-spec/blob/development/v3.0.1/docs/annexes/cross-reference.mdQuoted 4 lines
I.e. machine name, host type, an arbitrary string that means something to the agent construction the SPDX, etc. I.e. how would I know what is valid in these various things?
There are pretty good tools to validate SPDX documents (offline
even!). https://github.com/spdx/spdx-3-model/blob/main/serialization/json_ld/validation.md
gives an overview on how to do this. It still won't do the validation
that the external SPDX ID is valid for the same reasons as above, but
it's pretty good otherwise.Quoted 6 lines
This then leads to a second question, deterministic behavior/reproducibility. I believe the purpose of this is reproducible builds, but we should have a more deterministic approach in the Yocto Project where we provide (and/or check for host capabilities) to help allow this to be a more generic, many different host process.
I've attempted to make this process as deterministic as SPDX 3 allows. As an example, SPDX IDs are generated by hashing deterministic data (where as random SPDX IDs would be decidedly simpler!). However, there are parts of SPDX 3 that are simply not deterministic for various reasons (please read to the end of this section). They generally fall under a few categories:
The first category is probably best classified as "not very useful if deterministic" and omitted by default;
SPDX_BUILD_HOSTwould be one such examples, since if you are going to set this to the same value all the time and not reference the _actual_ host, you may as well not include it at all. For these, I've set no default value (there isn't one that would make sense anyway), so their omission keeps things deterministic. However users do actually want this, it will necessarily result in non-deterministic builds unless you always do your builds on the same host. I think it might be helpful to annotate in the doc string which variables will introduce such non-determinism, so I'll do that.The second category is "not very useful if deterministic", but are included in the output by this patch. Examples of this would be build timestamps and the bitbake parent build tracking (which basically tracks the invocation of bitbake itself as the "parent" build, so you can tell which tasks ran in the same invocation). These are useful pieces of information, and consumers do actually care about these things, so if push comes to shove we could add a flag to enable them, but I'm also leery of having too many configuration options for SPDX.
Quoted 14 lines
The last category are the require non-deterministic fields in SPDX 3. The primary offender here is the SPDX creation info "created"
datestamp: https://spdx.github.io/spdx-spec/v3.0/model/Core/Classes/CreationInfo/
. This is a mandatory field that is the timestamp of when the SPDX data itself was created, and every SPDX object you create links to this so you can track exactly when each object in a merged document was created. I did attempt to make the argument to SPDX that it was mandatory non-determinism, but it is very important to the SPDX community (for reasons I've not fully understood), and they _really_ want it to be the actual document creation date, notSOURCE_DATE_EPOCHor similar, so I really am not sure what to do about that one. I was more or less told to ignore these fields when calculating if output is "deterministic", which is a little annoying, but not an argument I could win.
Heh, of course I just read the spec a little more carefully, and SOURCE_DATE_EPOCH is allowed here, so we can do that and make everything deterministic by default, with options to enable non-determinism.
https://spdx.github.io/spdx-spec/v3.0/model/Core/Properties/created/
*facepalm*
Quoted 42 lines
I'm a little stuck between the SPDX side and the Yocto side on the determinism front. It's easy to say "SPDX must be deterministic", and "it's awful if non-deterministic" if your looking at it from just the point of view of you want to spit out some data, but equally, it's easy to say "determinism doesn't really matter" and "the non-deterministic data is important information" when you are consuming the data. This particular patch series errs on the side of making the data the most useful to the end consumers, in part because I really want Yocto to generate the most comprehensive and useful output it can; we are a pretty early adopter of SPDX 3, so being able to provide the most useful data we can early can means consumers writing downstream tools can use us as a reference which drastically improves their compatibility with our output. Yocto has a phenomenal supply chain story to tell, and I really want to tell it to the fullest extent that we can in our SPDX data. I can't reconcile that with "everything must be deterministic" though, so..... ?
Quoted 4 lines
Which leads to a third question, when a build uses sstate-cache, each host could be different then the build host that actually combines the builds into an image (SBOM). Is this a concern?
That is very much on purpose so you can track where your sstate came from, who built it, etc, but also why we don't set any of these by default (see above). This is pretty important from a supply chain tracking perspective, and part of the comprehensive story we can tell about the supply chain (IMHO anyway).
Currently, we are only tracking the actual
do_create_spdx()task as the "build" (which isn't clear in the generated SPDX, but I'll make change to fix that), so you have to be a little bit careful about some of the conclusions you draw from that. Once we get the base SPDX 3 support in place, I want to look at having other sstate tasks generate SPDX fragements when they run which would allow us to trace those tasks more precisely. I don't really want to solve that now though as it is going to be quite a bit more complex to solve in a satisfactory manner.Quoted 1 line
-- snip --
PATCH v4 4/10 classes-global/staging: Exclude do_create_spdx from automatic sysroot extension
In reply to Joshua Wattdo_create_spdx is a outlier in that it doesn't need the RSS to be extended just because it depends on do_populate_sysroot. In fact, it only depends on do_populate_sysroot so it can see the actual recipes sysroot, and attempting to extend the sysroot can cause problems for some recipes (e.g. if a recipe does do_populate_sysroot[noexec] = "1")
As such, explicitly exclude do_create_spdx from extending the sysroot just because it depends on do_populate_sysroot.
--- meta/classes-global/staging.bbclass | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-)
meta/classes-global/staging.bbclass +8 -1
@@ -652,10 +652,17 @@ python do_prepare_recipe_sysroot () { addtask do_prepare_recipe_sysroot before do_configure after do_fetch python staging_taskhandler() { + EXCLUDED_TASKS = ( + "do_prepare_recipe_sysroot", + "do_create_spdx", + ) bbtasks = e.tasklist for task in bbtasks: + if task in EXCLUDED_TASKS: + continue + deps = d.getVarFlag(task, "depends") - if task != 'do_prepare_recipe_sysroot' and (task == "do_configure" or (deps and "populate_sysroot" in deps)): + if task == "do_configure" or (deps and "populate_sysroot" in deps): d.prependVarFlag(task, "prefuncs", "extend_recipe_sysroot ") } staging_taskhandler[eventmask] = "bb.event.RecipeTaskPreProcess"
Signature · 1 line
2.43.2
PATCH v4 5/10 binutils-cross-testsuite: Rename to binutils-testsuite
In reply to Joshua WattThis recipe needs to be renamed because the "-cross-" substring in the name triggers the cross architecture detection in sstate, but this recipe is not actually a cross recipe.
---
meta/classes-global/mirrors.bbclass | 2 +-
meta/conf/distro/include/maintainers.inc | 2 +-
meta/lib/oeqa/selftest/cases/binutils.py | 2 +-
...tils-cross-testsuite_2.42.bb => binutils-testsuite_2.42.bb} | 3 +++
4 files changed, 6 insertions(+), 3 deletions(-)
rename meta/recipes-devtools/binutils/{binutils-cross-testsuite_2.42.bb => binutils-testsuite_2.42.bb} (95%)meta/classes-global/mirrors.bbclass +1 -1
@@ -88,7 +88,7 @@ git://.*/.* git://HOST/git/PATH;protocol=https \ BB_GIT_SHALLOW:pn-binutils = "1" BB_GIT_SHALLOW:pn-binutils-cross-${TARGET_ARCH} = "1" BB_GIT_SHALLOW:pn-binutils-cross-canadian-${TRANSLATED_TARGET_ARCH} = "1" -BB_GIT_SHALLOW:pn-binutils-cross-testsuite = "1" +BB_GIT_SHALLOW:pn-binutils-testsuite = "1" BB_GIT_SHALLOW:pn-binutils-crosssdk-${SDK_SYS} = "1" BB_GIT_SHALLOW:pn-binutils-native = "1" BB_GIT_SHALLOW:pn-nativesdk-binutils = "1"
meta/conf/distro/include/maintainers.inc +1 -1
@@ -64,7 +64,7 @@ RECIPE_MAINTAINER:pn-bind = "Unassigned <[email protected]>" RECIPE_MAINTAINER:pn-binutils = "Khem Raj <[email protected]>" RECIPE_MAINTAINER:pn-binutils-cross-${TARGET_ARCH} = "Khem Raj <[email protected]>" RECIPE_MAINTAINER:pn-binutils-cross-canadian-${TRANSLATED_TARGET_ARCH} = "Khem Raj <[email protected]>" -RECIPE_MAINTAINER:pn-binutils-cross-testsuite = "Khem Raj <[email protected]>" +RECIPE_MAINTAINER:pn-binutils-testsuite = "Khem Raj <[email protected]>" RECIPE_MAINTAINER:pn-binutils-crosssdk-${SDK_SYS} = "Khem Raj <[email protected]>" RECIPE_MAINTAINER:pn-bison = "Chen Qi <[email protected]>" RECIPE_MAINTAINER:pn-blktool = "Yi Zhao <[email protected]>"
meta/lib/oeqa/selftest/cases/binutils.py +1 -1
@@ -33,7 +33,7 @@ class BinutilsCrossSelfTest(OESelftestTestCase, OEPTestResultTestCase): features.append('CHECK_TARGETS = "{0}"'.format(suite)) self.write_config("\n".join(features)) - recipe = "binutils-cross-testsuite" + recipe = "binutils-testsuite" bb_vars = get_bb_vars(["B", "TARGET_SYS", "T"], recipe) builddir, target_sys, tdir = bb_vars["B"], bb_vars["TARGET_SYS"], bb_vars["T"]
meta/recipes-devtools/binutils/binutils-testsuite_2.42.bb renamed from meta/recipes-devtools/binutils/binutils-cross-testsuite_2.42.bb +3 -0
@@ -1,3 +1,6 @@ +# NOTE: This recipe cannot have -cross- in the file name because it triggers +# the cross build detection in sstate which causes it to use the wrong +# architecture require binutils.inc require binutils-${PV}.inc
Signature · 1 line
2.43.2
PATCH v4 6/10 classes-recipe/image_types: Add SPDX_IMAGE_PURPOSE to images
In reply to Joshua WattAdds the variable overrides to set the SPDX image purpose for various image types
--- meta/classes-recipe/image_types.bbclass | 2 ++ meta/classes-recipe/image_types_wic.bbclass | 1 + 2 files changed, 3 insertions(+)
meta/classes-recipe/image_types.bbclass +2 -0
@@ -146,6 +146,7 @@ IMAGE_CMD:vfat = "oe_mkvfatfs ${EXTRA_IMAGECMD}" IMAGE_CMD_TAR ?= "tar" # ignore return code 1 "file changed as we read it" as other tasks(e.g. do_image_wic) may be hardlinking rootfs IMAGE_CMD:tar = "${IMAGE_CMD_TAR} --sort=name --format=posix --numeric-owner -cf ${IMGDEPLOYDIR}/${IMAGE_NAME}.tar -C ${IMAGE_ROOTFS} . || [ $? -eq 1 ]" +SPDX_IMAGE_PURPOSE:tar = "archive" do_image_cpio[cleandirs] += "${WORKDIR}/cpio_append" IMAGE_CMD:cpio () {
@@ -167,6 +168,7 @@ IMAGE_CMD:cpio () { fi fi } +SPDX_IMAGE_PURPOSE:cpio = "archive" UBI_VOLNAME ?= "${MACHINE}-rootfs" UBI_VOLTYPE ?= "dynamic"
meta/classes-recipe/image_types_wic.bbclass +1 -0
@@ -91,6 +91,7 @@ IMAGE_CMD:wic () { mv "$build_wic/$(basename "${wks%.wks}")"*.${IMAGER} "$out.wic" } IMAGE_CMD:wic[vardepsexclude] = "WKS_FULL_PATH WKS_FILES TOPDIR" +SPDX_IMAGE_PURPOSE:wic = "diskImage" do_image_wic[cleandirs] = "${WORKDIR}/build-wic" PSEUDO_IGNORE_PATHS .= ",${WORKDIR}/build-wic"
Signature · 1 line
2.43.2
PATCH v4 7/10 selftest: spdx: Add SPDX 3.0 test cases
In reply to Joshua WattAdds test cases for SPDX 3.0. Reworks the SPDX 2.2 test setup so it can also be run even if the default is SPDX 3.0
--- meta/lib/oeqa/selftest/cases/spdx.py | 119 +++++++++++++++++++++++++-- 1 file changed, 110 insertions(+), 9 deletions(-)
meta/lib/oeqa/selftest/cases/spdx.py +110 -9
@@ -6,21 +6,26 @@ import json import os +import textwrap +from pathlib import Path from oeqa.selftest.case import OESelftestTestCase -from oeqa.utils.commands import bitbake, get_bb_var, runCmd +from oeqa.utils.commands import bitbake, get_bb_var, get_bb_vars, runCmd -class SPDXCheck(OESelftestTestCase): +class SPDX22Check(OESelftestTestCase): @classmethod def setUpClass(cls): - super(SPDXCheck, cls).setUpClass() + super().setUpClass() bitbake("python3-spdx-tools-native") bitbake("-c addto_recipe_sysroot python3-spdx-tools-native") def check_recipe_spdx(self, high_level_dir, spdx_file, target_name): - config = """ -INHERIT += "create-spdx" -""" + config = textwrap.dedent( + """\ + INHERIT:remove = "create-spdx" + INHERIT += "create-spdx-2.2" + """ + ) self.write_config(config) deploy_dir = get_bb_var("DEPLOY_DIR")
@@ -29,7 +34,9 @@ INHERIT += "create-spdx" # qemux86-64 creates the directory qemux86_64 machine_dir = machine_var.replace("-", "_") - full_file_path = os.path.join(deploy_dir, "spdx", spdx_version, machine_dir, high_level_dir, spdx_file) + full_file_path = os.path.join( + deploy_dir, "spdx", spdx_version, machine_dir, high_level_dir, spdx_file + ) try: os.remove(full_file_path)
@@ -44,8 +51,13 @@ INHERIT += "create-spdx" self.assertNotEqual(report, None) self.assertNotEqual(report["SPDXID"], None) - python = os.path.join(get_bb_var('STAGING_BINDIR', 'python3-spdx-tools-native'), 'nativepython3') - validator = os.path.join(get_bb_var('STAGING_BINDIR', 'python3-spdx-tools-native'), 'pyspdxtools') + python = os.path.join( + get_bb_var("STAGING_BINDIR", "python3-spdx-tools-native"), + "nativepython3", + ) + validator = os.path.join( + get_bb_var("STAGING_BINDIR", "python3-spdx-tools-native"), "pyspdxtools" + ) result = runCmd("{} {} -i {}".format(python, validator, filename)) self.assertExists(full_file_path)
@@ -53,3 +65,92 @@ INHERIT += "create-spdx" def test_spdx_base_files(self): self.check_recipe_spdx("packages", "base-files.spdx.json", "base-files") + + +class SPDX3CheckBase(object): + """ + Base class for checking SPDX 3 based tests + """ + + def check_spdx_file(self, filename): + import oe.spdx30 + + self.assertExists(filename) + + # Read the file + objset = oe.spdx30.SHACLObjectSet() + with open(filename, "r") as f: + d = oe.spdx30.JSONLDDeserializer() + d.read(f, objset) + + return objset + + def check_recipe_spdx(self, target_name, spdx_path, *, task=None): + config = textwrap.dedent( + f"""\ + INHERIT:remove = "create-spdx" + INHERIT += "{self.SPDX_CLASS}" + """ + ) + self.write_config(config) + + if task: + bitbake(f"-c {task} {target_name}") + else: + bitbake(target_name) + + filename = spdx_path.format( + **get_bb_vars( + [ + "DEPLOY_DIR_IMAGE", + "DEPLOY_DIR_SPDX", + "MACHINE", + "MACHINE_ARCH", + "SDKMACHINE", + "SDK_DEPLOY", + "SPDX_VERSION", + "TOOLCHAIN_OUTPUTNAME", + ], + target_name, + ) + ) + + return self.check_spdx_file(filename) + + def check_objset_missing_ids(self, objset): + if objset.missing_ids: + self.assertTrue( + False, + "The following SPDXIDs are unresolved:\n " + + "\n ".join(objset.missing_ids), + ) + + +class SPDX30Check(SPDX3CheckBase, OESelftestTestCase): + SPDX_CLASS = "create-spdx-3.0" + SCHEMA_URL = "https://spdx.org/schema/3.0.0/spdx-json-schema.json" + + def test_base_files(self): + self.check_recipe_spdx( + "base-files", + "{DEPLOY_DIR_SPDX}/{MACHINE_ARCH}/packages/base-files.spdx.json", + ) + + def test_core_image_minimal(self): + objset = self.check_recipe_spdx( + "core-image-minimal", + "{DEPLOY_DIR_IMAGE}/core-image-minimal-{MACHINE}.rootfs.spdx.json", + ) + + # Document should be fully linked + self.check_objset_missing_ids(objset) + + def test_core_image_minimal_sdk(self): + objset = self.check_recipe_spdx( + "core-image-minimal", + "{SDK_DEPLOY}/{TOOLCHAIN_OUTPUTNAME}.spdx.json", + task="populate_sdk", + ) + + # Document should be fully linked + self.check_objset_missing_ids(objset)
Signature · 1 line
2.43.2
PATCH v4 8/10 classes-recipe: nospdx: Add class
In reply to Joshua WattAdds a class that allows recipes to opt out of generating SPDX
--- meta/classes-recipe/nospdx.bbclass | 13 +++++++++++++ meta/recipes-core/meta/build-sysroots.bb | 5 +---- meta/recipes-core/meta/meta-world-pkgdata.bb | 3 +-- 3 files changed, 15 insertions(+), 6 deletions(-) create mode 100644 meta/classes-recipe/nospdx.bbclass
meta/classes-recipe/nospdx.bbclass new file +13 -0
@@ -0,0 +1,13 @@ +# +# Copyright OpenEmbedded Contributors +# +# SPDX-License-Identifier: MIT +# + +deltask do_collect_spdx_deps +deltask do_create_spdx +deltask do_create_spdx_runtime +deltask do_create_package_spdx +deltask do_create_rootfs_spdx +deltask do_create_image_spdx +deltask do_create_image_sbom
meta/recipes-core/meta/build-sysroots.bb +1 -4
@@ -7,7 +7,7 @@ STANDALONE_SYSROOT_NATIVE = "${STAGING_DIR}/${BUILD_ARCH}" PACKAGE_ARCH = "${MACHINE_ARCH}" EXCLUDE_FROM_WORLD = "1" -inherit nopackages +inherit nopackages nospdx deltask fetch deltask unpack deltask patch
@@ -17,9 +17,6 @@ deltask configure deltask compile deltask install deltask populate_sysroot -deltask create_spdx -deltask collect_spdx_deps -deltask create_runtime_spdx deltask recipe_qa do_build_warn () {
meta/recipes-core/meta/meta-world-pkgdata.bb +1 -2
@@ -27,14 +27,13 @@ python do_collect_packagedata() { oe.copy_buildsystem.generate_locked_sigs(sigfile, d) } +inherit nospdx deltask do_fetch deltask do_unpack deltask do_patch deltask do_configure deltask do_compile deltask do_install -deltask do_create_spdx -deltask do_create_spdx_runtime do_prepare_recipe_sysroot[deptask] = ""
Signature · 1 line
2.43.2
PATCH v4 9/10 classes/spdx-common: Move SPDX_SUPPLIER
In reply to Joshua WattMove the SPDX_SUPPLIER variable to create-spdx-2.2 since it's format only has meaning in SPDX 2.2 (SPDX 3 uses SPDX_PACKAGE_SUPPLIER with a different format)
--- meta/classes/create-spdx-2.2.bbclass | 8 ++++++++ meta/classes/spdx-common.bbclass | 7 ------- 2 files changed, 8 insertions(+), 7 deletions(-)
meta/classes/create-spdx-2.2.bbclass +8 -0
@@ -8,6 +8,14 @@ inherit spdx-common SPDX_VERSION = "2.2" +SPDX_ORG ??= "OpenEmbedded ()" +SPDX_SUPPLIER ??= "Organization: ${SPDX_ORG}" +SPDX_SUPPLIER[doc] = "The SPDX PackageSupplier field for SPDX packages created from \ + this recipe. For SPDX documents create using this class during the build, this \ + is the contact information for the person or organization who is doing the \ + build." + + def get_namespace(d, name): import uuid namespace_uuid = uuid.uuid5(uuid.NAMESPACE_DNS, d.getVar("SPDX_UUID_NAMESPACE"))
meta/classes/spdx-common.bbclass +0 -7
@@ -37,13 +37,6 @@ SPDX_LICENSES ??= "${COREBASE}/meta/files/spdx-licenses.json" SPDX_CUSTOM_ANNOTATION_VARS ??= "" -SPDX_ORG ??= "OpenEmbedded ()" -SPDX_SUPPLIER ??= "Organization: ${SPDX_ORG}" -SPDX_SUPPLIER[doc] = "The SPDX PackageSupplier field for SPDX packages created from \ - this recipe. For SPDX documents create using this class during the build, this \ - is the contact information for the person or organization who is doing the \ - build." - def extract_licenses(filename): import re
Signature · 1 line
2.43.2
PATCH v4 10/10 Switch default spdx version to 3.0
In reply to Joshua WattChanges the default SPDX version to 3.0
--- meta/classes/create-spdx.bbclass | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
meta/classes/create-spdx.bbclass +1 -1
@@ -5,4 +5,4 @@ # # Include this class when you don't care what version of SPDX you get; it will # be updated to the latest stable version that is supported -inherit create-spdx-2.2 +inherit create-spdx-3.0
Signature · 1 line
2.43.2
Hello,
I believe this is the cause of the following failures:
https://autobuilder.yoctoproject.org/typhoon/#/builders/117/builds/4986/steps/12/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/117/builds/4989/steps/13/logs/stdio
I'm not sure why they don't have any diffoscope output.
Quoted 63 lines · Joshua Watt
This patch series add support for SPDX 3.0 and sets it as the default. Currently it is not possible to have SPDX 2.2 and SPDX 3.0 enabled at the same time
v2: Added tests and addressed feedback
v3: Fixed several oe-selftest and build failures
v4: Fixed silly typo mistake in staging.bbclassJoshua Watt (10):
classes-recipe/image: Add image file manifest classes-recipe/baremetal-image: Add image file manifest classes/create-spdx-3.0: Add classes classes-global/staging: Excludedo_create_spdxfrom automatic sysroot
extension
binutils-cross-testsuite: Rename to binutils-testsuite classes-recipe/image_types: AddSPDX_IMAGE_PURPOSEto images
selftest: spdx: Add SPDX 3.0 test cases
classes-recipe: nospdx: Add class
classes/spdx-common: MoveSPDX_SUPPLIER
Switch default spdx version to 3.0meta/classes-global/mirrors.bbclass | 2 +- meta/classes-global/staging.bbclass | 9 +- meta/classes-recipe/baremetal-image.bbclass | 30 +- meta/classes-recipe/image.bbclass | 58 + meta/classes-recipe/image_types.bbclass | 2 + meta/classes-recipe/image_types_wic.bbclass | 1 + meta/classes-recipe/nospdx.bbclass | 13 + meta/classes-recipe/packagegroup.bbclass | 2 + meta/classes/create-spdx-2.2.bbclass | 8 + meta/classes/create-spdx-3.0.bbclass | 1179 ++++ meta/classes/create-spdx-image-3.0.bbclass | 224 + meta/classes/create-spdx.bbclass | 2 +- meta/classes/spdx-common.bbclass | 13 +- meta/conf/distro/include/maintainers.inc | 2 +- meta/lib/oe/sbom30.py | 1122 +++ meta/lib/oe/spdx30.py | 5996 +++++++++++++++++ meta/lib/oeqa/selftest/cases/binutils.py | 2 +- meta/lib/oeqa/selftest/cases/spdx.py | 119 +- meta/recipes-core/meta/build-sysroots.bb | 5 +- meta/recipes-core/meta/meta-world-pkgdata.bb | 3 +- ...ite_2.42.bb => binutils-testsuite_2.42.bb} | 3 + 21 files changed, 8765 insertions(+), 30 deletions(-) create mode 100644 meta/classes-recipe/nospdx.bbclass create mode 100644 meta/classes/create-spdx-3.0.bbclass create mode 100644 meta/classes/create-spdx-image-3.0.bbclass create mode 100644 meta/lib/oe/sbom30.py create mode 100644 meta/lib/oe/spdx30.py rename meta/recipes-devtools/binutils/{binutils-cross-testsuite_2.42.bb => binutils-testsuite_2.42.bb} (95%)Signature · 11 lines
2.43.2 -=-=-=-=-=-=-=-=-=-=-=- Links: You receive all messages sent to this group. View/Reply Online (#201102): https://lists.openembedded.org/g/openembedded-core/message/201102 Mute This Topic: https://lists.openembedded.org/mt/106856868/3617179 Group Owner: [email protected] Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [[email protected]] -=-=-=-=-=-=-=-=-=-=-=-
Signature · 3 lines
Alexandre Belloni, co-owner and COO, Bootlin Embedded Linux and Kernel engineering https://bootlin.com
Quoted 8 lines · Alexandre Belloni via lists.openembedded.org
Hello,
I believe this is the cause of the following failures:
https://autobuilder.yoctoproject.org/typhoon/#/builders/117/builds/4986/steps/12/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/117/builds/4989/steps/13/logs/stdioI'm not sure why they don't have any diffoscope output.
"The following rpm packages are missing and not in exclusion list:"
there is no diffoscope as they don't exist at all.
I suspect a cleandirs corrupted the build by deleting things before they were added to the sstate output.
Cheers,
Richard
v5 cover Add SPDX 3.0 support
In reply to Joshua WattThis patch series add support for SPDX 3.0 and sets it as the default. Currently it is not possible to have SPDX 2.2 and SPDX 3.0 enabled at the same time
v2: Added tests and addressed feedback
v3: Fixed several oe-selftest and build failures
v4: Fixed silly typo mistake in staging.bbclass
v5: Reworked to make SPDX 3 output reproducible by default. Variables
that introduce non-reproducible output are documented as such.
Joshua Watt (8):
classes-recipe/image: Add image file manifest classes-recipe/baremetal-image: Add image file manifest classes/create-spdx-3.0: Add classes classes-global/staging: Exclude do_create_spdx from automatic sysroot
extension
classes-recipe/image_types: Add SPDX_IMAGE_PURPOSE to images
selftest: spdx: Add SPDX 3.0 test cases
classes-recipe: nospdx: Add class
Switch default spdx version to 3.0
meta/classes-global/staging.bbclass | 9 +- meta/classes-recipe/baremetal-image.bbclass | 32 +- meta/classes-recipe/image.bbclass | 58 + meta/classes-recipe/image_types.bbclass | 2 + meta/classes-recipe/image_types_wic.bbclass | 1 + meta/classes-recipe/nospdx.bbclass | 13 + meta/classes-recipe/packagegroup.bbclass | 2 + meta/classes/create-spdx-3.0.bbclass | 1231 ++++ meta/classes/create-spdx-image-3.0.bbclass | 224 + meta/classes/create-spdx.bbclass | 2 +- meta/classes/spdx-common.bbclass | 6 +- meta/lib/oe/sbom30.py | 1138 ++++ meta/lib/oe/spdx30.py | 5996 ++++++++++++++++++ meta/lib/oeqa/selftest/cases/spdx.py | 118 +- meta/recipes-core/meta/build-sysroots.bb | 5 +- meta/recipes-core/meta/meta-world-pkgdata.bb | 3 +- 16 files changed, 8819 insertions(+), 21 deletions(-) create mode 100644 meta/classes-recipe/nospdx.bbclass create mode 100644 meta/classes/create-spdx-3.0.bbclass create mode 100644 meta/classes/create-spdx-image-3.0.bbclass create mode 100644 meta/lib/oe/sbom30.py create mode 100644 meta/lib/oe/spdx30.py
Signature · 1 line
2.45.2
PATCH v5 1/8 classes-recipe/image: Add image file manifest
Downstream tasks may want to know what image files were written by the do_image family of tasks (e.g. SPDX) so have each task write out a manifest file that describes the files it produced, then aggregate them in do_image_complete
--- meta/classes-recipe/image.bbclass | 58 +++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+)
meta/classes-recipe/image.bbclass +58 -0
@@ -88,6 +88,11 @@ PACKAGE_INSTALL_ATTEMPTONLY ?= "${FEATURE_INSTALL_OPTIONAL}" IMGDEPLOYDIR = "${WORKDIR}/deploy-${PN}-image-complete" +IMGMANIFESTDIR = "${WORKDIR}/image-task-manifest" + +IMAGE_OUTPUT_MANIFEST_DIR = "${WORKDIR}/deploy-image-output-manifest" +IMAGE_OUTPUT_MANIFEST = "${IMAGE_OUTPUT_MANIFEST_DIR}/manifest.json" + # Images are generally built explicitly, do not need to be part of world. EXCLUDE_FROM_WORLD = "1"
@@ -277,14 +282,28 @@ fakeroot python do_image () { execute_pre_post_process(d, pre_process_cmds) } do_image[dirs] = "${TOPDIR}" +do_image[cleandirs] += "${IMGMANIFESTDIR}" addtask do_image after do_rootfs fakeroot python do_image_complete () { from oe.utils import execute_pre_post_process + from pathlib import Path + import json post_process_cmds = d.getVar("IMAGE_POSTPROCESS_COMMAND") execute_pre_post_process(d, post_process_cmds) + + image_manifest_dir = Path(d.getVar('IMGMANIFESTDIR')) + + data = [] + + for manifest_path in image_manifest_dir.glob("*.json"): + with manifest_path.open("r") as f: + data.extend(json.load(f)) + + with open(d.getVar("IMAGE_OUTPUT_MANIFEST"), "w") as f: + json.dump(data, f) } do_image_complete[dirs] = "${TOPDIR}" SSTATETASKS += "do_image_complete"
@@ -292,6 +311,8 @@ SSTATE_SKIP_CREATION:task-image-complete = '1' do_image_complete[sstate-inputdirs] = "${IMGDEPLOYDIR}" do_image_complete[sstate-outputdirs] = "${DEPLOY_DIR_IMAGE}" do_image_complete[stamp-extra-info] = "${MACHINE_ARCH}" +do_image_complete[sstate-plaindirs] += "${IMAGE_OUTPUT_MANIFEST_DIR}" +do_image_complete[dirs] += "${IMAGE_OUTPUT_MANIFEST_DIR}" addtask do_image_complete after do_image before do_build python do_image_complete_setscene () { sstate_setscene(d)
@@ -507,12 +528,14 @@ python () { d.setVar(task, '\n'.join(cmds)) d.setVarFlag(task, 'func', '1') d.setVarFlag(task, 'fakeroot', '1') + d.setVarFlag(task, 'imagetype', t) d.appendVarFlag(task, 'prefuncs', ' ' + debug + ' set_image_size') d.prependVarFlag(task, 'postfuncs', 'create_symlinks ') d.appendVarFlag(task, 'subimages', ' ' + ' '.join(subimages)) d.appendVarFlag(task, 'vardeps', ' ' + ' '.join(vardeps)) d.appendVarFlag(task, 'vardepsexclude', ' DATETIME DATE ' + ' '.join(vardepsexclude)) + d.appendVarFlag(task, 'postfuncs', ' write_image_output_manifest') bb.debug(2, "Adding task %s before %s, after %s" % (task, 'do_image_complete', after)) bb.build.addtask(task, 'do_image_complete', after, d)
@@ -610,6 +633,41 @@ python create_symlinks() { bb.note("Skipping symlink, source does not exist: %s -> %s" % (dst, src)) } +python write_image_output_manifest() { + import json + from pathlib import Path + + taskname = d.getVar("BB_CURRENTTASK") + image_deploy_dir = Path(d.getVar('IMGDEPLOYDIR')) + image_manifest_dir = Path(d.getVar('IMGMANIFESTDIR')) + manifest_path = image_manifest_dir / ("do_" + d.getVar("BB_CURRENTTASK") + ".json") + + image_name = d.getVar("IMAGE_NAME") + image_basename = d.getVar("IMAGE_BASENAME") + machine = d.getVar("MACHINE") + + subimages = (d.getVarFlag("do_" + taskname, 'subimages', False) or "").split() + imagetype = d.getVarFlag("do_" + taskname, 'imagetype', False) + + data = { + "taskname": taskname, + "imagetype": imagetype, + "images": [] + } + + for type in subimages: + image_filename = image_name + "." + type + image_path = image_deploy_dir / image_filename + if not image_path.exists(): + continue + data["images"].append({ + "filename": image_filename, + }) + + with manifest_path.open("w") as f: + json.dump([data], f) +} + MULTILIBRE_ALLOW_REP += "${base_bindir} ${base_sbindir} ${bindir} ${sbindir} ${libexecdir} ${sysconfdir} ${nonarch_base_libdir}/udev /lib/modules/[^/]*/modules.*" MULTILIB_CHECK_FILE = "${WORKDIR}/multilib_check.py" MULTILIB_TEMP_ROOTFS = "${WORKDIR}/multilib"
Signature · 1 line
2.45.2