Skip to contentSkip to conversation

Add SPDX 3.0 support

PATCH Superseded Joshua Watt 10 participants 93 messages Jun 10, 2024 – Jul 17, 2024 3 messages not in the archive Page 1 of 2

lore
Share & tools
lore
#0 lore raw

cover Add SPDX 3.0 support

This patch series add support for SPDX 3.0. Currently it is not possible to have SPDX 2.2 and SPDX 3.0 enabled at the same time; users may choose to specifically use SPDX 3.0 instead of 2.2 by doing INHERIT += "create-spdx-3.0"

The default version is not changed yet, although that will likely happen soon

Joshua Watt (6):
classes-recipe/image: Add image file manifest classes/spdx-common: Move common SPDX to new class classes/spdx-common: Add SPDX version to path classes/spdx-common: Return empty list from extract_licenses classes/create-spdx-3.0: Add class classes-recipe/image_types: Add SPDX_IMAGE_PURPOSE to images

 meta/classes-recipe/image.bbclass           |   61 +
 meta/classes-recipe/image_types.bbclass     |    2 +
 meta/classes-recipe/image_types_wic.bbclass |    1 +
 meta/classes/create-spdx-2.2.bbclass        |  261 +-
 meta/classes/create-spdx-3.0.bbclass        | 1284 +++++
 meta/classes/spdx-common.bbclass            |  280 +
 meta/lib/oe/sbom30.py                       |  993 ++++
 meta/lib/oe/spdx30.py                       | 5413 +++++++++++++++++++
 8 files changed, 8046 insertions(+), 249 deletions(-)
 create mode 100644 meta/classes/create-spdx-3.0.bbclass
 create mode 100644 meta/classes/spdx-common.bbclass
 create mode 100644 meta/lib/oe/sbom30.py
 create mode 100644 meta/lib/oe/spdx30.py
Signature · 1 line
2.43.2
#1 lore raw

PATCH 1/6 classes-recipe/image: Add image file manifest

Downstream tasks may want to know what image files were written by the do_image family of tasks (e.g. SPDX) so have each task write out a manifest file that describes the files it produced, then aggregate them in do_image_complete

---
 meta/classes-recipe/image.bbclass | 61 +++++++++++++++++++++++++++++++
 1 file changed, 61 insertions(+)
meta/classes-recipe/image.bbclass +61 -0
index 28be6c63623..122af2b0328 100644
@@ -88,6 +88,11 @@ PACKAGE_INSTALL_ATTEMPTONLY ?= "${FEATURE_INSTALL_OPTIONAL}"
 
 IMGDEPLOYDIR = "${WORKDIR}/deploy-${PN}-image-complete"
 
+IMGMANIFESTDIR = "${WORKDIR}/image-task-manifest"
+
+IMAGE_FILE_MANIFEST_DIR = "${WORKDIR}/deploy-image-file-manifest"
+IMAGE_FILE_MANIFEST = "${IMAGE_FILE_MANIFEST_DIR}/manifest.json"
+
 # Images are generally built explicitly, do not need to be part of world.
 EXCLUDE_FROM_WORLD = "1"
 
@@ -277,14 +282,31 @@ fakeroot python do_image () {
     execute_pre_post_process(d, pre_process_cmds)
 }
 do_image[dirs] = "${TOPDIR}"
+do_image[cleandirs] = "${IMGMANIFESTDIR}"
 addtask do_image after do_rootfs
 
 fakeroot python do_image_complete () {
     from oe.utils import execute_pre_post_process
+    from pathlib import Path
+    import json
 
     post_process_cmds = d.getVar("IMAGE_POSTPROCESS_COMMAND")
 
     execute_pre_post_process(d, post_process_cmds)
+
+    image_manifest_dir = Path(d.getVar('IMGMANIFESTDIR'))
+
+    data = []
+
+    for taskname in bb.build.preceedtask("do_image_complete", False, d):
+        manifest_path = image_manifest_dir / (taskname + ".json")
+        if not manifest_path.exists():
+            continue
+        with manifest_path.open("r") as f:
+            data.extend(json.load(f))
+
+    with open(d.getVar("IMAGE_FILE_MANIFEST"), "w") as f:
+        json.dump(data, f)
 }
 do_image_complete[dirs] = "${TOPDIR}"
 SSTATETASKS += "do_image_complete"
@@ -292,6 +314,8 @@ SSTATE_SKIP_CREATION:task-image-complete = '1'
 do_image_complete[sstate-inputdirs] = "${IMGDEPLOYDIR}"
 do_image_complete[sstate-outputdirs] = "${DEPLOY_DIR_IMAGE}"
 do_image_complete[stamp-extra-info] = "${MACHINE_ARCH}"
+do_image_complete[sstate-plaindirs] =  "${IMAGE_FILE_MANIFEST_DIR}"
+do_image_complete[dirs] = "${IMAGE_FILE_MANIFEST_DIR}"
 addtask do_image_complete after do_image before do_build
 python do_image_complete_setscene () {
     sstate_setscene(d)
@@ -513,6 +537,8 @@ python () {
         d.appendVarFlag(task, 'subimages', ' ' + ' '.join(subimages))
         d.appendVarFlag(task, 'vardeps', ' ' + ' '.join(vardeps))
         d.appendVarFlag(task, 'vardepsexclude', ' DATETIME DATE ' + ' '.join(vardepsexclude))
+        d.appendVarFlag(task, 'postfuncs', ' write_image_file_manifest')
+        d.appendVarFlag(task, 'dirs', ' ' + d.getVar('IMGMANIFESTDIR'))
 
         bb.debug(2, "Adding task %s before %s, after %s" % (task, 'do_image_complete', after))
         bb.build.addtask(task, 'do_image_complete', after, d)
@@ -610,6 +636,41 @@ python create_symlinks() {
             bb.note("Skipping symlink, source does not exist: %s -> %s" % (dst, src))
 }
 
+python write_image_file_manifest() {
+    import json
+    from pathlib import Path
+
+    taskname = d.getVar("BB_CURRENTTASK")
+    image_deploy_dir = Path(d.getVar('IMGDEPLOYDIR'))
+    image_manifest_dir = Path(d.getVar('IMGMANIFESTDIR'))
+    manifest_path = image_manifest_dir / ("do_" + d.getVar("BB_CURRENTTASK") + ".json")
+
+    image_name = d.getVar("IMAGE_NAME")
+    image_basename = d.getVar("IMAGE_BASENAME")
+    machine = d.getVar("MACHINE")
+
+    subimages = (d.getVarFlag("do_" + taskname, 'subimages', False) or "").split()
+    imagetype = d.getVarFlag("do_" + taskname, 'imagetype', False)
+
+    data = {
+        "taskname": taskname,
+        "imagetype": imagetype,
+        "images": []
+    }
+
+    for type in subimages:
+        image_filename = image_name + "." + type
+        image_path = image_deploy_dir / image_filename
+        if not image_path.exists():
+            continue
+        data["images"].append({
+            "filename": image_filename,
+        })
+
+    with manifest_path.open("w") as f:
+        json.dump([data], f)
+}
+
 MULTILIBRE_ALLOW_REP += "${base_bindir} ${base_sbindir} ${bindir} ${sbindir} ${libexecdir} ${sysconfdir} ${nonarch_base_libdir}/udev /lib/modules/[^/]*/modules.*"
 MULTILIB_CHECK_FILE = "${WORKDIR}/multilib_check.py"
 MULTILIB_TEMP_ROOTFS = "${WORKDIR}/multilib"
Signature · 1 line
2.43.2
#2 lore raw

Re: [OE-core][PATCH 1/6] classes-recipe/image: Add image file manifest

Hi Joshua,

Two minor nits below.

Quoted 35 lines · Joshua Watt

Downstream tasks may want to know what image files were written by the
do_image family of tasks (e.g. SPDX) so have each task write out a manifest file that describes the files it produced, then aggregate them
in do_image_complete

---
 meta/classes-recipe/image.bbclass | 61
+++++++++++++++++++++++++++++++

 1 file changed, 61 insertions(+)
meta/classes- renamed from meta/classes-recipe/image.bbclass +0 -0

recipe/image.bbclass
index 28be6c63623..122af2b0328 100644

meta/classes-recipe/image.bbclass +0 -0
@@ -88,6 +88,11 @@ PACKAGE_INSTALL_ATTEMPTONLY ?=

"${FEATURE_INSTALL_OPTIONAL}"

 IMGDEPLOYDIR = "${WORKDIR}/deploy-${PN}-image-complete"

meta/classes-recipe/image.bbclass +5 -0
+IMGMANIFESTDIR = "${WORKDIR}/image-task-manifest"
+
+IMAGE_FILE_MANIFEST_DIR = "${WORKDIR}/deploy-image-file-manifest"
+IMAGE_FILE_MANIFEST = "${IMAGE_FILE_MANIFEST_DIR}/manifest.json"
+

 # Images are generally built explicitly, do not need to be part of world.
 EXCLUDE_FROM_WORLD = "1"

meta/classes-recipe/image.bbclass +0 -0
@@ -277,14 +282,31 @@ fakeroot python do_image () {
      execute_pre_post_process(d, pre_process_cmds)
  }
Quoted 2 lines

 do_image[dirs] = "${TOPDIR}"
+do_image[cleandirs] = "${IMGMANIFESTDIR}"

Should this be an append (i.e. +=) to avoid conflicting with recipes ?

Quoted 32 lines

 addtask do_image after do_rootfs

 fakeroot python do_image_complete () {
     from oe.utils import execute_pre_post_process
+    from pathlib import Path
+    import json

     post_process_cmds = d.getVar("IMAGE_POSTPROCESS_COMMAND")
     execute_pre_post_process(d, post_process_cmds)

meta/classes-recipe/image.bbclass +6 -0
+
+    image_manifest_dir = Path(d.getVar('IMGMANIFESTDIR'))
+
+    data = []
+
+    for taskname in bb.build.preceedtask("do_image_complete", False,

d):
+        manifest_path = image_manifest_dir / (taskname + ".json") +        if not manifest_path.exists():
+            continue
+        with manifest_path.open("r") as f:
+            data.extend(json.load(f))
+
+    with open(d.getVar("IMAGE_FILE_MANIFEST"), "w") as f:
+        json.dump(data, f)
 }
 do_image_complete[dirs] = "${TOPDIR}"
 SSTATETASKS += "do_image_complete"

meta/classes-recipe/image.bbclass +0 -0
@@ -292,6 +314,8 @@ SSTATE_SKIP_CREATION:task-image-complete = '1'

 do_image_complete[sstate-inputdirs] = "${IMGDEPLOYDIR}" do_image_complete[sstate-outputdirs] = "${DEPLOY_DIR_IMAGE}" do_image_complete[stamp-extra-info] = "${MACHINE_ARCH}"

meta/classes-recipe/image.bbclass SSTATE_SKIP_CREATION:task-image-complete = '1' · line 318

Quoted 2 lines
meta/classes-recipe/image.bbclass +2 -0
+do_image_complete[sstate-plaindirs] =  "${IMAGE_FILE_MANIFEST_DIR}"
+do_image_complete[dirs] = "${IMAGE_FILE_MANIFEST_DIR}"

Same here...

Thanks,
Martin

Quoted 76 lines

 addtask do_image_complete after do_image before do_build python do_image_complete_setscene () { sstate_setscene(d)

meta/classes-recipe/image.bbclass +0 -0
@@ -513,6 +537,8 @@ python () {

         d.appendVarFlag(task, 'subimages', ' ' + ' '.join(subimages))
         d.appendVarFlag(task, 'vardeps', ' ' + ' '.join(vardeps)) d.appendVarFlag(task, 'vardepsexclude', ' DATETIME DATE ' + ' '.join(vardepsexclude))
+        d.appendVarFlag(task, 'postfuncs', ' write_image_file_manifest')
+        d.appendVarFlag(task, 'dirs', ' ' + d.getVar('IMGMANIFESTDIR'))

         bb.debug(2, "Adding task %s before %s, after %s" % (task, 'do_image_complete', after))
         bb.build.addtask(task, 'do_image_complete', after, d)

meta/classes-recipe/image.bbclass +0 -0
@@ -610,6 +636,41 @@ python create_symlinks() {

             bb.note("Skipping symlink, source does not exist: %s -> %s" % (dst, src))
 }

meta/classes-recipe/image.bbclass +8 -0
+python write_image_file_manifest() {
+    import json
+    from pathlib import Path
+
+    taskname = d.getVar("BB_CURRENTTASK")
+    image_deploy_dir = Path(d.getVar('IMGDEPLOYDIR'))
+    image_manifest_dir = Path(d.getVar('IMGMANIFESTDIR'))
+    manifest_path = image_manifest_dir / ("do_" +

d.getVar("BB_CURRENTTASK") + ".json")

meta/classes-recipe/image.bbclass +6 -0
+
+    image_name = d.getVar("IMAGE_NAME")
+    image_basename = d.getVar("IMAGE_BASENAME")
+    machine = d.getVar("MACHINE")
+
+    subimages = (d.getVarFlag("do_" + taskname, 'subimages', False)

or "").split()
+    imagetype = d.getVarFlag("do_" + taskname, 'imagetype', False)

meta/classes-recipe/image.bbclass +20 -0
+
+    data = {
+        "taskname": taskname,
+        "imagetype": imagetype,
+        "images": []
+    }
+
+    for type in subimages:
+        image_filename = image_name + "." + type
+        image_path = image_deploy_dir / image_filename
+        if not image_path.exists():
+            continue
+        data["images"].append({
+            "filename": image_filename,
+        })
+
+    with manifest_path.open("w") as f:
+        json.dump([data], f)
+}
+

 MULTILIBRE_ALLOW_REP += "${base_bindir} ${base_sbindir} ${bindir} ${sbindir} ${libexecdir} ${sysconfdir} ${nonarch_base_libdir}/udev /lib/modules/[^/]*/modules.*"
 MULTILIB_CHECK_FILE = "${WORKDIR}/multilib_check.py" MULTILIB_TEMP_ROOTFS = "${WORKDIR}/multilib"

-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group. View/Reply Online (#200485):
https://lists.openembedded.org/g/openembedded-core/message/200485
Mute This Topic: https://lists.openembedded.org/mt/106602515/3618414
Group Owner: [email protected] Unsubscribe:
https://lists.openembedded.org/g/openembedded-core/unsub [
[email protected]]
-=-=-=-=-=-=-=-=-=-=-=-

#3 lore raw

PATCH 2/6 classes/spdx-common: Move common SPDX to new class

In reply to Joshua Watt

Moves SPDX code that can be shared between different SPDX versions into a common class

---
 meta/classes/create-spdx-2.2.bbclass | 259 ++-------------------------
 meta/classes/spdx-common.bbclass     | 256 ++++++++++++++++++++++++++
 2 files changed, 266 insertions(+), 249 deletions(-)
 create mode 100644 meta/classes/spdx-common.bbclass
meta/classes/create-spdx-2.2.bbclass +10 -249
index 7c8a0b8b0f7..94a172fbc94 100644
@@ -4,65 +4,13 @@
 # SPDX-License-Identifier: GPL-2.0-only
 #
 
-DEPLOY_DIR_SPDX ??= "${DEPLOY_DIR}/spdx"
-
-# The product name that the CVE database uses.  Defaults to BPN, but may need to
-# be overriden per recipe (for example tiff.bb sets CVE_PRODUCT=libtiff).
-CVE_PRODUCT ??= "${BPN}"
-CVE_VERSION ??= "${PV}"
-
-SPDXDIR ??= "${WORKDIR}/spdx"
-SPDXDEPLOY = "${SPDXDIR}/deploy"
-SPDXWORK = "${SPDXDIR}/work"
-SPDXIMAGEWORK = "${SPDXDIR}/image-work"
-SPDXSDKWORK = "${SPDXDIR}/sdk-work"
-SPDXDEPS = "${SPDXDIR}/deps.json"
-
-SPDX_TOOL_NAME ??= "oe-spdx-creator"
-SPDX_TOOL_VERSION ??= "1.0"
-
-SPDXRUNTIMEDEPLOY = "${SPDXDIR}/runtime-deploy"
-
-SPDX_INCLUDE_SOURCES ??= "0"
-SPDX_ARCHIVE_SOURCES ??= "0"
-SPDX_ARCHIVE_PACKAGED ??= "0"
-
-SPDX_UUID_NAMESPACE ??= "sbom.openembedded.org"
-SPDX_NAMESPACE_PREFIX ??= "http://spdx.org/spdxdocs"
-SPDX_PRETTY ??= "0"
-
-SPDX_LICENSES ??= "${COREBASE}/meta/files/spdx-licenses.json"
-
-SPDX_CUSTOM_ANNOTATION_VARS ??= ""
-
-SPDX_ORG ??= "OpenEmbedded ()"
-SPDX_SUPPLIER ??= "Organization: ${SPDX_ORG}"
-SPDX_SUPPLIER[doc] = "The SPDX PackageSupplier field for SPDX packages created from \
-    this recipe. For SPDX documents create using this class during the build, this \
-    is the contact information for the person or organization who is doing the \
-    build."
-
-def extract_licenses(filename):
-    import re
-
-    lic_regex = re.compile(rb'^\W*SPDX-License-Identifier:\s*([ \w\d.()+-]+?)(?:\s+\W*)?$', re.MULTILINE)
-
-    try:
-        with open(filename, 'rb') as f:
-            size = min(15000, os.stat(filename).st_size)
-            txt = f.read(size)
-            licenses = re.findall(lic_regex, txt)
-            if licenses:
-                ascii_licenses = [lic.decode('ascii') for lic in licenses]
-                return ascii_licenses
-    except Exception as e:
-        bb.warn(f"Exception reading {filename}: {e}")
-    return None
-
-def get_doc_namespace(d, doc):
+inherit spdx-common
+
+def get_namespace(d, name):
     import uuid
     namespace_uuid = uuid.uuid5(uuid.NAMESPACE_DNS, d.getVar("SPDX_UUID_NAMESPACE"))
-    return "%s/%s-%s" % (d.getVar("SPDX_NAMESPACE_PREFIX"), doc.name, str(uuid.uuid5(namespace_uuid, doc.name)))
+    return "%s/%s-%s" % (d.getVar("SPDX_NAMESPACE_PREFIX"), name, str(uuid.uuid5(namespace_uuid, name)))
+
 
 def create_annotation(d, comment):
     from datetime import datetime, timezone
@@ -80,26 +28,6 @@ def recipe_spdx_is_native(d, recipe):
       a.annotator == "Tool: %s - %s" % (d.getVar("SPDX_TOOL_NAME"), d.getVar("SPDX_TOOL_VERSION")) and
       a.comment == "isNative" for a in recipe.annotations)
 
-def is_work_shared_spdx(d):
-    return bb.data.inherits_class('kernel', d) or ('work-shared' in d.getVar('WORKDIR'))
-
-def get_json_indent(d):
-    if d.getVar("SPDX_PRETTY") == "1":
-        return 2
-    return None
-
-python() {
-    import json
-    if d.getVar("SPDX_LICENSE_DATA"):
-        return
-
-    with open(d.getVar("SPDX_LICENSES"), "r") as f:
-        data = json.load(f)
-        # Transform the license array to a dictionary
-        data["licenses"] = {l["licenseId"]: l for l in data["licenses"]}
-        d.setVar("SPDX_LICENSE_DATA", data)
-}
-
 def convert_license_to_spdx(lic, document, d, existing={}):
     from pathlib import Path
     import oe.spdx
@@ -172,34 +100,6 @@ def convert_license_to_spdx(lic, document, d, existing={}):
 
     return ' '.join(convert(l) for l in lic_split)
 
-def process_sources(d):
-    pn = d.getVar('PN')
-    assume_provided = (d.getVar("ASSUME_PROVIDED") or "").split()
-    if pn in assume_provided:
-        for p in d.getVar("PROVIDES").split():
-            if p != pn:
-                pn = p
-                break
-
-    # glibc-locale: do_fetch, do_unpack and do_patch tasks have been deleted,
-    # so avoid archiving source here.
-    if pn.startswith('glibc-locale'):
-        return False
-    if d.getVar('PN') == "libtool-cross":
-        return False
-    if d.getVar('PN') == "libgcc-initial":
-        return False
-    if d.getVar('PN') == "shadow-sysroot":
-        return False
-
-    # We just archive gcc-source for all the gcc related recipes
-    if d.getVar('BPN') in ['gcc', 'libgcc']:
-        bb.debug(1, 'spdx: There is bug in scan of %s is, do nothing' % pn)
-        return False
-
-    return True
-
-
 def add_package_files(d, doc, spdx_pkg, topdir, get_spdxid, get_types, *, archive=None, ignore_dirs=[], ignore_top_level_dirs=[]):
     from pathlib import Path
     import oe.spdx
@@ -348,14 +248,12 @@ def collect_dep_recipes(d, doc, spdx_recipe):
     import oe.spdx
 
     deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX"))
-    spdx_deps_file = Path(d.getVar("SPDXDEPS"))
     package_archs = d.getVar("SSTATE_ARCHS").split()
     package_archs.reverse()
 
     dep_recipes = []
 
-    with spdx_deps_file.open("r") as f:
-        deps = json.load(f)
+    deps = get_spdx_deps(d)
 
     for dep_pn, dep_hashfn, in_taskhash in deps:
         # If this dependency is not calculated in the taskhash skip it.
@@ -468,51 +366,6 @@ def add_download_packages(d, doc, recipe):
             # but this should be sufficient for now
             doc.add_relationship(package, "BUILD_DEPENDENCY_OF", recipe)
 
-def collect_direct_deps(d, dep_task):
-    current_task = "do_" + d.getVar("BB_CURRENTTASK")
-    pn = d.getVar("PN")
-
-    taskdepdata = d.getVar("BB_TASKDEPDATA", False)
-
-    for this_dep in taskdepdata.values():
-        if this_dep[0] == pn and this_dep[1] == current_task:
-            break
-    else:
-        bb.fatal(f"Unable to find this {pn}:{current_task} in taskdepdata")
-
-    deps = set()
-    for dep_name in this_dep[3]:
-        dep_data = taskdepdata[dep_name]
-        if dep_data[1] == dep_task and dep_data[0] != pn:
-            deps.add((dep_data[0], dep_data[7], dep_name in this_dep[8]))
-
-    return sorted(deps)
-
-collect_direct_deps[vardepsexclude] += "BB_TASKDEPDATA"
-collect_direct_deps[vardeps] += "DEPENDS"
-
-python do_collect_spdx_deps() {
-    # This task calculates the build time dependencies of the recipe, and is
-    # required because while a task can deptask on itself, those dependencies
-    # do not show up in BB_TASKDEPDATA. To work around that, this task does the
-    # deptask on do_create_spdx and writes out the dependencies it finds, then
-    # do_create_spdx reads in the found dependencies when writing the actual
-    # SPDX document
-    import json
-    from pathlib import Path
-
-    spdx_deps_file = Path(d.getVar("SPDXDEPS"))
-
-    deps = collect_direct_deps(d, "do_create_spdx")
-
-    with spdx_deps_file.open("w") as f:
-        json.dump(deps, f)
-}
-# NOTE: depending on do_unpack is a hack that is necessary to get it's dependencies for archive the source
-addtask do_collect_spdx_deps after do_unpack
-do_collect_spdx_deps[depends] += "${PATCHDEPENDENCY}"
-do_collect_spdx_deps[deptask] = "do_create_spdx"
-do_collect_spdx_deps[dirs] = "${SPDXDIR}"
 
 python do_create_spdx() {
     from datetime import datetime, timezone
@@ -551,7 +404,7 @@ python do_create_spdx() {
     doc = oe.spdx.SPDXDocument()
 
     doc.name = "recipe-" + d.getVar("PN")
-    doc.documentNamespace = get_doc_namespace(d, doc)
+    doc.documentNamespace = get_namespace(d, doc.name)
     doc.creationInfo.created = creation_time
     doc.creationInfo.comment = "This document was created by analyzing recipe files during the build."
     doc.creationInfo.licenseListVersion = d.getVar("SPDX_LICENSE_DATA")["licenseListVersion"]
@@ -655,7 +508,7 @@ python do_create_spdx() {
             package_doc = oe.spdx.SPDXDocument()
             pkg_name = d.getVar("PKG:%s" % package) or package
             package_doc.name = pkg_name
-            package_doc.documentNamespace = get_doc_namespace(d, package_doc)
+            package_doc.documentNamespace = get_namespace(d, package_doc.name)
             package_doc.creationInfo.created = creation_time
             package_doc.creationInfo.comment = "This document was created by analyzing packages created during the build."
             package_doc.creationInfo.licenseListVersion = d.getVar("SPDX_LICENSE_DATA")["licenseListVersion"]
@@ -716,44 +569,6 @@ do_create_spdx[dirs] = "${SPDXWORK}"
 do_create_spdx[cleandirs] = "${SPDXDEPLOY} ${SPDXWORK}"
 do_create_spdx[depends] += "${PATCHDEPENDENCY}"
 
-def collect_package_providers(d):
-    from pathlib import Path
-    import oe.sbom
-    import oe.spdx
-    import json
-
-    deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX"))
-
-    providers = {}
-
-    deps = collect_direct_deps(d, "do_create_spdx")
-    deps.append((d.getVar("PN"), d.getVar("BB_HASHFILENAME"), True))
-
-    for dep_pn, dep_hashfn, _ in deps:
-        localdata = d
-        recipe_data = oe.packagedata.read_pkgdata(dep_pn, localdata)
-        if not recipe_data:
-            localdata = bb.data.createCopy(d)
-            localdata.setVar("PKGDATA_DIR", "${PKGDATA_DIR_SDK}")
-            recipe_data = oe.packagedata.read_pkgdata(dep_pn, localdata)
-
-        for pkg in recipe_data.get("PACKAGES", "").split():
-
-            pkg_data = oe.packagedata.read_subpkgdata_dict(pkg, localdata)
-            rprovides = set(n for n, _ in bb.utils.explode_dep_versions2(pkg_data.get("RPROVIDES", "")).items())
-            rprovides.add(pkg)
-
-            if "PKG" in pkg_data:
-                pkg = pkg_data["PKG"]
-                rprovides.add(pkg)
-
-            for r in rprovides:
-                providers[r] = (pkg, dep_hashfn)
-
-    return providers
-
-collect_package_providers[vardepsexclude] += "BB_TASKDEPDATA"
-
 python do_create_runtime_spdx() {
     from datetime import datetime, timezone
     import oe.sbom
@@ -800,7 +615,7 @@ python do_create_runtime_spdx() {
 
             runtime_doc = oe.spdx.SPDXDocument()
             runtime_doc.name = "runtime-" + pkg_name
-            runtime_doc.documentNamespace = get_doc_namespace(localdata, runtime_doc)
+            runtime_doc.documentNamespace = get_namespace(localdata, runtime_doc.name)
             runtime_doc.creationInfo.created = creation_time
             runtime_doc.creationInfo.comment = "This document was created by analyzing package runtime dependencies."
             runtime_doc.creationInfo.licenseListVersion = d.getVar("SPDX_LICENSE_DATA")["licenseListVersion"]
@@ -891,60 +706,6 @@ do_create_runtime_spdx[dirs] = "${SPDXRUNTIMEDEPLOY}"
 do_create_runtime_spdx[cleandirs] = "${SPDXRUNTIMEDEPLOY}"
 do_create_runtime_spdx[rdeptask] = "do_create_spdx"
 
-def spdx_get_src(d):
-    """
-    save patched source of the recipe in SPDX_WORKDIR.
-    """
-    import shutil
-    spdx_workdir = d.getVar('SPDXWORK')
-    spdx_sysroot_native = d.getVar('STAGING_DIR_NATIVE')
-    pn = d.getVar('PN')
-
-    workdir = d.getVar("WORKDIR")
-
-    try:
-        # The kernel class functions require it to be on work-shared, so we dont change WORKDIR
-        if not is_work_shared_spdx(d):
-            # Change the WORKDIR to make do_unpack do_patch run in another dir.
-            d.setVar('WORKDIR', spdx_workdir)
-            # Restore the original path to recipe's native sysroot (it's relative to WORKDIR).
-            d.setVar('STAGING_DIR_NATIVE', spdx_sysroot_native)
-
-            # The changed 'WORKDIR' also caused 'B' changed, create dir 'B' for the
-            # possibly requiring of the following tasks (such as some recipes's
-            # do_patch required 'B' existed).
-            bb.utils.mkdirhier(d.getVar('B'))
-
-            bb.build.exec_func('do_unpack', d)
-        # Copy source of kernel to spdx_workdir
-        if is_work_shared_spdx(d):
-            share_src = d.getVar('WORKDIR')
-            d.setVar('WORKDIR', spdx_workdir)
-            d.setVar('STAGING_DIR_NATIVE', spdx_sysroot_native)
-            src_dir = spdx_workdir + "/" + d.getVar('PN')+ "-" + d.getVar('PV') + "-" + d.getVar('PR')
-            bb.utils.mkdirhier(src_dir)
-            if bb.data.inherits_class('kernel',d):
-                share_src = d.getVar('STAGING_KERNEL_DIR')
-            cmd_copy_share = "cp -rf " + share_src + "/* " + src_dir + "/"
-            cmd_copy_shared_res = os.popen(cmd_copy_share).read()
-            bb.note("cmd_copy_shared_result = " + cmd_copy_shared_res)
-
-            git_path = src_dir + "/.git"
-            if os.path.exists(git_path):
-                shutils.rmtree(git_path)
-
-        # Make sure gcc and kernel sources are patched only once
-        if not (d.getVar('SRC_URI') == "" or is_work_shared_spdx(d)):
-            bb.build.exec_func('do_patch', d)
-
-        # Some userland has no source.
-        if not os.path.exists( spdx_workdir ):
-            bb.utils.mkdirhier(spdx_workdir)
-    finally:
-        d.setVar("WORKDIR", workdir)
-
-spdx_get_src[vardepsexclude] += "STAGING_KERNEL_DIR"
-
 do_rootfs[recrdeptask] += "do_create_spdx do_create_runtime_spdx"
 do_rootfs[cleandirs] += "${SPDXIMAGEWORK}"
 
@@ -1019,7 +780,7 @@ def combine_spdx(d, rootfs_name, rootfs_deploydir, rootfs_spdxid, packages, spdx
 
     doc = oe.spdx.SPDXDocument()
     doc.name = rootfs_name
-    doc.documentNamespace = get_doc_namespace(d, doc)
+    doc.documentNamespace = get_namespace(d, doc.name)
     doc.creationInfo.created = creation_time
     doc.creationInfo.comment = "This document was created by analyzing the source of the Yocto recipe during the build."
     doc.creationInfo.licenseListVersion = d.getVar("SPDX_LICENSE_DATA")["licenseListVersion"]
meta/classes/spdx-common.bbclass new file +256 -0

256 lines not on this page · raw message

Signature · 1 line
2.43.2
#4 lore raw

Re: [OE-core][PATCH 2/6] classes/spdx-common: Move common SPDX to new class

Quoted 378 lines · Joshua Watt via lists.openembedded.org

Moves SPDX code that can be shared between different SPDX versions into
a common class

---
 meta/classes/create-spdx-2.2.bbclass | 259 ++----------------------- --
 meta/classes/spdx-common.bbclass     | 256
++++++++++++++++++++++++++

 2 files changed, 266 insertions(+), 249 deletions(-)

 create mode 100644 meta/classes/spdx-common.bbclass

meta/classes/create-spdx-2.2.bbclass +0 -0
index 7c8a0b8b0f7..94a172fbc94 100644
@@ -4,65 +4,13 @@
  # SPDX-License-Identifier: GPL-2.0-only
  #

meta/classes/create-spdx-2.2.bbclass +0 -3
-DEPLOY_DIR_SPDX ??= "${DEPLOY_DIR}/spdx"
-
-# The product name that the CVE database uses.  Defaults to BPN, but

may need to
-# be overriden per recipe (for example tiff.bb sets CVE_PRODUCT=libtiff).

meta/classes/create-spdx-2.2.bbclass +0 -30
-CVE_PRODUCT ??= "${BPN}"
-CVE_VERSION ??= "${PV}"
-
-SPDXDIR ??= "${WORKDIR}/spdx"
-SPDXDEPLOY = "${SPDXDIR}/deploy"
-SPDXWORK = "${SPDXDIR}/work"
-SPDXIMAGEWORK = "${SPDXDIR}/image-work"
-SPDXSDKWORK = "${SPDXDIR}/sdk-work"
-SPDXDEPS = "${SPDXDIR}/deps.json"
-
-SPDX_TOOL_NAME ??= "oe-spdx-creator"
-SPDX_TOOL_VERSION ??= "1.0"
-
-SPDXRUNTIMEDEPLOY = "${SPDXDIR}/runtime-deploy"
-
-SPDX_INCLUDE_SOURCES ??= "0"
-SPDX_ARCHIVE_SOURCES ??= "0"
-SPDX_ARCHIVE_PACKAGED ??= "0"
-
-SPDX_UUID_NAMESPACE ??= "sbom.openembedded.org"
-SPDX_NAMESPACE_PREFIX ??= "http://spdx.org/spdxdocs"
-SPDX_PRETTY ??= "0"
-
-SPDX_LICENSES ??= "${COREBASE}/meta/files/spdx-licenses.json"
-
-SPDX_CUSTOM_ANNOTATION_VARS ??= ""
-
-SPDX_ORG ??= "OpenEmbedded ()"
-SPDX_SUPPLIER ??= "Organization: ${SPDX_ORG}"
-SPDX_SUPPLIER[doc] = "The SPDX PackageSupplier field for SPDX

packages created from \
-    this recipe. For SPDX documents create using this class during the build, this \
-    is the contact information for the person or organization who is doing the \
-    build."

meta/classes/create-spdx-2.2.bbclass +0 -5
-
-def extract_licenses(filename):
-    import re
-
-    lic_regex = re.compile(rb'^\W*SPDX-License-Identifier:\s*([

\w\d.()+-]+?)(?:\s+\W*)?$', re.MULTILINE)
-
-    try:
-        with open(filename, 'rb') as f:
-            size = min(15000, os.stat(filename).st_size) -            txt = f.read(size)
-            licenses = re.findall(lic_regex, txt)
-            if licenses:
-                ascii_licenses = [lic.decode('ascii') for lic in licenses]
-                return ascii_licenses
-    except Exception as e:
-        bb.warn(f"Exception reading {filename}: {e}") -    return None

meta/classes/create-spdx-2.2.bbclass +3 -2
-
-def get_doc_namespace(d, doc):
+inherit spdx-common
+
+def get_namespace(d, name):

     import uuid
     namespace_uuid = uuid.uuid5(uuid.NAMESPACE_DNS, d.getVar("SPDX_UUID_NAMESPACE"))
-    return "%s/%s-%s" % (d.getVar("SPDX_NAMESPACE_PREFIX"), doc.name, str(uuid.uuid5(namespace_uuid, doc.name)))
+    return "%s/%s-%s" % (d.getVar("SPDX_NAMESPACE_PREFIX"), name, str(uuid.uuid5(namespace_uuid, name)))
+

 def create_annotation(d, comment):
     from datetime import datetime, timezone

meta/classes/create-spdx-2.2.bbclass +0 -0
@@ -80,26 +28,6 @@ def recipe_spdx_is_native(d, recipe):
        a.annotator == "Tool: %s - %s" % (d.getVar("SPDX_TOOL_NAME"),

d.getVar("SPDX_TOOL_VERSION")) and
       a.comment == "isNative" for a in recipe.annotations)
-def is_work_shared_spdx(d):
-    return bb.data.inherits_class('kernel', d) or ('work-shared' in d.getVar('WORKDIR'))

meta/classes/create-spdx-2.2.bbclass +0 -15
-
-def get_json_indent(d):
-    if d.getVar("SPDX_PRETTY") == "1":
-        return 2
-    return None
-
-python() {
-    import json
-    if d.getVar("SPDX_LICENSE_DATA"):
-        return
-
-    with open(d.getVar("SPDX_LICENSES"), "r") as f:
-        data = json.load(f)
-        # Transform the license array to a dictionary
-        data["licenses"] = {l["licenseId"]: l for l in

data["licenses"]}
-        d.setVar("SPDX_LICENSE_DATA", data)

meta/classes/create-spdx-2.2.bbclass +0 -2
-}
-

 def convert_license_to_spdx(lic, document, d, existing={}):
     from pathlib import Path
     import oe.spdx

meta/classes/create-spdx-2.2.bbclass +0 -0
@@ -172,34 +100,6 @@ def convert_license_to_spdx(lic, document, d,

existing={}):

     return ' '.join(convert(l) for l in lic_split)

meta/classes/create-spdx-2.2.bbclass +0 -10
-def process_sources(d):
-    pn = d.getVar('PN')
-    assume_provided = (d.getVar("ASSUME_PROVIDED") or "").split()
-    if pn in assume_provided:
-        for p in d.getVar("PROVIDES").split():
-            if p != pn:
-                pn = p
-                break
-
-    # glibc-locale: do_fetch, do_unpack and do_patch tasks have been

deleted,
-    # so avoid archiving source here.
-    if pn.startswith('glibc-locale'):
-        return False
-    if d.getVar('PN') == "libtool-cross":
-        return False
-    if d.getVar('PN') == "libgcc-initial":
-        return False
-    if d.getVar('PN') == "shadow-sysroot":
-        return False
-
-    # We just archive gcc-source for all the gcc related recipes -    if d.getVar('BPN') in ['gcc', 'libgcc']:
-        bb.debug(1, 'spdx: There is bug in scan of %s is, do nothing' % pn)
-        return False

meta/classes/create-spdx-2.2.bbclass +0 -4
-
-    return True
-
-

 def add_package_files(d, doc, spdx_pkg, topdir, get_spdxid, get_types, *, archive=None, ignore_dirs=[], ignore_top_level_dirs=[]):
     from pathlib import Path
     import oe.spdx

meta/classes/create-spdx-2.2.bbclass +0 -0
@@ -348,14 +248,12 @@ def collect_dep_recipes(d, doc, spdx_recipe):

     import oe.spdx

     deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX")) -    spdx_deps_file = Path(d.getVar("SPDXDEPS")) package_archs = d.getVar("SSTATE_ARCHS").split() package_archs.reverse()

     dep_recipes = []

-    with spdx_deps_file.open("r") as f:
-        deps = json.load(f)
+    deps = get_spdx_deps(d)

     for dep_pn, dep_hashfn, in_taskhash in deps:
         # If this dependency is not calculated in the taskhash skip it.

meta/classes/create-spdx-2.2.bbclass +0 -0
@@ -468,51 +366,6 @@ def add_download_packages(d, doc, recipe):
              # but this should be sufficient for now
              doc.add_relationship(package, "BUILD_DEPENDENCY_OF",
 recipe)

meta/classes/create-spdx-2.2.bbclass +0 -11
-def collect_direct_deps(d, dep_task):
-    current_task = "do_" + d.getVar("BB_CURRENTTASK")
-    pn = d.getVar("PN")
-
-    taskdepdata = d.getVar("BB_TASKDEPDATA", False)
-
-    for this_dep in taskdepdata.values():
-        if this_dep[0] == pn and this_dep[1] == current_task:
-            break
-    else:
-        bb.fatal(f"Unable to find this {pn}:{current_task} in

taskdepdata")
-
-    deps = set()
-    for dep_name in this_dep[3]:
-        dep_data = taskdepdata[dep_name]
-        if dep_data[1] == dep_task and dep_data[0] != pn:
-            deps.add((dep_data[0], dep_data[7], dep_name in this_dep[8]))

meta/classes/create-spdx-2.2.bbclass +0 -8
-
-    return sorted(deps)
-
-collect_direct_deps[vardepsexclude] += "BB_TASKDEPDATA"
-collect_direct_deps[vardeps] += "DEPENDS"
-
-python do_collect_spdx_deps() {
-    # This task calculates the build time dependencies of the

recipe, and is
-    # required because while a task can deptask on itself, those dependencies
-    # do not show up in BB_TASKDEPDATA. To work around that, this task does the
-    # deptask on do_create_spdx and writes out the dependencies it finds, then
-    # do_create_spdx reads in the found dependencies when writing the actual
-    # SPDX document
-    import json
-    from pathlib import Path

meta/classes/create-spdx-2.2.bbclass +0 -9
-
-    spdx_deps_file = Path(d.getVar("SPDXDEPS"))
-
-    deps = collect_direct_deps(d, "do_create_spdx")
-
-    with spdx_deps_file.open("w") as f:
-        json.dump(deps, f)
-}
-# NOTE: depending on do_unpack is a hack that is necessary to get

it's dependencies for archive the source

meta/classes/create-spdx-2.2.bbclass +0 -4
-addtask do_collect_spdx_deps after do_unpack
-do_collect_spdx_deps[depends] += "${PATCHDEPENDENCY}"
-do_collect_spdx_deps[deptask] = "do_create_spdx"
-do_collect_spdx_deps[dirs] = "${SPDXDIR}"


 python do_create_spdx() {
     from datetime import datetime, timezone

meta/classes/create-spdx-2.2.bbclass +0 -0
@@ -551,7 +404,7 @@ python do_create_spdx() {
      doc = oe.spdx.SPDXDocument()


     doc.name = "recipe-" + d.getVar("PN")
-    doc.documentNamespace = get_doc_namespace(d, doc) +    doc.documentNamespace = get_namespace(d, doc.name) doc.creationInfo.created = creation_time doc.creationInfo.comment = "This document was created by analyzing recipe files during the build." doc.creationInfo.licenseListVersion = d.getVar("SPDX_LICENSE_DATA")["licenseListVersion"]

meta/classes/create-spdx-2.2.bbclass +0 -0
@@ -655,7 +508,7 @@ python do_create_spdx() {
              package_doc = oe.spdx.SPDXDocument()

             pkg_name = d.getVar("PKG:%s" % package) or package package_doc.name = pkg_name
-            package_doc.documentNamespace = get_doc_namespace(d, package_doc)
+            package_doc.documentNamespace = get_namespace(d, package_doc.name)
             package_doc.creationInfo.created = creation_time package_doc.creationInfo.comment = "This document was created by analyzing packages created during the build." package_doc.creationInfo.licenseListVersion = d.getVar("SPDX_LICENSE_DATA")["licenseListVersion"]

meta/classes/create-spdx-2.2.bbclass +0 -0
@@ -716,44 +569,6 @@ do_create_spdx[dirs] = "${SPDXWORK}"

 do_create_spdx[cleandirs] = "${SPDXDEPLOY} ${SPDXWORK}" do_create_spdx[depends] += "${PATCHDEPENDENCY}"

meta/classes/create-spdx-2.2.bbclass +0 -20
-def collect_package_providers(d):
-    from pathlib import Path
-    import oe.sbom
-    import oe.spdx
-    import json
-
-    deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX"))
-
-    providers = {}
-
-    deps = collect_direct_deps(d, "do_create_spdx")
-    deps.append((d.getVar("PN"), d.getVar("BB_HASHFILENAME"), True))
-
-    for dep_pn, dep_hashfn, _ in deps:
-        localdata = d
-        recipe_data = oe.packagedata.read_pkgdata(dep_pn, localdata)
-        if not recipe_data:
-            localdata = bb.data.createCopy(d)
-            localdata.setVar("PKGDATA_DIR", "${PKGDATA_DIR_SDK}")
-            recipe_data = oe.packagedata.read_pkgdata(dep_pn,

localdata)

meta/classes/create-spdx-2.2.bbclass +0 -4
-
-        for pkg in recipe_data.get("PACKAGES", "").split():
-
-            pkg_data = oe.packagedata.read_subpkgdata_dict(pkg,

localdata)
-            rprovides = set(n for n, _ in bb.utils.explode_dep_versions2(pkg_data.get("RPROVIDES", "")).items())
-            rprovides.add(pkg)

meta/classes/create-spdx-2.2.bbclass +0 -12
-
-            if "PKG" in pkg_data:
-                pkg = pkg_data["PKG"]
-                rprovides.add(pkg)
-
-            for r in rprovides:
-                providers[r] = (pkg, dep_hashfn)
-
-    return providers
-
-collect_package_providers[vardepsexclude] += "BB_TASKDEPDATA"
-

 python do_create_runtime_spdx() {
     from datetime import datetime, timezone
     import oe.sbom

meta/classes/create-spdx-2.2.bbclass +0 -0
@@ -800,7 +615,7 @@ python do_create_runtime_spdx() {


             runtime_doc = oe.spdx.SPDXDocument() runtime_doc.name = "runtime-" + pkg_name
-            runtime_doc.documentNamespace = get_doc_namespace(localdata, runtime_doc)
+            runtime_doc.documentNamespace = get_namespace(localdata, runtime_doc.name)
             runtime_doc.creationInfo.created = creation_time runtime_doc.creationInfo.comment = "This document was created by analyzing package runtime dependencies." runtime_doc.creationInfo.licenseListVersion = d.getVar("SPDX_LICENSE_DATA")["licenseListVersion"]

meta/classes/create-spdx-2.2.bbclass +0 -0
@@ -891,60 +706,6 @@ do_create_runtime_spdx[dirs] =

"${SPDXRUNTIMEDEPLOY}" do_create_runtime_spdx[cleandirs] = "${SPDXRUNTIMEDEPLOY}" do_create_runtime_spdx[rdeptask] = "do_create_spdx"

meta/classes/create-spdx-2.2.bbclass +0 -13
-def spdx_get_src(d):
-    """
-    save patched source of the recipe in SPDX_WORKDIR.
-    """
-    import shutil
-    spdx_workdir = d.getVar('SPDXWORK')
-    spdx_sysroot_native = d.getVar('STAGING_DIR_NATIVE')
-    pn = d.getVar('PN')
-
-    workdir = d.getVar("WORKDIR")
-
-    try:
-        # The kernel class functions require it to be on work-

shared, so we dont change WORKDIR
-        if not is_work_shared_spdx(d):
-            # Change the WORKDIR to make do_unpack do_patch run in another dir.
-            d.setVar('WORKDIR', spdx_workdir)
-            # Restore the original path to recipe's native sysroot (it's relative to WORKDIR).
-            d.setVar('STAGING_DIR_NATIVE', spdx_sysroot_native) -
-            # The changed 'WORKDIR' also caused 'B' changed, create dir 'B' for the
-            # possibly requiring of the following tasks (such as some recipes's
-            # do_patch required 'B' existed).
-            bb.utils.mkdirhier(d.getVar('B'))
-
-            bb.build.exec_func('do_unpack', d)
-        # Copy source of kernel to spdx_workdir
-        if is_work_shared_spdx(d):
-            share_src = d.getVar('WORKDIR')
-            d.setVar('WORKDIR', spdx_workdir)
-            d.setVar('STAGING_DIR_NATIVE', spdx_sysroot_native) -            src_dir = spdx_workdir + "/" + d.getVar('PN')+ "-" + d.getVar('PV') + "-" + d.getVar('PR')
-            bb.utils.mkdirhier(src_dir)
-            if bb.data.inherits_class('kernel',d):
-                share_src = d.getVar('STAGING_KERNEL_DIR')

Quoted 8 lines

-            cmd_copy_share = "cp -rf " + share_src + "/* " + src_dir
+ "/"
-            cmd_copy_shared_res = os.popen(cmd_copy_share).read() -            bb.note("cmd_copy_shared_result = " + cmd_copy_shared_res)
-
-            git_path = src_dir + "/.git"
-            if os.path.exists(git_path):

import os.path is missing

Quoted 1 line

-                shutils.rmtree(git_path)

Should be: shutil.rmtree(git_path)

Thank you for SPDX 3.0 support.
Adrian

Quoted 336 lines

-
-        # Make sure gcc and kernel sources are patched only once -        if not (d.getVar('SRC_URI') == "" or is_work_shared_spdx(d)):
-            bb.build.exec_func('do_patch', d)

meta/classes/create-spdx-2.2.bbclass +0 -9
-
-        # Some userland has no source.
-        if not os.path.exists( spdx_workdir ):
-            bb.utils.mkdirhier(spdx_workdir)
-    finally:
-        d.setVar("WORKDIR", workdir)
-
-spdx_get_src[vardepsexclude] += "STAGING_KERNEL_DIR"
-

 do_rootfs[recrdeptask] += "do_create_spdx do_create_runtime_spdx" do_rootfs[cleandirs] += "${SPDXIMAGEWORK}"

meta/classes/create-spdx-2.2.bbclass +0 -0
@@ -1019,7 +780,7 @@ def combine_spdx(d, rootfs_name,

rootfs_deploydir, rootfs_spdxid, packages, spdx

     doc = oe.spdx.SPDXDocument()
     doc.name = rootfs_name
-    doc.documentNamespace = get_doc_namespace(d, doc) +    doc.documentNamespace = get_namespace(d, doc.name) doc.creationInfo.created = creation_time doc.creationInfo.comment = "This document was created by analyzing the source of the Yocto recipe during the build." doc.creationInfo.licenseListVersion = d.getVar("SPDX_LICENSE_DATA")["licenseListVersion"]

meta/classes/spdx- renamed from meta/classes/spdx-common.bbclass +0 -0

common.bbclass
new file mode 100644
index 00000000000..468a11ca3e2

meta/classes/spdx-common.bbclass new file +9 -0
@@ -0,0 +1,256 @@
+#
+# Copyright OpenEmbedded Contributors
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+
+DEPLOY_DIR_SPDX ??= "${DEPLOY_DIR}/spdx"
+
+# The product name that the CVE database uses.  Defaults to BPN, but

may need to
+# be overriden per recipe (for example tiff.bb sets CVE_PRODUCT=libtiff).

meta/classes/spdx-common.bbclass new file +30 -0
+CVE_PRODUCT ??= "${BPN}"
+CVE_VERSION ??= "${PV}"
+
+SPDXDIR ??= "${WORKDIR}/spdx"
+SPDXDEPLOY = "${SPDXDIR}/deploy"
+SPDXWORK = "${SPDXDIR}/work"
+SPDXIMAGEWORK = "${SPDXDIR}/image-work"
+SPDXSDKWORK = "${SPDXDIR}/sdk-work"
+SPDXDEPS = "${SPDXDIR}/deps.json"
+
+SPDX_TOOL_NAME ??= "oe-spdx-creator"
+SPDX_TOOL_VERSION ??= "1.0"
+
+SPDXRUNTIMEDEPLOY = "${SPDXDIR}/runtime-deploy"
+
+SPDX_INCLUDE_SOURCES ??= "0"
+SPDX_ARCHIVE_SOURCES ??= "0"
+SPDX_ARCHIVE_PACKAGED ??= "0"
+
+SPDX_UUID_NAMESPACE ??= "sbom.openembedded.org"
+SPDX_NAMESPACE_PREFIX ??= "http://spdx.org/spdxdocs"
+SPDX_PRETTY ??= "0"
+
+SPDX_LICENSES ??= "${COREBASE}/meta/files/spdx-licenses.json"
+
+SPDX_CUSTOM_ANNOTATION_VARS ??= ""
+
+SPDX_ORG ??= "OpenEmbedded ()"
+SPDX_SUPPLIER ??= "Organization: ${SPDX_ORG}"
+SPDX_SUPPLIER[doc] = "The SPDX PackageSupplier field for SPDX

packages created from \
+    this recipe. For SPDX documents create using this class during the build, this \
+    is the contact information for the person or organization who is doing the \
+    build."

meta/classes/spdx-common.bbclass new file +5 -0
+
+def extract_licenses(filename):
+    import re
+
+    lic_regex = re.compile(rb'^\W*SPDX-License-Identifier:\s*([

\w\d.()+-]+?)(?:\s+\W*)?$', re.MULTILINE)
+
+    try:
+        with open(filename, 'rb') as f:
+            size = min(15000, os.stat(filename).st_size) +            txt = f.read(size)
+            licenses = re.findall(lic_regex, txt)
+            if licenses:
+                ascii_licenses = [lic.decode('ascii') for lic in licenses]
+                return ascii_licenses
+    except Exception as e:
+        bb.warn(f"Exception reading {filename}: {e}") +    return None

meta/classes/spdx-common.bbclass new file +3 -0
+
+def is_work_shared_spdx(d):
+    return bb.data.inherits_class('kernel', d) or ('work-shared' in

d.getVar('WORKDIR'))

meta/classes/spdx-common.bbclass new file +15 -0
+
+def get_json_indent(d):
+    if d.getVar("SPDX_PRETTY") == "1":
+        return 2
+    return None
+
+python() {
+    import json
+    if d.getVar("SPDX_LICENSE_DATA"):
+        return
+
+    with open(d.getVar("SPDX_LICENSES"), "r") as f:
+        data = json.load(f)
+        # Transform the license array to a dictionary
+        data["licenses"] = {l["licenseId"]: l for l in

data["licenses"]}
+        d.setVar("SPDX_LICENSE_DATA", data)

meta/classes/spdx-common.bbclass new file +12 -0
+}
+
+def process_sources(d):
+    pn = d.getVar('PN')
+    assume_provided = (d.getVar("ASSUME_PROVIDED") or "").split()
+    if pn in assume_provided:
+        for p in d.getVar("PROVIDES").split():
+            if p != pn:
+                pn = p
+                break
+
+    # glibc-locale: do_fetch, do_unpack and do_patch tasks have been

deleted,
+    # so avoid archiving source here.
+    if pn.startswith('glibc-locale'):
+        return False
+    if d.getVar('PN') == "libtool-cross":
+        return False
+    if d.getVar('PN') == "libgcc-initial":
+        return False
+    if d.getVar('PN') == "shadow-sysroot":
+        return False
+
+    # We just archive gcc-source for all the gcc related recipes +    if d.getVar('BPN') in ['gcc', 'libgcc']:
+        bb.debug(1, 'spdx: There is bug in scan of %s is, do nothing' % pn)
+        return False

meta/classes/spdx-common.bbclass new file +14 -0
+
+    return True
+
+def collect_direct_deps(d, dep_task):
+    current_task = "do_" + d.getVar("BB_CURRENTTASK")
+    pn = d.getVar("PN")
+
+    taskdepdata = d.getVar("BB_TASKDEPDATA", False)
+
+    for this_dep in taskdepdata.values():
+        if this_dep[0] == pn and this_dep[1] == current_task:
+            break
+    else:
+        bb.fatal(f"Unable to find this {pn}:{current_task} in

taskdepdata")

meta/classes/spdx-common.bbclass new file +7 -0
+
+    deps = set()
+
+    for dep_name in this_dep.deps:
+        dep_data = taskdepdata[dep_name]
+        if dep_data.taskname == dep_task and dep_data.pn != pn:
+            deps.add((dep_data.pn, dep_data.hashfn, dep_name in

this_dep.taskhash_deps))

meta/classes/spdx-common.bbclass new file +8 -0
+
+    return sorted(deps)
+
+collect_direct_deps[vardepsexclude] += "BB_TASKDEPDATA"
+collect_direct_deps[vardeps] += "DEPENDS"
+
+python do_collect_spdx_deps() {
+    # This task calculates the build time dependencies of the

recipe, and is
+    # required because while a task can deptask on itself, those dependencies
+    # do not show up in BB_TASKDEPDATA. To work around that, this task does the
+    # deptask on do_create_spdx and writes out the dependencies it finds, then
+    # do_create_spdx reads in the found dependencies when writing the actual
+    # SPDX document
+    import json
+    from pathlib import Path

meta/classes/spdx-common.bbclass new file +9 -0
+
+    spdx_deps_file = Path(d.getVar("SPDXDEPS"))
+
+    deps = collect_direct_deps(d, "do_create_spdx")
+
+    with spdx_deps_file.open("w") as f:
+        json.dump(deps, f)
+}
+# NOTE: depending on do_unpack is a hack that is necessary to get

it's dependencies for archive the source

meta/classes/spdx-common.bbclass new file +34 -0
+addtask do_collect_spdx_deps after do_unpack
+do_collect_spdx_deps[depends] += "${PATCHDEPENDENCY}"
+do_collect_spdx_deps[deptask] = "do_create_spdx"
+do_collect_spdx_deps[dirs] = "${SPDXDIR}"
+
+def get_spdx_deps(d):
+    import json
+    from pathlib import Path
+
+    spdx_deps_file = Path(d.getVar("SPDXDEPS"))
+
+    with spdx_deps_file.open("r") as f:
+        return json.load(f)
+
+def collect_package_providers(d):
+    from pathlib import Path
+    import oe.sbom
+    import oe.spdx
+    import json
+
+    deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX"))
+
+    providers = {}
+
+    deps = collect_direct_deps(d, "do_create_spdx")
+    deps.append((d.getVar("PN"), d.getVar("BB_HASHFILENAME"), True))
+
+    for dep_pn, dep_hashfn, _ in deps:
+        localdata = d
+        recipe_data = oe.packagedata.read_pkgdata(dep_pn, localdata)
+        if not recipe_data:
+            localdata = bb.data.createCopy(d)
+            localdata.setVar("PKGDATA_DIR", "${PKGDATA_DIR_SDK}")
+            recipe_data = oe.packagedata.read_pkgdata(dep_pn,

localdata)

meta/classes/spdx-common.bbclass new file +4 -0
+
+        for pkg in recipe_data.get("PACKAGES", "").split():
+
+            pkg_data = oe.packagedata.read_subpkgdata_dict(pkg,

localdata)
+            rprovides = set(n for n, _ in bb.utils.explode_dep_versions2(pkg_data.get("RPROVIDES", "")).items())
+            rprovides.add(pkg)

meta/classes/spdx-common.bbclass new file +25 -0
+
+            if "PKG" in pkg_data:
+                pkg = pkg_data["PKG"]
+                rprovides.add(pkg)
+
+            for r in rprovides:
+                providers[r] = (pkg, dep_hashfn)
+
+    return providers
+
+collect_package_providers[vardepsexclude] += "BB_TASKDEPDATA"
+
+def spdx_get_src(d):
+    """
+    save patched source of the recipe in SPDX_WORKDIR.
+    """
+    import shutil
+    spdx_workdir = d.getVar('SPDXWORK')
+    spdx_sysroot_native = d.getVar('STAGING_DIR_NATIVE')
+    pn = d.getVar('PN')
+
+    workdir = d.getVar("WORKDIR")
+
+    try:
+        # The kernel class functions require it to be on work-

shared, so we dont change WORKDIR
+        if not is_work_shared_spdx(d):
+            # Change the WORKDIR to make do_unpack do_patch run in another dir.
+            d.setVar('WORKDIR', spdx_workdir)
+            # Restore the original path to recipe's native sysroot (it's relative to WORKDIR).
+            d.setVar('STAGING_DIR_NATIVE', spdx_sysroot_native) +
+            # The changed 'WORKDIR' also caused 'B' changed, create dir 'B' for the
+            # possibly requiring of the following tasks (such as some recipes's
+            # do_patch required 'B' existed).
+            bb.utils.mkdirhier(d.getVar('B'))
+
+            bb.build.exec_func('do_unpack', d)
+        # Copy source of kernel to spdx_workdir
+        if is_work_shared_spdx(d):
+            share_src = d.getVar('WORKDIR')
+            d.setVar('WORKDIR', spdx_workdir)
+            d.setVar('STAGING_DIR_NATIVE', spdx_sysroot_native) +            src_dir = spdx_workdir + "/" + d.getVar('PN')+ "-" + d.getVar('PV') + "-" + d.getVar('PR')
+            bb.utils.mkdirhier(src_dir)
+            if bb.data.inherits_class('kernel',d):
+                share_src = d.getVar('STAGING_KERNEL_DIR') +            cmd_copy_share = "cp -rf " + share_src + "/* " + src_dir
+ "/"
+            cmd_copy_shared_res = os.popen(cmd_copy_share).read() +            bb.note("cmd_copy_shared_result = " + cmd_copy_shared_res)

meta/classes/spdx-common.bbclass new file +7 -0
+
+            git_path = src_dir + "/.git"
+            if os.path.exists(git_path):
+                shutils.rmtree(git_path)
+
+        # Make sure gcc and kernel sources are patched only once
+        if not (d.getVar('SRC_URI') == "" or

is_work_shared_spdx(d)):
+            bb.build.exec_func('do_patch', d)

meta/classes/spdx-common.bbclass new file +9 -1
+
+        # Some userland has no source.
+        if not os.path.exists( spdx_workdir ):
+            bb.utils.mkdirhier(spdx_workdir)
+    finally:
+        d.setVar("WORKDIR", workdir)
+
+spdx_get_src[vardepsexclude] += "STAGING_KERNEL_DIR"
+
 
-=-=-=-=-=-=-=-=-=-=-=-

Links: You receive all messages sent to this group. View/Reply Online (#200486):
https://lists.openembedded.org/g/openembedded-core/message/200486
Mute This Topic: https://lists.openembedded.org/mt/106602516/4454582
Group Owner: [email protected] Unsubscribe:
https://lists.openembedded.org/g/openembedded-core/unsub [
[email protected]]
-=-=-=-=-=-=-=-=-=-=-=-

#5 lore raw

PATCH 3/6 classes/spdx-common: Add SPDX version to path

In reply to Joshua Watt

Since multiple versions of SPDX are available, use the version in the path

---
 meta/classes/create-spdx-2.2.bbclass | 2 ++
 meta/classes/spdx-common.bbclass     | 5 +++--
 2 files changed, 5 insertions(+), 2 deletions(-)
meta/classes/create-spdx-2.2.bbclass +2 -0
index 94a172fbc94..3ebf92b5e12 100644
@@ -6,6 +6,8 @@
 
 inherit spdx-common
 
+SPDX_VERSION = "2.2"
+
 def get_namespace(d, name):
     import uuid
     namespace_uuid = uuid.uuid5(uuid.NAMESPACE_DNS, d.getVar("SPDX_UUID_NAMESPACE"))
meta/classes/spdx-common.bbclass +3 -2
index 468a11ca3e2..f6fc182838d 100644
@@ -4,14 +4,15 @@
 # SPDX-License-Identifier: GPL-2.0-only
 #
 
-DEPLOY_DIR_SPDX ??= "${DEPLOY_DIR}/spdx"
+SPDX_VERSION ??= ""
+DEPLOY_DIR_SPDX ??= "${DEPLOY_DIR}/spdx/${SPDX_VERSION}"
 
 # The product name that the CVE database uses.  Defaults to BPN, but may need to
 # be overriden per recipe (for example tiff.bb sets CVE_PRODUCT=libtiff).
 CVE_PRODUCT ??= "${BPN}"
 CVE_VERSION ??= "${PV}"
 
-SPDXDIR ??= "${WORKDIR}/spdx"
+SPDXDIR ??= "${WORKDIR}/spdx/${SPDX_VERSION}"
 SPDXDEPLOY = "${SPDXDIR}/deploy"
 SPDXWORK = "${SPDXDIR}/work"
 SPDXIMAGEWORK = "${SPDXDIR}/image-work"
Signature · 1 line
2.43.2
#6 lore raw

PATCH 4/6 classes/spdx-common: Return empty list from extract_licenses

In reply to Joshua Watt

This is nicer as the normal return type is a list, so the calling code doesn't have to deal with a None sometimes and a list others.

---
 meta/classes/spdx-common.bbclass | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
meta/classes/spdx-common.bbclass +1 -1
index f6fc182838d..03f1d0cc278 100644
@@ -58,7 +58,7 @@ def extract_licenses(filename):
                 return ascii_licenses
     except Exception as e:
         bb.warn(f"Exception reading {filename}: {e}")
-    return None
+    return []
 
 def is_work_shared_spdx(d):
     return bb.data.inherits_class('kernel', d) or ('work-shared' in d.getVar('WORKDIR'))
Signature · 1 line
2.43.2
#7 lore raw

PATCH 5/6 classes/create-spdx-3.0: Add class

In reply to Joshua Watt

Adds a class to generate SPDX 3.0 output.

---
 meta/classes/create-spdx-3.0.bbclass | 1284 ++++++
 meta/classes/spdx-common.bbclass     |   25 +-
 meta/lib/oe/sbom30.py                |  993 +++++
 meta/lib/oe/spdx30.py                | 5413 ++++++++++++++++++++++++++
 4 files changed, 7714 insertions(+), 1 deletion(-)
 create mode 100644 meta/classes/create-spdx-3.0.bbclass
 create mode 100644 meta/lib/oe/sbom30.py
 create mode 100644 meta/lib/oe/spdx30.py
meta/classes/create-spdx-3.0.bbclass new file +1284 -0
new file mode 100644
index 00000000000..cdbf80b8c1e
@@ -0,0 +1,1284 @@
+#
+# Copyright OpenEmbedded Contributors
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+
+inherit spdx-common
+
+SPDX_VERSION = "3.0.0"
+
+# The list of SPDX profiles generated documents will conform to
+SPDX_PROFILES ?= "core build software simpleLicensing security"
+
+SPDX_INCLUDE_BUILD_VARIABLES ??= "0"
+SPDX_INCLUDE_BUILD_VARIABLES[doc] = "If set to '1', the bitbake variables for a \
+    recipe will be included in the Build object"
+
+SPDX_IMPORTS ??= ""
+SPDX_IMPORTS[doc] = "SPDX_IMPORTS is the base variable that describes how to \
+    reference external SPDX ids. Each import is defined as a key in this \
+    variable with a suffix to describe to as a suffix to look up more \
+    information about the import. Each key can have the following variables: \
+        SPDX_IMPORTS_<key>_spdxid: The Fully qualified SPDX ID of the object \
+        SPDX_IMPORTS_<key>_uri: The URI where the SPDX Document that contains \
+            the external object can be found. Optional but recommended \
+        SPDX_IMPORTS_<key>_hash_<hash>: The Checksum of the SPDX Document that \
+            contains the External ID. <hash> must be one the valid SPDX hashing \
+            algorithms, as described by the HashAlgorithm vocabulary in the\
+            SPDX 3 spec. Optional but recommended \
+"
+
+# Agents
+#   Bitbake variables can be used to describe an SPDX Agent that may be used
+#   during the build. An Agent is specified using a set of variables which all
+#   start with some common base name:
+#
+#   <BASE>_name: The name of the Agent (required)
+#   <BASE>_type: The type of Agent. Must be one of "person", "organization",
+#       "software", or "agent" (the default if not specified)
+#   <BASE>_comment: The comment for the Agent (optional)
+#   <BASE>_id_<ID>: And External Identifier for the Agent. <ID> must be a valid
+#       ExternalIdentifierType from the SPDX 3 spec. Commonly, an E-mail address
+#       can be specified with <BASE>_id_email
+#
+#   Alternatively, an Agent can be an external reference by referencing a key
+#   in SPDX_IMPORTS like so:
+#
+#   <BASE>_import = "<key>"
+#
+#   Finally, the same agent described by another set of agent variables can be
+#   referenced by specifying the basename of the variable that should be
+#   referenced:
+#
+#   SPDX_SUPPLIER_ref = "SPDX_AUTHORS_openembedded"
+
+SPDX_AUTHORS ??= "openembedded"
+SPDX_AUTHORS[doc] = "A space separated list of the document authors. Each item \
+    is used to name a base variable like SPDX_AUTHORS_<AUTHOR> that \
+    describes the author."
+
+SPDX_AUTHORS_openembedded_name = "OpenEmbedded"
+SPDX_AUTHORS_openembedded_type = "organization"
+
+SPDX_BUILD_HOST[doc] = "The base variable name to describe the build host on \
+    which a build is running. Must be an SPDX_IMPORTS key"
+
+SPDX_INVOKED_BY[doc] = "The base variable name to describe the Agent that \
+    invoked the build, which builds will link to if specified."
+
+SPDX_ON_BEHALF_OF[doc] = "The base variable name to describe the Agent on who's \
+    behalf the invoking Agent (SPDX_INVOKED_BY) is running the build."
+
+SPDX_SUPPLIER[doc] = "The base variable name to describe the Agent who is supplying \
+    artifacts produced by the build"
+
+SPDX_PACKAGE_ADDITIONAL_PURPOSE ?= ""
+SPDX_PACKAGE_ADDITIONAL_PURPOSE[doc] = "The list of additional purposes to assign to \
+    the generated packages for a recipe. The primary purpose is always `install`. \
+    Packages overrides are allowed to override the additional purposes for \
+    individual packages."
+
+SPDX_IMAGE_PURPOSE ?= "filesystemImage"
+SPDX_IMAGE_PURPOSE[doc] = "The list of purposes to assign to the generated images. \
+    The first listed item will be the Primary Purpose and all additional items will \
+    be added as additional purposes"
+
+SPDX_SDK_PURPOSE ?= "install"
+SPDX_SDK_PURPOSE[doc] = "The list of purposes to assign to the generate SDK installer. \
+    The first listed item will be the Primary Purpose and all additional items will \
+    be added as additional purposes"
+
+SPDX_HASH_ALGORITHMS ?= "sha256 sha1"
+
+
+def add_license_expression(d, objset, license_expression):
+    from pathlib import Path
+    import oe.spdx30
+    import oe.sbom30
+
+    license_data = d.getVar("SPDX_LICENSE_DATA")
+    simple_license_text = {}
+    license_text_map = {}
+    license_ref_idx = 0
+
+    def add_license_text(name):
+        nonlocal objset
+        nonlocal simple_license_text
+
+        if name in simple_license_text:
+            return name
+
+        lic = objset.find_filter(
+            oe.spdx30.simplelicensing_SimpleLicensingText,
+            name=name,
+        )
+
+        if lic is not None:
+            simple_license_text[name] = lic
+            return lic
+
+        lic = objset.add(oe.spdx30.simplelicensing_SimpleLicensingText(
+            _id=objset.new_spdxid("license-text", name),
+            creationInfo=objset.doc.creationInfo,
+            name=name,
+        ))
+        simple_license_text[name] = lic
+
+        if name == "PD":
+            lic.simplelicensing_licenseText = "Software released to the public domain"
+            return lic
+
+        # Seach for the license in COMMON_LICENSE_DIR and LICENSE_PATH
+        for directory in [d.getVar('COMMON_LICENSE_DIR')] + (d.getVar('LICENSE_PATH') or '').split():
+            try:
+                with (Path(directory) / name).open(errors="replace") as f:
+                    lic.simplelicensing_licenseText = f.read()
+                    return lic
+
+            except FileNotFoundError:
+                pass
+
+        # If it's not SPDX or PD, then NO_GENERIC_LICENSE must be set
+        filename = d.getVarFlag('NO_GENERIC_LICENSE', name)
+        if filename:
+            filename = d.expand("${S}/" + filename)
+            with open(filename, errors="replace") as f:
+                lic.simplelicensing_licenseText = f.read()
+                return lic
+        else:
+            bb.fatal("Cannot find any text for license %s" % name)
+
+    def convert(l):
+        nonlocal license_text_map
+        nonlocal license_ref_idx
+
+        if l == "(" or l == ")":
+            return l
+
+        if l == "&":
+            return "AND"
+
+        if l == "|":
+            return "OR"
+
+        if l == "CLOSED":
+            return "NONE"
+
+        spdx_license = d.getVarFlag("SPDXLICENSEMAP", l) or l
+        if spdx_license in license_data["licenses"]:
+            return spdx_license
+
+        spdx_license = "LicenseRef-" + l
+        license_text_map[spdx_license] = add_license_text(l)._id
+
+        return spdx_license
+
+    lic_split = license_expression.replace("(", " ( ").replace(")", " ) ").replace("|", " | ").replace("&", " & ").split()
+    spdx_license_expression = ' '.join(convert(l) for l in lic_split)
+
+    return objset.new_license_expression(spdx_license_expression, license_text_map)
+
+
+def add_package_files(d, objset, topdir, get_spdxid, get_purposes, *, archive=None, ignore_dirs=[], ignore_top_level_dirs=[]):
+    from pathlib import Path
+    import oe.spdx30
+    import oe.sbom30
+
+    license_cache = {}
+
+    source_date_epoch = d.getVar("SOURCE_DATE_EPOCH")
+    if source_date_epoch:
+        source_date_epoch = int(source_date_epoch)
+
+    spdx_files = set()
+
+    file_counter = 1
+    for subdir, dirs, files in os.walk(topdir):
+        dirs[:] = [d for d in dirs if d not in ignore_dirs]
+        if subdir == str(topdir):
+            dirs[:] = [d for d in dirs if d not in ignore_top_level_dirs]
+
+        for file in files:
+            filepath = Path(subdir) / file
+            if filepath.is_symlink() or not filepath.is_file():
+                continue
+
+            filename = str(filepath.relative_to(topdir))
+
+            spdx_file = oe.spdx30.software_File(
+                _id=get_spdxid(file_counter),
+                creationInfo=objset.doc.creationInfo,
+                name=filename,
+            )
+
+            file_purposes = get_purposes(filepath)
+            is_source = oe.spdx30.software_SoftwarePurpose.source in file_purposes
+            if file_purposes:
+                spdx_file.software_primaryPurpose = file_purposes[0]
+                spdx_file.software_additionalPurpose = file_purposes[1:]
+
+            objset.add(spdx_file)
+            spdx_files.add(spdx_file)
+
+            if archive is not None:
+                with filepath.open("rb") as f:
+                    info = archive.gettarinfo(fileobj=f)
+                    info.name = filename
+                    info.uid = 0
+                    info.gid = 0
+                    info.uname = "root"
+                    info.gname = "root"
+
+                    if source_date_epoch is not None and info.mtime > source_date_epoch:
+                        info.mtime = source_date_epoch
+
+                    archive.addfile(info, f)
+
+            spdx_file.verifiedUsing.append(
+                oe.spdx30.Hash(
+                    algorithm=oe.spdx30.HashAlgorithm.sha1,
+                    hashValue=bb.utils.sha1_file(filepath),
+                )
+            )
+
+            spdx_file.verifiedUsing.append(
+                oe.spdx30.Hash(
+                    algorithm=oe.spdx30.HashAlgorithm.sha256,
+                    hashValue=bb.utils.sha256_file(filepath),
+                )
+            )
+
+            bb.debug(1, "Adding file %s to %s" % (filepath, objset.doc._id))
+
+            if is_source:
+                file_licenses = set()
+                for extracted_lic in extract_licenses(filepath):
+                    file_licenses.add(objset.new_license_expression(extracted_lic))
+
+                if file_licenses:
+                    objset.new_relationship(
+                        [spdx_file],
+                        oe.spdx30.RelationshipType.hasDeclaredLicense,
+                        file_licenses
+                    )
+
+            file_counter += 1
+
+    return spdx_files
+
+
+def get_package_sources_from_debug(d, package, package_files, sources, source_hash_cache):
+    from pathlib import Path
+    import oe.packagedata
+
+    debug_search_paths = [
+        Path(d.getVar('PKGD')),
+        Path(d.getVar('STAGING_DIR_TARGET')),
+        Path(d.getVar('STAGING_DIR_NATIVE')),
+        Path(d.getVar('STAGING_KERNEL_DIR')),
+    ]
+
+    pkg_data = oe.packagedata.read_subpkgdata_extended(package, d)
+
+    if pkg_data is None:
+        return
+
+    dep_source_files = set()
+
+    for file_path, file_data in pkg_data["files_info"].items():
+        if not "debugsrc" in file_data:
+            continue
+
+        for pkg_file in package_files:
+            if file_path.lstrip("/") == pkg_file.name.lstrip("/"):
+                break
+        else:
+            bb.fatal("No package file found for %s in %s; SPDX found: %s" % (str(file_path), package,
+                " ".join(p.name for p in package_files)))
+            continue
+
+        for debugsrc in file_data["debugsrc"]:
+            for search in debug_search_paths:
+                if debugsrc.startswith("/usr/src/kernel"):
+                    debugsrc_path = search / debugsrc.replace('/usr/src/kernel/', '')
+                else:
+                    debugsrc_path = search / debugsrc.lstrip("/")
+
+                if debugsrc_path in source_hash_cache:
+                    file_sha256 = source_hash_cache[debugsrc_path]
+                    if file_sha256 is None:
+                        continue
+                else:
+                    if not debugsrc_path.exists():
+                        source_hash_cache[debugsrc_path] = None
+                        continue
+
+                    file_sha256 = bb.utils.sha256_file(debugsrc_path)
+                    source_hash_cache[debugsrc_path] = file_sha256
+
+                if file_sha256 in sources:
+                    source_file = sources[file_sha256]
+                    dep_source_files.add(source_file)
+                else:
+                    bb.debug(1, "Debug source %s with SHA256 %s not found in any dependency" % (str(debugsrc_path), file_sha256))
+                break
+            else:
+                bb.debug(1, "Debug source %s not found" % debugsrc)
+
+    return dep_source_files
+
+get_package_sources_from_debug[vardepsexclude] += "STAGING_KERNEL_DIR"
+
+def collect_dep_objsets(d, build):
+    import json
+    from pathlib import Path
+    import oe.sbom30
+    import oe.spdx30
+
+    deps = get_spdx_deps(d)
+
+    dep_objsets = []
+    dep_builds = set()
+
+    dep_build_spdxids = set()
+    for dep_pn, _, in_taskhash  in deps:
+        bb.debug(1, "Fetching SPDX for dependency %s" % (dep_pn))
+        dep_build, dep_objset = oe.sbom30.find_root_obj_in_jsonld(d, "recipes", dep_pn, oe.spdx30.build_Build)
+        # If the dependency is part of the taskhash, return it to be linked
+        # against. Otherwise, it cannot be linked against because this recipe
+        # will not rebuilt if dependency changes
+        if in_taskhash:
+            dep_objsets.append(dep_objset)
+
+        # The build _can_ be linked against (by alias)
+        dep_builds.add(dep_build)
+
+    return dep_objsets, dep_builds
+
+collect_dep_objsets[vardepsexclude] = "SSTATE_ARCHS"
+
+def collect_dep_sources(dep_objsets):
+    import oe.spdx30
+    import oe.sbom30
+
+    sources = {}
+    for objset in dep_objsets:
+        # Don't collect sources from native recipes as they
+        # match non-native sources also.
+        if objset.is_native():
+            continue
+
+        bb.debug(1, "Fetching Sources for dependency %s" % (objset.doc.name))
+
+        dep_build = objset.find_root(oe.spdx30.build_Build)
+        if not dep_build:
+            bb.fatal("Unable to find a build")
+
+        for e in objset.foreach_type(oe.spdx30.Relationship):
+            if dep_build is not e.from_:
+                continue
+
+            if e.relationshipType != oe.spdx30.RelationshipType.hasInputs:
+                continue
+
+            for to in e.to:
+                if not isinstance(to, oe.spdx30.software_File):
+                    continue
+
+                if to.software_primaryPurpose != oe.spdx30.software_SoftwarePurpose.source:
+                    continue
+
+                for v in to.verifiedUsing:
+                    if v.algorithm == oe.spdx30.HashAlgorithm.sha256:
+                        sources[v.hashValue] = to
+                        break
+                else:
+                    bb.fatal("No SHA256 found for %s in %s" % (to.name, objset.doc.name))
+
+    return sources
+
+def add_hashes(d, element, filepath):
+    import hashlib
+
+    algorithms = d.getVar("SPDX_HASH_ALGORITHMS").split()
+    algorithms.sort()
+
+    hashes = [hashlib.new(a) for a in algorithms]
+
+    with open(filepath, "rb") as f:
+        while True:
+            data = f.read(4096)
+            if not data:
+                break
+
+            for h in hashes:
+                h.update(data)
+
+    element.verifiedUsing.extend(
+        oe.spdx30.Hash(
+            algorithm=getattr(oe.spdx30.HashAlgorithm, h.name),
+            hashValue=h.hexdigest(),
+        ) for h in hashes
+    )
+
+
+def add_download_files(d, objset):
+    import oe.patch
+    import oe.spdx30
+    import os
+
+    inputs = set()
+
+    urls = d.getVar("SRC_URI").split()
+    fetch = bb.fetch2.Fetch(urls, d)
+
+    for download_idx, src_uri in enumerate(urls):
+        fd = fetch.ud[src_uri]
+
+        for name in fd.names:
+            file_name = os.path.basename(fetch.localpath(src_uri))
+            if oe.patch.patch_path(src_uri, fetch, '', expand=False):
+                primary_purpose = oe.spdx30.software_SoftwarePurpose.patch
+            else:
+                primary_purpose = oe.spdx30.software_SoftwarePurpose.source
+
+            if fd.type == "file":
+                file = objset.add(oe.spdx30.software_File(
+                    _id=objset.new_spdxid("source", str(download_idx + 1)),
+                    creationInfo=objset.doc.creationInfo,
+                    name=file_name,
+                    software_primaryPurpose=primary_purpose,
+                ))
+                add_hashes(d, file, fd.localpath)
+
+                inputs.add(file)
+            else:
+                uri = fd.type
+                proto = getattr(fd, "proto", None)
+                if proto is not None:
+                    uri = uri + "+" + proto
+                uri = uri + "://" + fd.host + fd.path
+
+                if fd.method.supports_srcrev():
+                    uri = uri + "@" + fd.revisions[name]
+
+                dl = objset.add(oe.spdx30.software_Package(
+                    _id=objset.new_spdxid("source", str(download_idx + 1)),
+                    creationInfo=objset.doc.creationInfo,
+                    name=file_name,
+                    software_primaryPurpose=primary_purpose,
+                    software_downloadLocation=uri,
+                ))
+
+                if fd.method.supports_checksum(fd):
+                    # TODO Need something better than hard coding this
+                    for checksum_id in ["sha256", "sha1"]:
+                        expected_checksum = getattr(fd, "%s_expected" % checksum_id, None)
+                        if expected_checksum is None:
+                            continue
+
+                        dl.verifiedUsing.append(
+                            oe.spdx30.Hash(
+                                algorithm=getattr(oe.spdx30.HashAlgorithm, checksum_id),
+                                hashValue=expected_checksum,
+                            )
+                        )
+
+                inputs.add(dl)
+
+    return inputs
+
+
+def collect_build_package_inputs(d, objset, build, packages):
+    providers = collect_package_providers(d)
+
+    build_deps = set()
+
+    for name in sorted(packages.keys()):
+        if name not in providers:
+            bb.fatal("Unable to find SPDX provider for '%s'" % name)
+
+        pkg_name, pkg_hashfn = providers[name]
+
+        # Copy all of the package SPDX files into the Sbom elements
+        pkg_spdx, _ = oe.sbom30.find_root_obj_in_jsonld(
+            d,
+            "packages",
+            pkg_name,
+            oe.spdx30.software_Package,
+            software_primaryPurpose=oe.spdx30.software_SoftwarePurpose.install,
+        )
+        build_deps.add(pkg_spdx._id)
+
+    objset.new_scoped_relationship(
+        [build],
+        oe.spdx30.RelationshipType.hasInputs,
+        oe.spdx30.LifecycleScopeType.build,
+        sorted(list(build_deps)),
+    )
+
+def set_purposes(d, element, *var_names, force_purposes=[]):
+    purposes = force_purposes[:]
+
+    for var_name in var_names:
+        val = d.getVar(var_name)
+        if val:
+            purposes.extend(val.split())
+            break
+
+    if not purposes:
+        bb.warn("No SPDX purposes found in %s" % " ".join(var_names))
+        return
+
+    element.software_primaryPurpose = getattr(oe.spdx30.software_SoftwarePurpose, purposes[0])
+    element.software_additionalPurpose = [getattr(oe.spdx30.software_SoftwarePurpose, p) for p in purposes[1:]]
+
+
+python do_create_spdx() {
+    import oe.sbom30
+    import oe.spdx30
+    from pathlib import Path
+    from contextlib import contextmanager
+    import oe.cve_check
+    from datetime import datetime
+
+
+    def set_var_field(var, obj, name, package=None):
+        val = None
+        if package:
+            val = d.getVar("%s:%s" % (var, package))
+
+        if not val:
+            val = d.getVar(var)
+
+        if val:
+            setattr(obj, name, val)
+
+    deploydir = Path(d.getVar("SPDXDEPLOY"))
+    deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX"))
+    spdx_workdir = Path(d.getVar("SPDXWORK"))
+    include_sources = d.getVar("SPDX_INCLUDE_SOURCES") == "1"
+    pkg_arch = d.getVar("SSTATE_PKGARCH")
+    is_native = bb.data.inherits_class("native", d) or bb.data.inherits_class("cross", d)
+
+    build_objset = oe.sbom30.ObjectSet.new_objset(d, d.getVar("PN"))
+
+    build = build_objset.new_sub_build("recipe", "recipe")
+    build_objset.doc.rootElement.append(build)
+
+    build_objset.set_is_native(is_native)
+
+    for var in (d.getVar('SPDX_CUSTOM_ANNOTATION_VARS') or "").split():
+        new_annotation(
+            d,
+            build_objset,
+            build,
+            "%s=%s" % (var, d.getVar(var)),
+            oe.spdx30.AnnotationType.other
+        )
+
+    build_inputs = set()
+
+    # Add CVEs
+    cve_by_status = {}
+    cve_by_status["Patched"] = {cve: build_objset.new_cve_vuln(cve) for cve in oe.cve_check.get_patched_cves(d)}
+
+    for cve in (d.getVarFlags("CVE_STATUS") or {}):
+        status, _, _ = oe.cve_check.decode_cve_status(d, cve)
+        # Already added above
+        if status == "Patched":
+            continue
+
+        cve_by_status.setdefault(status, {})[cve] = build_objset.new_cve_vuln(cve)
+
+    cpe_ids = oe.cve_check.get_cpe_ids(d.getVar("CVE_PRODUCT"), d.getVar("CVE_VERSION"))
+
+    source_files = add_download_files(d, build_objset)
+    build_inputs |= source_files
+
+    recipe_spdx_license = add_license_expression(d, build_objset, d.getVar("LICENSE"))
+    build_objset.new_relationship(
+        source_files,
+        oe.spdx30.RelationshipType.hasConcludedLicense,
+        [recipe_spdx_license],
+    )
+
+    if process_sources(d) and include_sources:
+        bb.debug(1, "Adding source files to SPDX")
+        spdx_get_src(d)
+
+        build_inputs |= add_package_files(
+            d,
+            build_objset,
+            spdx_workdir,
+            lambda file_counter: build_objset.new_spdxid("sourcefile", str(file_counter)),
+            lambda filepath: [oe.spdx30.software_SoftwarePurpose.source],
+            ignore_dirs=[".git"],
+            ignore_top_level_dirs=["temp"],
+            archive=None,
+        )
+
+    dep_objsets, dep_builds = collect_dep_objsets(d, build)
+    build_objset.new_scoped_relationship(
+        [build],
+        oe.spdx30.RelationshipType.dependsOn,
+        oe.spdx30.LifecycleScopeType.build,
+        sorted(oe.sbom30.get_element_link_id(b) for b in dep_builds),
+    )
+
+    debug_source_ids = set()
+    source_hash_cache = {}
+
+    # Write out the package SPDX data now. It is not complete as we cannot
+    # write the runtime data, so write it to a staging area and a later task
+    # will write out the final collection
+
+    # TODO: Handle native recipe output
+    if not is_native:
+        bb.debug(1, "Collecting Dependency sources files")
+        sources = collect_dep_sources(dep_objsets)
+
+        bb.build.exec_func("read_subpackage_metadata", d)
+
+        pkgdest = Path(d.getVar("PKGDEST"))
+        for package in d.getVar("PACKAGES").split():
+            if not oe.packagedata.packaged(package, d):
+                continue
+
+            pkg_name = d.getVar("PKG:%s" % package) or package
+
+            bb.debug(1, "Creating SPDX for package %s" % pkg_name)
+
+            pkg_objset = oe.sbom30.ObjectSet.new_objset(d, pkg_name)
+
+            spdx_package = pkg_objset.add_root(oe.spdx30.software_Package(
+                _id=pkg_objset.new_spdxid("package", pkg_name),
+                creationInfo=pkg_objset.doc.creationInfo,
+                name=pkg_name,
+                software_packageVersion=d.getVar("PV"),
+                builtTime=oe.sbom30.spdx_now(),
+            ))
+
+            set_purposes(
+                d,
+                spdx_package,
+                "SPDX_PACKAGE_ADDITIONAL_PURPOSE:%s" % package,
+                "SPDX_PACKAGE_ADDITIONAL_PURPOSE",
+                force_purposes=["install"],
+            )
+
+
+            supplier = build_objset.new_agent("SPDX_SUPPLIER")
+            if supplier is not None:
+                spdx_package.supplier = supplier if isinstance(supplier, str) else supplier._id
+
+            set_var_field("HOMEPAGE", spdx_package, "software_homePage", package=package)
+            set_var_field("SUMMARY", spdx_package, "summary", package=package)
+            set_var_field("DESCRIPTION", spdx_package, "description", package=package)
+
+            pkg_objset.new_scoped_relationship(
+                [build._id],
+                oe.spdx30.RelationshipType.hasOutputs,
+                oe.spdx30.LifecycleScopeType.build,
+                [spdx_package],
+            )
+
+            if cpe_ids:
+                for cpe_id in cpe_ids:
+                    spdx_package.externalIdentifier.append(
+                        oe.spdx30.ExternalIdentifier(
+                            externalIdentifierType=oe.spdx30.ExternalIdentifierType.cpe23,
+                            identifier=cpe_id,
+                    ))
+
+            # TODO: Generate a file for each actual IPK/DEB/RPM/TGZ file
+            # generated and link it to the package
+            #spdx_package_file = pkg_objset.add(oe.spdx30.software_File(
+            #    _id=pkg_objset.new_spdxid("distribution", pkg_name),
+            #    creationInfo=pkg_objset.doc.creationInfo,
+            #    name=pkg_name,
+            #    builtTime=oe.sbom30.spdx_now(),
+            #    software_primaryPurpose=spdx_package.software_primaryPurpose,
+            #    software_additionalPurpose=spdx_package.software_additionalPurpose,
+            #))
+
+            ## TODO add hashes
+            #pkg_objset.new_relationship(
+            #    [spdx_package],
+            #    oe.spdx30.RelationshipType.hasDistributionArtifact,
+            #    [spdx_package_file],
+            #)
+
+            # NOTE: licenses live in the recipe collection and are referenced
+            # by ID in the package collection(s). This helps reduce duplication
+            # (since a lot of packages will have the same license), and also
+            # prevents duplicate license SPDX IDs in the packages
+            package_license = d.getVar("LICENSE:%s" % package)
+            if package_license and package_license != d.getVar("LICENSE"):
+                package_spdx_license = add_license_expression(d, build_objset, package_license)
+            else:
+                package_spdx_license = recipe_spdx_license
+
+            pkg_objset.new_relationship(
+                [spdx_package],
+                oe.spdx30.RelationshipType.hasConcludedLicense,
+                [package_spdx_license._id],
+            )
+
+            # NOTE: CVE Elements live in the recipe collection
+            for status, cves in cve_by_status.items():
+                pkg_objset.new_relationship(
+                    [spdx_package],
+                    oe.spdx30.RelationshipType.hasAssociatedVulnerability,
+                    sorted(spdx_cve._id for _, spdx_cve in cves.items()),
+                )
+
+                for cve, spdx_cve in cves.items():
+                    if status == "Patched":
+                        pkg_objset.new_vex_patched_relationship(spdx_cve._id, [spdx_package])
+                    elif status == "Unpatched":
+                        pkg_objset.new_vex_unpatched_relationship(spdx_cve._id, [spdx_package])
+                    elif status == "Ignored":
+                        pkg_objset.new_vex_ignored_relationship(
+                            spdx_cve._id,
+                            [spdx_package],
+                            impact_statement=d.getVarFlag("CVE_STATUS", cve).split(":", 1)[1],
+                        )
+                    else:
+                        bb.fatal(f"Unknown CVE status {status}")
+
+            bb.debug(1, "Adding package files to SPDX for package %s" % pkg_name)
+            package_files = add_package_files(
+                d,
+                pkg_objset,
+                pkgdest / package,
+                lambda file_counter: pkg_objset.new_spdxid("package", pkg_name, "file", str(file_counter)),
+                # TODO: Can we know the purpose here?
+                lambda filepath: [],
+                ignore_top_level_dirs=['CONTROL', 'DEBIAN'],
+                archive=None,
+            )
+
+            pkg_objset.new_relationship(
+                [spdx_package],
+                oe.spdx30.RelationshipType.contains,
+                sorted(list(package_files)),
+            )
+
+            debug_sources = get_package_sources_from_debug(d, package, package_files, sources, source_hash_cache)
+            debug_source_ids |= set(oe.sbom30.get_element_link_id(d) for d in debug_sources)
+
+            oe.sbom30.write_recipe_jsonld_doc(d, pkg_objset, "packages-staging", deploydir, create_spdx_id_links=False)
+
+    if include_sources:
+        bb.debug(1, "Adding sysroot files to SPDX")
+        sysroot_files = add_package_files(
+            d,
+            build_objset,
+            d.expand("${COMPONENTS_DIR}/${PACKAGE_ARCH}/${PN}"),
+            lambda file_counter: build_objset.new_spdxid("sysroot", str(file_counter)),
+            lambda filepath: [],
+            archive=None,
+        )
+
+        if sysroot_files:
+            build_objset.new_scoped_relationship(
+                [build],
+                oe.spdx30.RelationshipType.hasOutputs,
+                oe.spdx30.LifecycleScopeType.build,
+                sorted(list(sysroot_files)),
+            )
+
+    build_objset.new_scoped_relationship(
+        [build],
+        oe.spdx30.RelationshipType.hasInputs,
+        oe.spdx30.LifecycleScopeType.build,
+        sorted(list(build_inputs)) + sorted(list(debug_source_ids)),
+    )
+
+    oe.sbom30.write_recipe_jsonld_doc(d, build_objset, "recipes", deploydir)
+}
+do_create_spdx[vardepsexclude] += "BB_NUMBER_THREADS"
+# NOTE: depending on do_unpack is a hack that is necessary to get it's dependencies for archive the source
+addtask do_create_spdx after do_populate_sysroot do_package do_packagedata do_unpack do_collect_spdx_deps do_package_write_ipk do_pacakge_write_deb do_package_write_rpm before do_populate_sdk do_build do_rm_work
+
+SSTATETASKS += "do_create_spdx"
+do_create_spdx[sstate-inputdirs] = "${SPDXDEPLOY}"
+do_create_spdx[sstate-outputdirs] = "${DEPLOY_DIR_SPDX}"
+
+python do_create_spdx_setscene () {
+    sstate_setscene(d)
+}
+addtask do_create_spdx_setscene
+
+do_create_spdx[dirs] = "${SPDXWORK}"
+do_create_spdx[cleandirs] = "${SPDXDEPLOY} ${SPDXWORK}"
+do_create_spdx[depends] += "${PATCHDEPENDENCY}"
+
+python do_create_package_spdx() {
+    import oe.sbom30
+    import oe.spdx30
+    import oe.packagedata
+    from pathlib import Path
+
+    deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX"))
+    deploydir = Path(d.getVar("SPDXRUNTIMEDEPLOY"))
+    is_native = bb.data.inherits_class("native", d) or bb.data.inherits_class("cross", d)
+
+    providers = collect_package_providers(d)
+    pkg_arch = d.getVar("SSTATE_PKGARCH")
+
+    if not is_native:
+        bb.build.exec_func("read_subpackage_metadata", d)
+
+        dep_package_cache = {}
+
+        # Any element common to all packages that need to be referenced by ID
+        # should be written into this objset set
+        common_objset = oe.sbom30.ObjectSet.new_objset(d, "%s-package-common" % d.getVar("PN"))
+
+        pkgdest = Path(d.getVar("PKGDEST"))
+        for package in d.getVar("PACKAGES").split():
+            bb.debug(1, "Processing package %s" % package)
+            localdata = bb.data.createCopy(d)
+            pkg_name = d.getVar("PKG:%s" % package) or package
+            localdata.setVar("PKG", pkg_name)
+            localdata.setVar('OVERRIDES', d.getVar("OVERRIDES", False) + ":" + package)
+
+            if not oe.packagedata.packaged(package, localdata):
+                continue
+
+            spdx_package, pkg_objset = oe.sbom30.load_obj_in_jsonld(
+                d,
+                pkg_arch,
+                "packages-staging",
+                pkg_name,
+                oe.spdx30.software_Package,
+                software_primaryPurpose=oe.spdx30.software_SoftwarePurpose.install,
+            )
+
+            # We will write out a new collection, so link it to the new
+            # creation info in the common package data. The old creation info
+            # should still exist and be referenced by all the existing elements
+            # in the package
+            pkg_objset.creationInfo = pkg_objset.copy_creation_info(common_objset.doc.creationInfo)
+
+            runtime_spdx_deps = set()
+
+            deps = bb.utils.explode_dep_versions2(localdata.getVar("RDEPENDS") or "")
+            seen_deps = set()
+            for dep, _ in deps.items():
+                if dep in seen_deps:
+                    continue
+
+                if dep not in providers:
+                    continue
+
+                (dep, _) = providers[dep]
+
+                if not oe.packagedata.packaged(dep, localdata):
+                    continue
+
+                dep_pkg_data = oe.packagedata.read_subpkgdata_dict(dep, d)
+                dep_pkg = dep_pkg_data["PKG"]
+
+                if dep in dep_package_cache:
+                    dep_spdx_package = dep_package_cache[dep]
+                else:
+                    bb.debug(1, "Searching for %s" % dep_pkg)
+                    dep_spdx_package, _ = oe.sbom30.find_root_obj_in_jsonld(
+                        d,
+                        "packages-staging",
+                        dep_pkg,
+                        oe.spdx30.software_Package,
+                        software_primaryPurpose=oe.spdx30.software_SoftwarePurpose.install,
+                    )
+                    dep_package_cache[dep] = dep_spdx_package
+
+                runtime_spdx_deps.add(dep_spdx_package)
+                seen_deps.add(dep)
+
+            pkg_objset.new_scoped_relationship(
+                [spdx_package],
+                oe.spdx30.RelationshipType.dependsOn,
+                oe.spdx30.LifecycleScopeType.runtime,
+                [oe.sbom30.get_element_link_id(dep) for dep in runtime_spdx_deps],
+            )
+
+            oe.sbom30.write_recipe_jsonld_doc(d, pkg_objset, "packages", deploydir)
+
+        oe.sbom30.write_recipe_jsonld_doc(d, common_objset, "common-package", deploydir)
+}
+
+do_create_package_spdx[vardepsexclude] += "OVERRIDES SSTATE_ARCHS"
+
+addtask do_create_package_spdx after do_create_spdx before do_build do_rm_work
+SSTATETASKS += "do_create_package_spdx"
+do_create_package_spdx[sstate-inputdirs] = "${SPDXRUNTIMEDEPLOY}"
+do_create_package_spdx[sstate-outputdirs] = "${DEPLOY_DIR_SPDX}"
+
+python do_create_package_spdx_setscene () {
+    sstate_setscene(d)
+}
+addtask do_create_package_spdx_setscene
+
+do_create_package_spdx[dirs] = "${SPDXRUNTIMEDEPLOY}"
+do_create_package_spdx[cleandirs] = "${SPDXRUNTIMEDEPLOY}"
+do_create_package_spdx[rdeptask] = "do_create_spdx"
+
+python do_create_rootfs_spdx() {
+    import json
+    from pathlib import Path
+    import oe.spdx30
+    import oe.sbom30
+    from datetime import datetime
+
+    deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX"))
+    deploydir = Path(d.getVar("SPDXROOTFSDEPLOY"))
+    root_packages_file = Path(d.getVar("SPDX_ROOTFS_PACKAGES"))
+    image_basename = d.getVar("IMAGE_BASENAME")
+    machine = d.getVar("MACHINE")
+
+    with root_packages_file.open("r") as f:
+        packages = json.load(f)
+
+    objset = oe.sbom30.ObjectSet.new_objset(d, "%s-%s" % (image_basename, machine))
+
+    rootfs = objset.add_root(oe.spdx30.software_SoftwareArtifact(
+        _id=objset.new_spdxid("rootfs", image_basename),
+        creationInfo=objset.doc.creationInfo,
+        name=image_basename,
+        software_primaryPurpose=oe.spdx30.software_SoftwarePurpose.archive,
+        builtTime=oe.sbom30.spdx_now()
+    ))
+
+    rootfs_build = objset.add_root(objset.new_sub_build("rootfs", "rootfs"))
+    rootfs_build.build_buildEndTime = oe.sbom30.spdx_now()
+
+    objset.new_scoped_relationship(
+        [rootfs_build],
+        oe.spdx30.RelationshipType.hasOutputs,
+        oe.spdx30.LifecycleScopeType.build,
+        [rootfs],
+    )
+
+    collect_build_package_inputs(d, objset, rootfs_build, packages)
+
+    oe.sbom30.write_recipe_jsonld_doc(d, objset, "rootfs", deploydir)
+}
+addtask do_create_rootfs_spdx after do_rootfs before do_image
+SSTATETASKS += "do_create_rootfs_spdx"
+do_create_rootfs_spdx[sstate-inputdirs] = "${SPDXROOTFSDEPLOY}"
+do_create_rootfs_spdx[sstate-outputdirs] = "${DEPLOY_DIR_SPDX}"
+do_create_rootfs_spdx[recrdeptask] += "do_create_spdx do_create_package_spdx"
+do_create_rootfs_spdx[cleandirs] += "${SPDXROOTFSDEPLOY}"
+
+python do_create_rootfs_spdx_setscene() {
+    sstate_setscene(d)
+}
+addtask do_create_rootfs_spdx_setscene
+
+python do_create_image_spdx() {
+    import oe.spdx30
+    import oe.sbom30
+    import json
+    from pathlib import Path
+
+    image_deploy_dir = Path(d.getVar('IMGDEPLOYDIR'))
+    manifest_path = Path(d.getVar("IMAGE_FILE_MANIFEST"))
+    spdx_work_dir = Path(d.getVar('SPDXIMAGEWORK'))
+
+    image_basename = d.getVar('IMAGE_BASENAME')
+    machine = d.getVar("MACHINE")
+
+    objset = oe.sbom30.ObjectSet.new_objset(d, "%s-%s" % (image_basename, machine))
+
+    with manifest_path.open("r") as f:
+        manifest = json.load(f)
+
+    builds = []
+    for task in manifest:
+        imagetype = task["imagetype"]
+        taskname = task["taskname"]
+
+        image_build = objset.add_root(objset.new_sub_build(taskname, "image/%s" % imagetype))
+        image_build.build_buildEndTime = oe.sbom30.spdx_now()
+        builds.append(image_build)
+
+        artifacts = []
+
+        for image in task["images"]:
+            image_filename = image["filename"]
+            image_path = image_deploy_dir / image_filename
+            a = objset.add_root(oe.spdx30.software_File(
+                _id=objset.new_spdxid("image", image_filename),
+                creationInfo=objset.doc.creationInfo,
+                name=image_filename,
+                builtTime=oe.sbom30.spdx_now()
+            ))
+            set_purposes(d, a, "SPDX_IMAGE_PURPOSE:%s" % imagetype, "SPDX_IMAGE_PURPOSE")
+            add_hashes(d, a, image_path)
+
+            artifacts.append(a)
+
+        if artifacts:
+            objset.new_scoped_relationship(
+                [image_build],
+                oe.spdx30.RelationshipType.hasOutputs,
+                oe.spdx30.LifecycleScopeType.build,
+                artifacts,
+            )
+
+    if builds:
+        rootfs_image, _ = oe.sbom30.find_root_obj_in_jsonld(
+            d,
+            "rootfs",
+            "%s-%s" % (image_basename, machine),
+            oe.spdx30.software_SoftwareArtifact,
+            # TODO: Should use a purpose to filter here?
+        )
+        objset.new_scoped_relationship(
+            builds,
+            oe.spdx30.RelationshipType.hasInputs,
+            oe.spdx30.LifecycleScopeType.build,
+            [rootfs_image._id],
+        )
+
+    objset.add_aliases()
+    objset.link()
+    oe.sbom30.write_recipe_jsonld_doc(d, objset, "image", spdx_work_dir)
+}
+addtask do_create_image_spdx after do_image_complete do_create_rootfs_spdx before do_build
+SSTATETASKS += "do_create_image_spdx"
+SSTATE_SKIP_CREATION:task-combine-image-type-spdx = "1"
+do_create_image_spdx[sstate-inputdirs] = "${SPDXIMAGEWORK}"
+do_create_image_spdx[sstate-outputdirs] = "${DEPLOY_DIR_SPDX}"
+do_create_image_spdx[cleandirs] = "${SPDXIMAGEWORK}"
+do_create_image_spdx[dirs] = "${SPDXIMAGEWORK}"
+
+python do_create_image_spdx_setscene() {
+    sstate_setscene(d)
+}
+addtask do_create_image_spdx_setscene
+
+
+python do_create_image_sbom() {
+    import os
+    from pathlib import Path
+    import oe.spdx30
+    import oe.sbom30
+
+    image_name = d.getVar("IMAGE_NAME")
+    image_basename = d.getVar("IMAGE_BASENAME")
+    image_link_name = d.getVar("IMAGE_LINK_NAME")
+    imgdeploydir = Path(d.getVar("SPDXIMAGEDEPLOYDIR"))
+    machine = d.getVar("MACHINE")
+
+    spdx_path = imgdeploydir / (image_name + ".spdx.json")
+
+    root_elements = []
+
+    # TODO: Do we need to add the rootfs or are the image files sufficient?
+    rootfs_image, _ = oe.sbom30.find_root_obj_in_jsonld(
+        d,
+        "rootfs",
+        "%s-%s" % (image_basename, machine),
+        oe.spdx30.software_SoftwareArtifact,
+        # TODO: Should use a purpose here?
+    )
+    root_elements.append(rootfs_image._id)
+
+    image_objset, _ = oe.sbom30.find_jsonld(d, "image", "%s-%s" % (image_basename, machine), required=True)
+    for o in image_objset.foreach_root(oe.spdx30.software_File):
+        root_elements.append(o._id)
+
+    objset, sbom = oe.sbom30.create_sbom(d, image_name, root_elements)
+
+    oe.sbom30.write_jsonld_doc(d, objset, spdx_path)
+
+    def make_image_link(target_path, suffix):
+        if image_link_name:
+            link = imgdeploydir / (image_link_name + suffix)
+            if link != target_path:
+                link.symlink_to(os.path.relpath(target_path, link.parent))
+
+    make_image_link(spdx_path, ".spdx.json")
+}
+addtask do_create_image_sbom after do_create_rootfs_spdx do_create_image_spdx before do_build
+SSTATETASKS += "do_create_image_sbom"
+SSTATE_SKIP_CREATION:task-create-image-sbom = "1"
+do_create_image_sbom[sstate-inputdirs] = "${SPDXIMAGEDEPLOYDIR}"
+do_create_image_sbom[sstate-outputdirs] = "${DEPLOY_DIR_IMAGE}"
+do_create_image_sbom[stamp-extra-info] = "${MACHINE_ARCH}"
+do_create_image_sbom[cleandirs] = "${SPDXIMAGEDEPLOYDIR}"
+
+python do_create_image_sbom_setscene() {
+    sstate_setscene(d)
+}
+addtask do_create_image_sbom_setscene
+
+do_populate_sdk[recrdeptask] += "do_create_spdx do_create_package_spdx"
+do_populate_sdk[cleandirs] += "${SPDXSDKWORK}"
+do_populate_sdk[postfuncs] += "sdk_create_sbom"
+POPULATE_SDK_POST_HOST_COMMAND:append:task-populate-sdk = " sdk_host_create_spdx"
+POPULATE_SDK_POST_TARGET_COMMAND:append:task-populate-sdk = " sdk_target_create_spdx"
+
+python sdk_host_create_spdx() {
+    sdk_create_spdx(d, "host")
+}
+
+python sdk_target_create_spdx() {
+    sdk_create_spdx(d, "target")
+}
+
+def sdk_create_spdx(d, sdk_type):
+    from pathlib import Path
+    from oe.sdk import sdk_list_installed_packages
+    import oe.spdx30
+    import oe.sbom30
+    from datetime import datetime
+
+    sdk_name = d.getVar("TOOLCHAIN_OUTPUTNAME") + "-" + sdk_type
+    sdk_packages = sdk_list_installed_packages(d, sdk_type == "target")
+    spdx_work_dir = Path(d.getVar('SPDXSDKWORK'))
+
+    objset = oe.sbom30.ObjectSet.new_objset(d, sdk_name)
+
+    sdk_rootfs = objset.add_root(oe.spdx30.software_SoftwareArtifact(
+        _id=objset.new_spdxid("sdk-rootfs", sdk_name),
+        creationInfo=objset.doc.creationInfo,
+        name=sdk_name,
+        softwre_primaryPurpose=oe.spdx30.software_SoftwarePurpose.archive,
+        builtTime=oe.sbom30.spdx_now(),
+    ))
+
+    sdk_build = objset.add_root(objset.new_sub_build("sdk-rootfs", "sdk-rootfs"))
+    sdk_build.build_buildEndTime = oe.sbom30.spdx_now()
+
+    objset.new_scoped_relationship(
+        [sdk_build],
+        oe.spdx30.RelationshipType.hasOutputs,
+        oe.spdx30.LifecycleScopeType.build,
+        [sdk_rootfs],
+    )
+
+    collect_build_package_inputs(d, objset, sdk_build, sdk_packages)
+
+    objset.add_aliases()
+    oe.sbom30.write_jsonld_doc(d, objset, spdx_work_dir / "sdk-rootfs.spdx.json")
+
+python sdk_create_sbom() {
+    import oe.spdx30
+    import oe.sbom30
+    from pathlib import Path
+    from datetime import datetime
+
+    toolchain_outputname = d.getVar("TOOLCHAIN_OUTPUTNAME")
+    sdk_filename = toolchain_outputname + ".sh"
+    sdk_deploydir = Path(d.getVar("SDKDEPLOYDIR"))
+    spdx_work_dir = Path(d.getVar('SPDXSDKWORK'))
+
+    # Load the document written earlier
+    rootfs_objset = oe.sbom30.load_jsonld(d, spdx_work_dir / "sdk-rootfs.spdx.json", required=True)
+
+    # Create a new build for the SDK installer
+    sdk_build = rootfs_objset.new_sub_build("sdk-populate", "sdk-populate")
+    sdk_build.buildEndTime = oe.sbom30.spdx_now()
+
+    rootfs = rootfs_objset.find_root(oe.spdx30.software_SoftwareArtifact)
+    if rootfs is None:
+        bb.fatal("Unable to find rootfs artifact")
+
+    rootfs_objset.new_scoped_relationship(
+        [sdk_build],
+        oe.spdx30.RelationshipType.hasInputs,
+        oe.spdx30.LifecycleScopeType.build,
+        [rootfs]
+    )
+
+    # Create a file for the SDK installer
+    sdk_installer = objset.add(oe.spdx30.software_File(
+        _id=objset.new_spdxid("sdk-installer", toolchain_outputname),
+        creationInfo=rootfs_objset.doc.creationInfo,
+        name=sdk_filename,
+        builtTime=oe.sbom30.spdx_now(),
+    ))
+    set_purposes(d, sdk_installer, "SPDX_SDK_PURPOSE")
+    add_hashes(d, sdk_installer, sdk_deploydir / sdk_filename)
+
+    rootfs_objset.new_scoped_relationship(
+        [sdk_build],
+        oe.spdx30.RelationshipType.hasOutputs,
+        oe.spdx30.LifecycleScopeType.build,
+        [sdk_installer],
+    )
+
+    objset, sbom = oe.sbom30.create_sbom(d, sdk_filename, [sdk_installer], [rootfs_objset])
+
+    oe.sbom30.write_jsonld_doc(d, objset, sdk_deploydir / (toolchain_outputname + ".spdx.json"))
+}
+
+python spdx30_build_started_handler () {
+    import oe.spdx30
+    import oe.sbom30
+    import os
+    from pathlib import Path
+
+    # Create a copy of the datastore. Set PN to "bitbake" so that SPDX IDs can
+    # be generated
+    d = e.data.createCopy()
+    d.setVar("PN", "bitbake")
+    d.setVar("BB_TASKHASH", "bitbake")
+    load_spdx_license_data(d)
+
+    deploy_dir_spdx = Path(e.data.getVar("DEPLOY_DIR_SPDX"))
+
+    nonce = os.urandom(16).hex()
+
+    objset = oe.sbom30.ObjectSet.new_objset(d, "bitbake", False)
+
+    build = objset.add_root(oe.spdx30.build_Build(
+        _id=objset.new_spdxid(nonce, include_unihash=False),
+        creationInfo=objset.doc.creationInfo,
+        build_buildType=oe.sbom30.SPDX_BUILD_TYPE,
+        build_buildStartTime=oe.sbom30.spdx_now()
+    ))
+
+    host_import_key = d.getVar("SPDX_BUILD_HOST")
+    if host_import_key:
+        objset.new_scoped_relationship(
+            [build],
+            oe.spdx30.RelationshipType.hasHost,
+            oe.spdx30.LifecycleScopeType.build,
+            [objset.new_import("SPDX_BUILD_HOST")],
+        )
+
+    invoked_by = objset.new_agent("SPDX_INVOKED_BY")
+    if invoked_by:
+        invoked_by_spdx = objset.new_scoped_relationship(
+            [build],
+            oe.spdx30.RelationshipType.invokedBy,
+            oe.spdx30.LifecycleScopeType.build,
+            [invoked_by],
+        )
+
+        on_behalf_of = objset.new_agent("SPDX_ON_BEHALF_OF")
+        if on_behalf_of:
+            objset.new_scoped_relationship(
+                [on_behalf_of],
+                oe.spdx30.RelationshipType.delegatedTo,
+                oe.spdx30.LifecycleScopeType.build,
+                invoked_by_spdx,
+            )
+
+    for obj in objset.foreach_type(oe.spdx30.Element):
+        obj.extension.append(oe.sbom30.OELinkExtension(link_spdx_id=False))
+        obj.extension.append(oe.sbom30.OEIdAliasExtension())
+
+    oe.sbom30.write_jsonld_doc(d, objset, deploy_dir_spdx / "bitbake.spdx.json")
+}
+
+addhandler spdx30_build_started_handler
+spdx30_build_started_handler[eventmask] = "bb.event.ConfigParsed"
+
meta/classes/spdx-common.bbclass +24 -1
index 03f1d0cc278..dc2ecb01009 100644
@@ -16,13 +16,17 @@ SPDXDIR ??= "${WORKDIR}/spdx/${SPDX_VERSION}"
 SPDXDEPLOY = "${SPDXDIR}/deploy"
 SPDXWORK = "${SPDXDIR}/work"
 SPDXIMAGEWORK = "${SPDXDIR}/image-work"
+SPDXROOTFSWORK = "${SPDXDIR}/rootfs-work"
 SPDXSDKWORK = "${SPDXDIR}/sdk-work"
 SPDXDEPS = "${SPDXDIR}/deps.json"
+SPDXIMAGEDEPLOYDIR = "${SPDXDIR}/image-deploy"
+SPDX_ROOTFS_PACKAGES = "${SPDXDIR}/rootfs-packages.json"
 
 SPDX_TOOL_NAME ??= "oe-spdx-creator"
 SPDX_TOOL_VERSION ??= "1.0"
 
 SPDXRUNTIMEDEPLOY = "${SPDXDIR}/runtime-deploy"
+SPDXROOTFSDEPLOY = "${SPDXDIR}/rootfs-deploy"
 
 SPDX_INCLUDE_SOURCES ??= "0"
 SPDX_ARCHIVE_SOURCES ??= "0"
@@ -68,7 +72,7 @@ def get_json_indent(d):
         return 2
     return None
 
-python() {
+def load_spdx_license_data(d):
     import json
     if d.getVar("SPDX_LICENSE_DATA"):
         return
@@ -78,6 +82,9 @@ python() {
         # Transform the license array to a dictionary
         data["licenses"] = {l["licenseId"]: l for l in data["licenses"]}
         d.setVar("SPDX_LICENSE_DATA", data)
+
+python() {
+    load_spdx_license_data(d)
 }
 
 def process_sources(d):
@@ -255,3 +262,19 @@ def spdx_get_src(d):
 
 spdx_get_src[vardepsexclude] += "STAGING_KERNEL_DIR"
 
+
+python spdx_collect_rootfs_packages() {
+    import json
+    from pathlib import Path
+    from oe.rootfs import image_list_installed_packages
+
+    root_packages_file = Path(d.getVar("SPDX_ROOTFS_PACKAGES"))
+
+    packages = image_list_installed_packages(d)
+
+    root_packages_file.parent.mkdir(parents=True, exist_ok=True)
+    with root_packages_file.open("w") as f:
+        json.dump(packages, f)
+}
+ROOTFS_POSTUNINSTALL_COMMAND =+ "spdx_collect_rootfs_packages"
+
meta/lib/oe/sbom30.py new file +993 -0

993 lines not on this page · raw message

meta/lib/oe/spdx30.py new file +5413 -0

670 lines not on this page · raw message

The rest of this diff is not shown. View the raw message.

Signature · 1 line
2.43.2

Part of the diff is not shown here. View the raw message.

#8 lore raw

Patchtest results for [OE-core][PATCH 5/6] classes/create-spdx-3.0: Add class

Thank you for your submission. Patchtest identified one or more issues with the patch. Please see the log below for more information:

---
Testing patch /home/patchtest/share/mboxes/5-6-classes-create-spdx-3.0-Add-class.patch

FAIL: test max line length: Patch line too long (current length 216, maximum is 200) (test_metadata.TestMetadata.test_max_line_length)

PASS: pretest pylint (test_python_pylint.PyLint.pretest_pylint)
PASS: test Signed-off-by presence (test_mbox.TestMbox.test_signed_off_by_presence)
PASS: test author valid (test_mbox.TestMbox.test_author_valid)
PASS: test commit message presence (test_mbox.TestMbox.test_commit_message_presence)
PASS: test mbox format (test_mbox.TestMbox.test_mbox_format)
PASS: test non-AUH upgrade (test_mbox.TestMbox.test_non_auh_upgrade)
PASS: test pylint (test_python_pylint.PyLint.test_pylint)
PASS: test shortlog format (test_mbox.TestMbox.test_shortlog_format)
PASS: test shortlog length (test_mbox.TestMbox.test_shortlog_length)
PASS: test target mailing list (test_mbox.TestMbox.test_target_mailing_list)

SKIP: pretest src uri left files: Patch cannot be merged (test_metadata.TestMetadata.pretest_src_uri_left_files)
SKIP: test CVE check ignore: No modified recipes or older target branch, skipping test (test_metadata.TestMetadata.test_cve_check_ignore)
SKIP: test CVE tag format: No new CVE patches introduced (test_patch.TestPatch.test_cve_tag_format)
SKIP: test Signed-off-by presence: No new CVE patches introduced (test_patch.TestPatch.test_signed_off_by_presence)
SKIP: test Upstream-Status presence: No new CVE patches introduced (test_patch.TestPatch.test_upstream_status_presence_format)
SKIP: test bugzilla entry format: No bug ID found (test_mbox.TestMbox.test_bugzilla_entry_format)
SKIP: test lic files chksum modified not mentioned: No modified recipes, skipping test (test_metadata.TestMetadata.test_lic_files_chksum_modified_not_mentioned)
SKIP: test lic files chksum presence: No added recipes, skipping test (test_metadata.TestMetadata.test_lic_files_chksum_presence)
SKIP: test license presence: No added recipes, skipping test (test_metadata.TestMetadata.test_license_presence)
SKIP: test series merge on head: Merge test is disabled for now (test_mbox.TestMbox.test_series_merge_on_head)
SKIP: test src uri left files: Patch cannot be merged (test_metadata.TestMetadata.test_src_uri_left_files)
SKIP: test summary presence: No added recipes, skipping test (test_metadata.TestMetadata.test_summary_presence)

---

Please address the issues identified and
submit a new revision of the patch, or alternatively, reply to this email with an explanation of why the patch should be accepted. If you believe these results are due to an error in patchtest, please submit a bug at https://bugzilla.yoctoproject.org/ (use the 'Patchtest' category under 'Yocto Project Subprojects'). For more information on specific failures, see: https://wiki.yoctoproject.org/wiki/Patchtest. Thank you!

#9 lore raw

Re: [OE-core][PATCH 5/6] classes/create-spdx-3.0: Add class

In reply to Joshua Watt

Hello,
This is my first round of the review, before applying the code.

On Mon, Jun 10, 2024 at 11:45 PM Joshua Watt via lists.openembedded.org <[email protected]> wrote:

| +SPDX_PROFILES ?= "core build software simpleLicensing security"

I would like to discuss this choice by default. People who do security won't need the 'build' profile (and this likely adds
much data). On the other hand, people who do licensing won't need the 'security' profile.

It would be really nice to generate a basic SPDX without doing the build...

Review

Quoted 3 lines
(quoted diff) +2 -0
+
+SPDX_HASH_ALGORITHMS ?= "sha256 sha1"

Any reason to still use sha1? It is obsolete. We could use sha3 instead if we want to have multiple ones.

Quoted 64 lines
meta/classes/create-spdx-3.0.bbclass new file +28 -0
+
+
+python do_create_spdx() {
+    import oe.sbom30
+    import oe.spdx30
+    from pathlib import Path
+    from contextlib import contextmanager
+    import oe.cve_check
+    from datetime import datetime
+
+
+    def set_var_field(var, obj, name, package=None):
+        val = None
+        if package:
+            val = d.getVar("%s:%s" % (var, package))
+
+        if not val:
+            val = d.getVar(var)
+
+        if val:
+            setattr(obj, name, val)
+
+    deploydir = Path(d.getVar("SPDXDEPLOY"))
+    deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX"))
+    spdx_workdir = Path(d.getVar("SPDXWORK"))
+    include_sources = d.getVar("SPDX_INCLUDE_SOURCES") == "1"
+    pkg_arch = d.getVar("SSTATE_PKGARCH")
+    is_native = bb.data.inherits_class("native", d) or

bb.data.inherits_class("cross", d)

meta/classes/create-spdx-3.0.bbclass new file +22 -0
+
+    build_objset = oe.sbom30.ObjectSet.new_objset(d, d.getVar("PN"))
+
+    build = build_objset.new_sub_build("recipe", "recipe")
+    build_objset.doc.rootElement.append(build)
+
+    build_objset.set_is_native(is_native)
+
+    for var in (d.getVar('SPDX_CUSTOM_ANNOTATION_VARS') or "").split():
+        new_annotation(
+            d,
+            build_objset,
+            build,
+            "%s=%s" % (var, d.getVar(var)),
+            oe.spdx30.AnnotationType.other
+        )
+
+    build_inputs = set()
+
+    # Add CVEs
+    cve_by_status = {}
+    cve_by_status["Patched"] = {cve: build_objset.new_cve_vuln(cve) for

cve in oe.cve_check.get_patched_cves(d)}

meta/classes/create-spdx-3.0.bbclass new file +8 -0
+
+    for cve in (d.getVarFlags("CVE_STATUS") or {}):
+        status, _, _ = oe.cve_check.decode_cve_status(d, cve)
+        # Already added above
+        if status == "Patched":
+            continue
+
+        cve_by_status.setdefault(status, {})[cve] =

build_objset.new_cve_vuln(cve)

(quoted diff) +2 -0
+
+    cpe_ids = oe.cve_check.get_cpe_ids(d.getVar("CVE_PRODUCT"),
Quoted 2 lines

d.getVar("CVE_VERSION"))

This will be adding only those from patches and everything with CVE_STATUS. In some cases (extra-exclusions) vulnerabilities
that have no real link with the given package. Also, the list isn't complete, does not contain most of the unpatched ones.

+do_create_spdx[vardepsexclude] += "BB_NUMBER_THREADS"

Quoted 8 lines

+# NOTE: depending on do_unpack is a hack that is necessary to get it's dependencies for archive the source
+addtask do_create_spdx after do_populate_sysroot do_package do_packagedata do_unpack do_collect_spdx_deps do_package_write_ipk do_pacakge_write_deb do_package_write_rpm before do_populate_sdk do_build do_rm_work
+

This is really late... I fear that if we get that in, it will be too hard to make the generation mode modulable and output
certain profiles at a different place than the whole build. With the same problem for the world build as we have today
with 2.2.

+def spdxid_hash(*items):

meta/lib/oe/sbom30.py line 138

Quoted 9 lines
meta/lib/oe/sbom30.py new file +8 -0
+    h = hashlib.md5()
+    for i in items:
+        if isinstance(i, oe.spdx30.Element):
+            h.update(i._id.encode("utf-8"))
+        else:
+            h.update(i.encode("utf-8"))
+    return h.hexdigest()
+

Can we avoid MD5?

Quoted 14 lines
meta/lib/oe/sbom30.py new file +8 -0
+    def new_cve_vuln(self, cve):
+        v = oe.spdx30.security_Vulnerability()
+        v._id = self.new_spdxid("vulnerability", cve)
+        v.creationInfo = self.doc.creationInfo
+
+        v.externalIdentifier.append(
+            oe.spdx30.ExternalIdentifier(
+

externalIdentifierType=oe.spdx30.ExternalIdentifierType.cve,

meta/lib/oe/sbom30.py new file +3 -0
+                identifier=cve,
+                identifierLocator=[
+                    f"

https://cve.mitre.org/cgi-bin/cvename.cgi?name={cve}",

meta/lib/oe/sbom30.py line 565

Quoted 2 lines
meta/lib/oe/sbom30.py new file +2 -0
+                    f"https://www.cve.org/CVERecord?id={cve}",
+                ],

Better to link to the JSON format https://cveawg.mitre.org/api/cve/{cve}

Quoted 6 lines
meta/lib/oe/sbom30.py new file +5 -0
+            )
+        )
+        return self.add(v)
+
+    def new_vex_patched_relationship(self, from_, to):

meta/lib/oe/sbom30.py line 580

Quoted 10 lines
meta/lib/oe/sbom30.py new file +10 -0
+        self.add(
+            oe.spdx30.security_VexFixedVulnAssessmentRelationship(
+                _id=self.new_spdxid("vex-fixed", spdxid_hash(from_, *to)),
+                creationInfo=self.doc.creationInfo,
+                from_=from_,
+                relationshipType=oe.spdx30.RelationshipType.fixedIn,
+                to=to,
+                security_vexVersion=VEX_VERSION,
+            )
+        )

For machine-readable entries, this should be linking to a proof of a fix, for example in
a relationship to the fix file.

Quoted 22 lines
meta/lib/oe/sbom30.py new file +5 -0
+
+    def new_vex_unpatched_relationship(self, from_, to):
+        self.add(
+            oe.spdx30.security_VexAffectedVulnAssessmentRelationship(
+                _id=self.new_spdxid("vex-affected", spdxid_hash(from_,

*to)),

meta/lib/oe/sbom30.py new file +9 -0
+                creationInfo=self.doc.creationInfo,
+                from_=from_,
+                relationshipType=oe.spdx30.RelationshipType.affects,
+                to=to,
+                security_vexVersion=VEX_VERSION,
+            )
+        )
+
+    def new_vex_ignored_relationship(self, from_, to, *,

impact_statement):

meta/lib/oe/sbom30.py new file +4 -0
+        self.add(
+            oe.spdx30.security_VexNotAffectedVulnAssessmentRelationship(
+                _id=self.new_spdxid(
+                    "vex-not-affected", spdxid_hash(impact_statement,

from_, *to)

meta/lib/oe/sbom30.py line 607

Quoted 9 lines
meta/lib/oe/sbom30.py new file +9 -0
+                ),
+                creationInfo=self.doc.creationInfo,
+                from_=from_,
+                relationshipType=oe.spdx30.RelationshipType.doesNotAffect,
+                to=to,
+                security_vexVersion=VEX_VERSION,
+                security_impactStatement=impact_statement,
+            )
+        )

Missing conversion of 'Ignored' to 'justificationType' which is necessary to allow machine processing of VEX statements. With 'impactStatement' only, which is freeform text, this is human readable only.

That's it in the first pass. There will be more when I can have a look in the output.

Regards,
Marta

#10 lore raw

Re: [OE-core][PATCH 5/6] classes/create-spdx-3.0: Add class

Quoted 4 lines · Marta Rybczynska

Hello,
This is my first round of the review, before applying the code.

Quoted 8 lines

On Mon, Jun 10, 2024 at 11:45 PM Joshua Watt via lists.openembedded.org <[email protected]> wrote:

| +SPDX_PROFILES ?= "core build software simpleLicensing security"

I would like to discuss this choice by default. People who do security won't need the 'build' profile (and this likely adds much data). On the other hand, people who do licensing won't need the 'security' profile.

It would be really nice to generate a basic SPDX without doing the build...

I think we should look at that later just to keep the scope reasonable; This is purposely trying to stick close to what SPDX 2.2 gives us.

Quoted 7 lines

Review

Quoted 4 lines
(quoted diff) +2 -0
+
+SPDX_HASH_ALGORITHMS ?= "sha256 sha1"

Any reason to still use sha1? It is obsolete. We could use sha3 instead if we want to have multiple ones.

I've dropped this in the latest version, in favor of only using sha256 to make my life easier :)

Quoted 69 lines
Quoted 54 lines
meta/classes/create-spdx-3.0.bbclass new file +50 -0
+
+
+python do_create_spdx() {
+    import oe.sbom30
+    import oe.spdx30
+    from pathlib import Path
+    from contextlib import contextmanager
+    import oe.cve_check
+    from datetime import datetime
+
+
+    def set_var_field(var, obj, name, package=None):
+        val = None
+        if package:
+            val = d.getVar("%s:%s" % (var, package))
+
+        if not val:
+            val = d.getVar(var)
+
+        if val:
+            setattr(obj, name, val)
+
+    deploydir = Path(d.getVar("SPDXDEPLOY"))
+    deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX"))
+    spdx_workdir = Path(d.getVar("SPDXWORK"))
+    include_sources = d.getVar("SPDX_INCLUDE_SOURCES") == "1"
+    pkg_arch = d.getVar("SSTATE_PKGARCH")
+    is_native = bb.data.inherits_class("native", d) or bb.data.inherits_class("cross", d)
+
+    build_objset = oe.sbom30.ObjectSet.new_objset(d, d.getVar("PN"))
+
+    build = build_objset.new_sub_build("recipe", "recipe")
+    build_objset.doc.rootElement.append(build)
+
+    build_objset.set_is_native(is_native)
+
+    for var in (d.getVar('SPDX_CUSTOM_ANNOTATION_VARS') or "").split():
+        new_annotation(
+            d,
+            build_objset,
+            build,
+            "%s=%s" % (var, d.getVar(var)),
+            oe.spdx30.AnnotationType.other
+        )
+
+    build_inputs = set()
+
+    # Add CVEs
+    cve_by_status = {}
+    cve_by_status["Patched"] = {cve: build_objset.new_cve_vuln(cve) for cve in oe.cve_check.get_patched_cves(d)}

meta/classes/create-spdx-3.0.bbclass line 595

Quoted 10 lines
meta/classes/create-spdx-3.0.bbclass new file +10 -0
+
+    for cve in (d.getVarFlags("CVE_STATUS") or {}):
+        status, _, _ = oe.cve_check.decode_cve_status(d, cve)
+        # Already added above
+        if status == "Patched":
+            continue
+
+        cve_by_status.setdefault(status, {})[cve] = build_objset.new_cve_vuln(cve)
+
+    cpe_ids = oe.cve_check.get_cpe_ids(d.getVar("CVE_PRODUCT"), d.getVar("CVE_VERSION"))

This will be adding only those from patches and everything with CVE_STATUS. In some cases (extra-exclusions) vulnerabilities that have no real link with the given package. Also, the list isn't complete, does not contain most of the unpatched ones.

Quoted 9 lines

meta/classes/create-spdx-3.0.bbclass line 805

Quoted 4 lines
meta/classes/create-spdx-3.0.bbclass new file +4 -0
+do_create_spdx[vardepsexclude] += "BB_NUMBER_THREADS"
+# NOTE: depending on do_unpack is a hack that is necessary to get it's dependencies for archive the source
+addtask do_create_spdx after do_populate_sysroot do_package do_packagedata do_unpack do_collect_spdx_deps do_package_write_ipk do_pacakge_write_deb do_package_write_rpm before do_populate_sdk do_build do_rm_work
+

This is really late... I fear that if we get that in, it will be too hard to make the generation mode modulable and output certain profiles at a different place than the whole build. With the same problem for the world build as we have today with 2.2.

Yep, but I think we need to focus on that after the initial implementation. It's a solvable problem, but it's going to to be tricky so it will take time, and I'd rather get SPDX 3 support out early.

Quoted 13 lines

meta/lib/oe/sbom30.py line 138

Quoted 10 lines
meta/lib/oe/sbom30.py new file +9 -0
+def spdxid_hash(*items):
+    h = hashlib.md5()
+    for i in items:
+        if isinstance(i, oe.spdx30.Element):
+            h.update(i._id.encode("utf-8"))
+        else:
+            h.update(i.encode("utf-8"))
+    return h.hexdigest()
+

Can we avoid MD5?

It's not cryptographic, so it should not matter. I don't want the hash strings to be _terribly_ long, so unless you have some other short hashing algorithm, I'd rather stick with this.

Quoted 16 lines
Quoted 4 lines
meta/lib/oe/sbom30.py new file +3 -0
+    def new_cve_vuln(self, cve):
+        v = oe.spdx30.security_Vulnerability()
+        v._id = self.new_spdxid("vulnerability", cve)

meta/lib/oe/sbom30.py line 565

Quoted 10 lines
meta/lib/oe/sbom30.py new file +10 -0
+        v.creationInfo = self.doc.creationInfo
+
+        v.externalIdentifier.append(
+            oe.spdx30.ExternalIdentifier(
+                externalIdentifierType=oe.spdx30.ExternalIdentifierType.cve,
+                identifier=cve,
+                identifierLocator=[
+                    f"https://cve.mitre.org/cgi-bin/cvename.cgi?name={cve}",
+                    f"https://www.cve.org/CVERecord?id={cve}",
+                ],
Quoted 1 line

Better to link to the JSON format https://cveawg.mitre.org/api/cve/{cve}

Will do, thanks. Should that be a third entry, or the only entry, or.... ?

Quoted 19 lines
Quoted 7 lines
meta/lib/oe/sbom30.py new file +5 -0
+            )
+        )
+        return self.add(v)
+
+    def new_vex_patched_relationship(self, from_, to):

meta/lib/oe/sbom30.py line 580

Quoted 10 lines
meta/lib/oe/sbom30.py new file +10 -0
+        self.add(
+            oe.spdx30.security_VexFixedVulnAssessmentRelationship(
+                _id=self.new_spdxid("vex-fixed", spdxid_hash(from_, *to)),
+                creationInfo=self.doc.creationInfo,
+                from_=from_,
+                relationshipType=oe.spdx30.RelationshipType.fixedIn,
+                to=to,
+                security_vexVersion=VEX_VERSION,
+            )
+        )
Quoted 2 lines

For machine-readable entries, this should be linking to a proof of a fix, for example in a relationship to the fix file.

Yes, I think we can do that, but we need cve_check.py to be able to give us the file names (or hashes?) so we can correlate them.... I'll log a bug to do that later.

Quoted 24 lines
Quoted 24 lines
meta/lib/oe/sbom30.py new file +22 -0
+
+    def new_vex_unpatched_relationship(self, from_, to):
+        self.add(
+            oe.spdx30.security_VexAffectedVulnAssessmentRelationship(
+                _id=self.new_spdxid("vex-affected", spdxid_hash(from_, *to)),
+                creationInfo=self.doc.creationInfo,
+                from_=from_,
+                relationshipType=oe.spdx30.RelationshipType.affects,
+                to=to,
+                security_vexVersion=VEX_VERSION,
+            )
+        )
+
+    def new_vex_ignored_relationship(self, from_, to, *, impact_statement):
+        self.add(
+            oe.spdx30.security_VexNotAffectedVulnAssessmentRelationship(
+                _id=self.new_spdxid(
+                    "vex-not-affected", spdxid_hash(impact_statement, from_, *to)
+                ),
+                creationInfo=self.doc.creationInfo,
+                from_=from_,
+                relationshipType=oe.spdx30.RelationshipType.doesNotAffect,
Quoted 8 lines

meta/lib/oe/sbom30.py line 607

Quoted 5 lines
meta/lib/oe/sbom30.py new file +5 -0
+                to=to,
+                security_vexVersion=VEX_VERSION,
+                security_impactStatement=impact_statement,
+            )
+        )

Missing conversion of 'Ignored' to 'justificationType' which is necessary to allow machine processing of VEX statements. With 'impactStatement' only, which is freeform text, this is human readable only.

Ya, I think I can add that based on the not-applicable-platform or not-applicable-config

Quoted 5 lines

That's it in the first pass. There will be more when I can have a look in the output.

Regards,
Marta

#11 lore raw

PATCH 6/6 classes-recipe/image_types: Add SPDX_IMAGE_PURPOSE to images

In reply to Joshua Watt

Adds the variable overrides to set the SPDX image purpose for various image types

---
 meta/classes-recipe/image_types.bbclass     | 2 ++
 meta/classes-recipe/image_types_wic.bbclass | 1 +
 2 files changed, 3 insertions(+)
meta/classes-recipe/image_types.bbclass +2 -0
index 2f948ecbf88..506b9934cb7 100644
@@ -146,6 +146,7 @@ IMAGE_CMD:vfat = "oe_mkvfatfs ${EXTRA_IMAGECMD}"
 IMAGE_CMD_TAR ?= "tar"
 # ignore return code 1 "file changed as we read it" as other tasks(e.g. do_image_wic) may be hardlinking rootfs
 IMAGE_CMD:tar = "${IMAGE_CMD_TAR} --sort=name --format=posix --numeric-owner -cf ${IMGDEPLOYDIR}/${IMAGE_NAME}.tar -C ${IMAGE_ROOTFS} . || [ $? -eq 1 ]"
+SPDX_IMAGE_PURPOSE:tar = "archive"
 
 do_image_cpio[cleandirs] += "${WORKDIR}/cpio_append"
 IMAGE_CMD:cpio () {
@@ -167,6 +168,7 @@ IMAGE_CMD:cpio () {
 		fi
 	fi
 }
+SPDX_IMAGE_PURPOSE:cpio = "archive"
 
 UBI_VOLNAME ?= "${MACHINE}-rootfs"
 UBI_VOLTYPE ?= "dynamic"
meta/classes-recipe/image_types_wic.bbclass +1 -0
index cf3be909b30..86f40633ebc 100644
@@ -91,6 +91,7 @@ IMAGE_CMD:wic () {
 	mv "$build_wic/$(basename "${wks%.wks}")"*.${IMAGER} "$out.wic"
 }
 IMAGE_CMD:wic[vardepsexclude] = "WKS_FULL_PATH WKS_FILES TOPDIR"
+SPDX_IMAGE_PURPOSE:wic = "diskImage"
 do_image_wic[cleandirs] = "${WORKDIR}/build-wic"
 
 PSEUDO_IGNORE_PATHS .= ",${WORKDIR}/build-wic"
Signature · 1 line
2.43.2
#12 lore raw
In reply to Joshua Watt
Quoted 7 lines · Joshua Watt via lists.openembedded.org

This patch series add support for SPDX 3.0. Currently it is not possible to have SPDX 2.2 and SPDX 3.0 enabled at the same time; users may choose to specifically use SPDX 3.0 instead of 2.2 by doing INHERIT += "create-spdx-3.0"

The default version is not changed yet, although that will likely happen soon

Does this series depend on other patches?

Cheers,

Richard

#13 lore raw
In reply to Joshua Watt
Quoted 11 lines · Joshua Watt via lists.openembedded.org

This patch series add support for SPDX 3.0. Currently it is not possible
to have SPDX 2.2 and SPDX 3.0 enabled at the same time; users may choose
to specifically use SPDX 3.0 instead of 2.2 by doing INHERIT += "create-spdx-3.0"

The default version is not changed yet, although that will likely happen
soon

Quoted 7 lines

Joshua Watt (6):
  classes-recipe/image: Add image file manifest classes/spdx-common: Move common SPDX to new class classes/spdx-common: Add SPDX version to path classes/spdx-common: Return empty list from extract_licenses classes/create-spdx-3.0: Add class classes-recipe/image_types: Add SPDX_IMAGE_PURPOSE to images

I set 3.0 to the default for testing and there were some interesting looking failures:

https://valkyrie.yoctoproject.org/#/builders/43/builds/52/steps/11/logs/stdio

I'm guessing:

https://valkyrie.yoctoproject.org/#/builders/37/builds/60

is due to a missing patch?

Other failures:

https://valkyrie.yoctoproject.org/#/builders/74/builds/44/steps/13/logs/stdio
https://valkyrie.yoctoproject.org/#/builders/93/builds/51/steps/11/logs/stdio

but I stopped the build at this point.

Cheers,

Richard

#14 lore raw
Quoted 25 lines · Richard Purdie
Quoted 11 lines · Joshua Watt via lists.openembedded.org

This patch series add support for SPDX 3.0. Currently it is not possible
to have SPDX 2.2 and SPDX 3.0 enabled at the same time; users may choose
to specifically use SPDX 3.0 instead of 2.2 by doing INHERIT += "create-spdx-3.0"

The default version is not changed yet, although that will likely happen
soon

Quoted 7 lines

Joshua Watt (6):
classes-recipe/image: Add image file manifest classes/spdx-common: Move common SPDX to new class classes/spdx-common: Add SPDX version to path classes/spdx-common: Return empty list from extract_licenses classes/create-spdx-3.0: Add class classes-recipe/image_types: Add SPDX_IMAGE_PURPOSE to images

I set 3.0 to the default for testing and there were some interesting looking failures:

Quoted 7 lines

https://valkyrie.yoctoproject.org/#/builders/43/builds/52/steps/11/logs/stdio

I'm guessing:

https://valkyrie.yoctoproject.org/#/builders/37/builds/60

is due to a missing patch?

Hah, I don't think so, I just neglected to test SDK builds. My bad.

Quoted 5 lines

Other failures:

https://valkyrie.yoctoproject.org/#/builders/74/builds/44/steps/13/logs/stdio
https://valkyrie.yoctoproject.org/#/builders/93/builds/51/steps/11/logs/stdio

That one appears to be some corner case I need to figure out.

Quoted 2 lines

but I stopped the build at this point.

Thanks

Quoted 4 lines

Cheers,

Richard

#15 lore raw

v2 cover Add SPDX 3.0 support

In reply to Joshua Watt

This patch series add support for SPDX 3.0 and sets it as the default. Currently it is not possible to have SPDX 2.2 and SPDX 3.0 enabled at the same time; users may choose to specifically use SPDX 3.0 instead of 2.2 by doing INHERIT += "create-spdx-3.0"

v2: Added tests and addressed feedback

Joshua Watt (7):
classes-recipe/image: Add image file manifest classes/create-spdx-3.0: Add class classes-recipe/image_types: Add SPDX_IMAGE_PURPOSE to images
selftest: spdx: Add SPDX 3.0 test cases
classes-recipe: nospdx: Add class classes/spdx-common: Move SPDX_SUPPLIER
Switch default spdx version to 3.0

 meta/classes-recipe/image.bbclass            |   58 +
 meta/classes-recipe/image_types.bbclass      |    2 +
 meta/classes-recipe/image_types_wic.bbclass  |    1 +
 meta/classes-recipe/nospdx.bbclass           |   13 +
 meta/classes-recipe/packagegroup.bbclass     |    2 +
 meta/classes/create-spdx-2.2.bbclass         |    8 +
 meta/classes/create-spdx-3.0.bbclass         | 1116 ++++
 meta/classes/create-spdx-image-3.0.bbclass   |  203 +
 meta/classes/create-spdx.bbclass             |    2 +-
 meta/classes/spdx-common.bbclass             |   32 +-
 meta/lib/oe/sbom30.py                        | 1118 ++++
 meta/lib/oe/spdx30.py                        | 5996 ++++++++++++++++++
 meta/lib/oeqa/selftest/cases/spdx.py         |  119 +-
 meta/recipes-core/meta/build-sysroots.bb     |    5 +-
 meta/recipes-core/meta/meta-world-pkgdata.bb |    3 +-
 15 files changed, 8654 insertions(+), 24 deletions(-)
 create mode 100644 meta/classes-recipe/nospdx.bbclass
 create mode 100644 meta/classes/create-spdx-3.0.bbclass
 create mode 100644 meta/classes/create-spdx-image-3.0.bbclass
 create mode 100644 meta/lib/oe/sbom30.py
 create mode 100644 meta/lib/oe/spdx30.py
Signature · 1 line
2.43.2
#16 lore raw

PATCH v2 1/7 classes-recipe/image: Add image file manifest

Downstream tasks may want to know what image files were written by the do_image family of tasks (e.g. SPDX) so have each task write out a manifest file that describes the files it produced, then aggregate them in do_image_complete

---
 meta/classes-recipe/image.bbclass | 58 +++++++++++++++++++++++++++++++
 1 file changed, 58 insertions(+)
meta/classes-recipe/image.bbclass +58 -0
index 28be6c63623..a9347adb832 100644
@@ -88,6 +88,11 @@ PACKAGE_INSTALL_ATTEMPTONLY ?= "${FEATURE_INSTALL_OPTIONAL}"
 
 IMGDEPLOYDIR = "${WORKDIR}/deploy-${PN}-image-complete"
 
+IMGMANIFESTDIR = "${WORKDIR}/image-task-manifest"
+
+IMAGE_FILE_MANIFEST_DIR = "${WORKDIR}/deploy-image-file-manifest"
+IMAGE_FILE_MANIFEST = "${IMAGE_FILE_MANIFEST_DIR}/manifest.json"
+
 # Images are generally built explicitly, do not need to be part of world.
 EXCLUDE_FROM_WORLD = "1"
 
@@ -277,14 +282,28 @@ fakeroot python do_image () {
     execute_pre_post_process(d, pre_process_cmds)
 }
 do_image[dirs] = "${TOPDIR}"
+do_image[cleandirs] += "${IMGMANIFESTDIR}"
 addtask do_image after do_rootfs
 
 fakeroot python do_image_complete () {
     from oe.utils import execute_pre_post_process
+    from pathlib import Path
+    import json
 
     post_process_cmds = d.getVar("IMAGE_POSTPROCESS_COMMAND")
 
     execute_pre_post_process(d, post_process_cmds)
+
+    image_manifest_dir = Path(d.getVar('IMGMANIFESTDIR'))
+
+    data = []
+
+    for manifest_path in image_manifest_dir.glob("*.json"):
+        with manifest_path.open("r") as f:
+            data.extend(json.load(f))
+
+    with open(d.getVar("IMAGE_FILE_MANIFEST"), "w") as f:
+        json.dump(data, f)
 }
 do_image_complete[dirs] = "${TOPDIR}"
 SSTATETASKS += "do_image_complete"
@@ -292,6 +311,8 @@ SSTATE_SKIP_CREATION:task-image-complete = '1'
 do_image_complete[sstate-inputdirs] = "${IMGDEPLOYDIR}"
 do_image_complete[sstate-outputdirs] = "${DEPLOY_DIR_IMAGE}"
 do_image_complete[stamp-extra-info] = "${MACHINE_ARCH}"
+do_image_complete[sstate-plaindirs] =  "${IMAGE_FILE_MANIFEST_DIR}"
+do_image_complete[dirs] += "${IMAGE_FILE_MANIFEST_DIR}"
 addtask do_image_complete after do_image before do_build
 python do_image_complete_setscene () {
     sstate_setscene(d)
@@ -507,12 +528,14 @@ python () {
         d.setVar(task, '\n'.join(cmds))
         d.setVarFlag(task, 'func', '1')
         d.setVarFlag(task, 'fakeroot', '1')
+        d.setVarFlag(task, 'imagetype', t)
 
         d.appendVarFlag(task, 'prefuncs', ' ' + debug + ' set_image_size')
         d.prependVarFlag(task, 'postfuncs', 'create_symlinks ')
         d.appendVarFlag(task, 'subimages', ' ' + ' '.join(subimages))
         d.appendVarFlag(task, 'vardeps', ' ' + ' '.join(vardeps))
         d.appendVarFlag(task, 'vardepsexclude', ' DATETIME DATE ' + ' '.join(vardepsexclude))
+        d.appendVarFlag(task, 'postfuncs', ' write_image_file_manifest')
 
         bb.debug(2, "Adding task %s before %s, after %s" % (task, 'do_image_complete', after))
         bb.build.addtask(task, 'do_image_complete', after, d)
@@ -610,6 +633,41 @@ python create_symlinks() {
             bb.note("Skipping symlink, source does not exist: %s -> %s" % (dst, src))
 }
 
+python write_image_file_manifest() {
+    import json
+    from pathlib import Path
+
+    taskname = d.getVar("BB_CURRENTTASK")
+    image_deploy_dir = Path(d.getVar('IMGDEPLOYDIR'))
+    image_manifest_dir = Path(d.getVar('IMGMANIFESTDIR'))
+    manifest_path = image_manifest_dir / ("do_" + d.getVar("BB_CURRENTTASK") + ".json")
+
+    image_name = d.getVar("IMAGE_NAME")
+    image_basename = d.getVar("IMAGE_BASENAME")
+    machine = d.getVar("MACHINE")
+
+    subimages = (d.getVarFlag("do_" + taskname, 'subimages', False) or "").split()
+    imagetype = d.getVarFlag("do_" + taskname, 'imagetype', False)
+
+    data = {
+        "taskname": taskname,
+        "imagetype": imagetype,
+        "images": []
+    }
+
+    for type in subimages:
+        image_filename = image_name + "." + type
+        image_path = image_deploy_dir / image_filename
+        if not image_path.exists():
+            continue
+        data["images"].append({
+            "filename": image_filename,
+        })
+
+    with manifest_path.open("w") as f:
+        json.dump([data], f)
+}
+
 MULTILIBRE_ALLOW_REP += "${base_bindir} ${base_sbindir} ${bindir} ${sbindir} ${libexecdir} ${sysconfdir} ${nonarch_base_libdir}/udev /lib/modules/[^/]*/modules.*"
 MULTILIB_CHECK_FILE = "${WORKDIR}/multilib_check.py"
 MULTILIB_TEMP_ROOTFS = "${WORKDIR}/multilib"
Signature · 1 line
2.43.2
#17 lore raw

PATCH v2 2/7 classes/create-spdx-3.0: Add classs

In reply to Joshua Watt

Adds a class to generate SPDX 3.0 output and an image class that is used when generating images

---
 meta/classes-recipe/packagegroup.bbclass   |    2 +
 meta/classes/create-spdx-3.0.bbclass       | 1116 ++++
 meta/classes/create-spdx-image-3.0.bbclass |  203 +
 meta/classes/spdx-common.bbclass           |   25 +-
 meta/lib/oe/sbom30.py                      | 1118 ++++
 meta/lib/oe/spdx30.py                      | 5996 ++++++++++++++++++++
 6 files changed, 8459 insertions(+), 1 deletion(-)
 create mode 100644 meta/classes/create-spdx-3.0.bbclass
 create mode 100644 meta/classes/create-spdx-image-3.0.bbclass
 create mode 100644 meta/lib/oe/sbom30.py
 create mode 100644 meta/lib/oe/spdx30.py
meta/classes-recipe/packagegroup.bbclass +2 -0
index cf6fc354a81..2099c8b37d6 100644
@@ -56,6 +56,8 @@ deltask do_populate_sysroot
 
 do_create_runtime_spdx[deptask] = "do_create_spdx"
 do_create_runtime_spdx[rdeptask] = ""
+do_create_package_spdx[deptask] = "do_create_spdx"
+do_create_package_spdx[rdeptask] = ""
 
 INHIBIT_DEFAULT_DEPS = "1"
 
meta/classes/create-spdx-3.0.bbclass new file +1116 -0

1,116 lines not on this page · raw message

meta/classes/create-spdx-image-3.0.bbclass new file +203 -0

203 lines not on this page · raw message

meta/classes/spdx-common.bbclass +24 -1
index 03f1d0cc278..dc2ecb01009 100644
@@ -16,13 +16,17 @@ SPDXDIR ??= "${WORKDIR}/spdx/${SPDX_VERSION}"
 SPDXDEPLOY = "${SPDXDIR}/deploy"
 SPDXWORK = "${SPDXDIR}/work"
 SPDXIMAGEWORK = "${SPDXDIR}/image-work"
+SPDXROOTFSWORK = "${SPDXDIR}/rootfs-work"
 SPDXSDKWORK = "${SPDXDIR}/sdk-work"
 SPDXDEPS = "${SPDXDIR}/deps.json"
+SPDXIMAGEDEPLOYDIR = "${SPDXDIR}/image-deploy"
+SPDX_ROOTFS_PACKAGES = "${SPDXDIR}/rootfs-packages.json"
 
 SPDX_TOOL_NAME ??= "oe-spdx-creator"
 SPDX_TOOL_VERSION ??= "1.0"
 
 SPDXRUNTIMEDEPLOY = "${SPDXDIR}/runtime-deploy"
+SPDXROOTFSDEPLOY = "${SPDXDIR}/rootfs-deploy"
 
 SPDX_INCLUDE_SOURCES ??= "0"
 SPDX_ARCHIVE_SOURCES ??= "0"
@@ -68,7 +72,7 @@ def get_json_indent(d):
         return 2
     return None
 
-python() {
+def load_spdx_license_data(d):
     import json
     if d.getVar("SPDX_LICENSE_DATA"):
         return
@@ -78,6 +82,9 @@ python() {
         # Transform the license array to a dictionary
         data["licenses"] = {l["licenseId"]: l for l in data["licenses"]}
         d.setVar("SPDX_LICENSE_DATA", data)
+
+python() {
+    load_spdx_license_data(d)
 }
 
 def process_sources(d):
@@ -255,3 +262,19 @@ def spdx_get_src(d):
 
 spdx_get_src[vardepsexclude] += "STAGING_KERNEL_DIR"
 
+
+python spdx_collect_rootfs_packages() {
+    import json
+    from pathlib import Path
+    from oe.rootfs import image_list_installed_packages
+
+    root_packages_file = Path(d.getVar("SPDX_ROOTFS_PACKAGES"))
+
+    packages = image_list_installed_packages(d)
+
+    root_packages_file.parent.mkdir(parents=True, exist_ok=True)
+    with root_packages_file.open("w") as f:
+        json.dump(packages, f)
+}
+ROOTFS_POSTUNINSTALL_COMMAND =+ "spdx_collect_rootfs_packages"
+
meta/lib/oe/sbom30.py new file +1118 -0

1,118 lines not on this page · raw message

meta/lib/oe/spdx30.py new file +5996 -0

502 lines not on this page · raw message

The rest of this diff is not shown. View the raw message.

Signature · 1 line
2.43.2

Part of the diff is not shown here. View the raw message.

#18 lore raw

Patchtest results for [OE-core][PATCH v2 2/7] classes/create-spdx-3.0: Add classs

Thank you for your submission. Patchtest identified one or more issues with the patch. Please see the log below for more information:

---
Testing patch /home/patchtest/share/mboxes/v2-2-7-classes-create-spdx-3.0-Add-classs.patch

FAIL: test max line length: Patch line too long (current length 216, maximum is 200) (test_metadata.TestMetadata.test_max_line_length)

PASS: pretest pylint (test_python_pylint.PyLint.pretest_pylint)
PASS: test Signed-off-by presence (test_mbox.TestMbox.test_signed_off_by_presence)
PASS: test author valid (test_mbox.TestMbox.test_author_valid)
PASS: test commit message presence (test_mbox.TestMbox.test_commit_message_presence)
PASS: test mbox format (test_mbox.TestMbox.test_mbox_format)
PASS: test non-AUH upgrade (test_mbox.TestMbox.test_non_auh_upgrade)
PASS: test pylint (test_python_pylint.PyLint.test_pylint)
PASS: test shortlog format (test_mbox.TestMbox.test_shortlog_format)
PASS: test shortlog length (test_mbox.TestMbox.test_shortlog_length)

SKIP: pretest src uri left files: No modified recipes, skipping pretest (test_metadata.TestMetadata.pretest_src_uri_left_files)
SKIP: test CVE check ignore: No modified recipes or older target branch, skipping test (test_metadata.TestMetadata.test_cve_check_ignore)
SKIP: test CVE tag format: No new CVE patches introduced (test_patch.TestPatch.test_cve_tag_format)
SKIP: test Signed-off-by presence: No new CVE patches introduced (test_patch.TestPatch.test_signed_off_by_presence)
SKIP: test Upstream-Status presence: No new CVE patches introduced (test_patch.TestPatch.test_upstream_status_presence_format)
SKIP: test bugzilla entry format: No bug ID found (test_mbox.TestMbox.test_bugzilla_entry_format)
SKIP: test lic files chksum modified not mentioned: No modified recipes, skipping test (test_metadata.TestMetadata.test_lic_files_chksum_modified_not_mentioned)
SKIP: test lic files chksum presence: No added recipes, skipping test (test_metadata.TestMetadata.test_lic_files_chksum_presence)
SKIP: test license presence: No added recipes, skipping test (test_metadata.TestMetadata.test_license_presence)
SKIP: test series merge on head: Merge test is disabled for now (test_mbox.TestMbox.test_series_merge_on_head)
SKIP: test src uri left files: No modified recipes, skipping pretest (test_metadata.TestMetadata.test_src_uri_left_files)
SKIP: test summary presence: No added recipes, skipping test (test_metadata.TestMetadata.test_summary_presence)
SKIP: test target mailing list: Series merged, no reason to check other mailing lists (test_mbox.TestMbox.test_target_mailing_list)

---

Please address the issues identified and
submit a new revision of the patch, or alternatively, reply to this email with an explanation of why the patch should be accepted. If you believe these results are due to an error in patchtest, please submit a bug at https://bugzilla.yoctoproject.org/ (use the 'Patchtest' category under 'Yocto Project Subprojects'). For more information on specific failures, see: https://wiki.yoctoproject.org/wiki/Patchtest. Thank you!

#19 lore raw

PATCH v2 3/7 classes-recipe/image_types: Add SPDX_IMAGE_PURPOSE to images

In reply to Joshua Watt

Adds the variable overrides to set the SPDX image purpose for various image types

---
 meta/classes-recipe/image_types.bbclass     | 2 ++
 meta/classes-recipe/image_types_wic.bbclass | 1 +
 2 files changed, 3 insertions(+)
meta/classes-recipe/image_types.bbclass +2 -0
index 2f948ecbf88..506b9934cb7 100644
@@ -146,6 +146,7 @@ IMAGE_CMD:vfat = "oe_mkvfatfs ${EXTRA_IMAGECMD}"
 IMAGE_CMD_TAR ?= "tar"
 # ignore return code 1 "file changed as we read it" as other tasks(e.g. do_image_wic) may be hardlinking rootfs
 IMAGE_CMD:tar = "${IMAGE_CMD_TAR} --sort=name --format=posix --numeric-owner -cf ${IMGDEPLOYDIR}/${IMAGE_NAME}.tar -C ${IMAGE_ROOTFS} . || [ $? -eq 1 ]"
+SPDX_IMAGE_PURPOSE:tar = "archive"
 
 do_image_cpio[cleandirs] += "${WORKDIR}/cpio_append"
 IMAGE_CMD:cpio () {
@@ -167,6 +168,7 @@ IMAGE_CMD:cpio () {
 		fi
 	fi
 }
+SPDX_IMAGE_PURPOSE:cpio = "archive"
 
 UBI_VOLNAME ?= "${MACHINE}-rootfs"
 UBI_VOLTYPE ?= "dynamic"
meta/classes-recipe/image_types_wic.bbclass +1 -0
index cf3be909b30..86f40633ebc 100644
@@ -91,6 +91,7 @@ IMAGE_CMD:wic () {
 	mv "$build_wic/$(basename "${wks%.wks}")"*.${IMAGER} "$out.wic"
 }
 IMAGE_CMD:wic[vardepsexclude] = "WKS_FULL_PATH WKS_FILES TOPDIR"
+SPDX_IMAGE_PURPOSE:wic = "diskImage"
 do_image_wic[cleandirs] = "${WORKDIR}/build-wic"
 
 PSEUDO_IGNORE_PATHS .= ",${WORKDIR}/build-wic"
Signature · 1 line
2.43.2
#20 lore raw

PATCH v2 4/7 selftest: spdx: Add SPDX 3.0 test cases

In reply to Joshua Watt

Adds test cases for SPDX 3.0. Reworks the SPDX 2.2 test setup so it can also be run even if the default is SPDX 3.0

---
 meta/lib/oeqa/selftest/cases/spdx.py | 119 +++++++++++++++++++++++++--
 1 file changed, 110 insertions(+), 9 deletions(-)
meta/lib/oeqa/selftest/cases/spdx.py +110 -9
index 7685a81e7fb..0ae49aec333 100644
@@ -6,21 +6,26 @@
 
 import json
 import os
+import textwrap
+from pathlib import Path
 from oeqa.selftest.case import OESelftestTestCase
-from oeqa.utils.commands import bitbake, get_bb_var, runCmd
+from oeqa.utils.commands import bitbake, get_bb_var, get_bb_vars, runCmd
 
-class SPDXCheck(OESelftestTestCase):
 
+class SPDX22Check(OESelftestTestCase):
     @classmethod
     def setUpClass(cls):
-        super(SPDXCheck, cls).setUpClass()
+        super().setUpClass()
         bitbake("python3-spdx-tools-native")
         bitbake("-c addto_recipe_sysroot python3-spdx-tools-native")
 
     def check_recipe_spdx(self, high_level_dir, spdx_file, target_name):
-        config = """
-INHERIT += "create-spdx"
-"""
+        config = textwrap.dedent(
+            """\
+            INHERIT:remove = "create-spdx"
+            INHERIT += "create-spdx-2.2"
+            """
+        )
         self.write_config(config)
 
         deploy_dir = get_bb_var("DEPLOY_DIR")
@@ -29,7 +34,9 @@ INHERIT += "create-spdx"
         # qemux86-64 creates the directory qemux86_64
         machine_dir = machine_var.replace("-", "_")
 
-        full_file_path = os.path.join(deploy_dir, "spdx", spdx_version, machine_dir, high_level_dir, spdx_file)
+        full_file_path = os.path.join(
+            deploy_dir, "spdx", spdx_version, machine_dir, high_level_dir, spdx_file
+        )
 
         try:
             os.remove(full_file_path)
@@ -44,8 +51,13 @@ INHERIT += "create-spdx"
                 self.assertNotEqual(report, None)
                 self.assertNotEqual(report["SPDXID"], None)
 
-            python = os.path.join(get_bb_var('STAGING_BINDIR', 'python3-spdx-tools-native'), 'nativepython3')
-            validator = os.path.join(get_bb_var('STAGING_BINDIR', 'python3-spdx-tools-native'), 'pyspdxtools')
+            python = os.path.join(
+                get_bb_var("STAGING_BINDIR", "python3-spdx-tools-native"),
+                "nativepython3",
+            )
+            validator = os.path.join(
+                get_bb_var("STAGING_BINDIR", "python3-spdx-tools-native"), "pyspdxtools"
+            )
             result = runCmd("{} {} -i {}".format(python, validator, filename))
 
         self.assertExists(full_file_path)
@@ -53,3 +65,92 @@ INHERIT += "create-spdx"
 
     def test_spdx_base_files(self):
         self.check_recipe_spdx("packages", "base-files.spdx.json", "base-files")
+
+
+class SPDX3CheckBase(object):
+    """
+    Base class for checking SPDX 3 based tests
+    """
+
+    def check_spdx_file(self, filename):
+        import oe.spdx30
+
+        self.assertExists(filename)
+
+        # Read the file
+        objset = oe.spdx30.SHACLObjectSet()
+        with open(filename, "r") as f:
+            d = oe.spdx30.JSONLDDeserializer()
+            d.read(f, objset)
+
+        return objset
+
+    def check_recipe_spdx(self, target_name, spdx_path, *, task=None):
+        config = textwrap.dedent(
+            f"""\
+            INHERIT:remove = "create-spdx"
+            INHERIT += "{self.SPDX_CLASS}"
+            """
+        )
+        self.write_config(config)
+
+        if task:
+            bitbake(f"-c {task} {target_name}")
+        else:
+            bitbake(target_name)
+
+        filename = spdx_path.format(
+            **get_bb_vars(
+                [
+                    "DEPLOY_DIR_IMAGE",
+                    "DEPLOY_DIR_SPDX",
+                    "MACHINE",
+                    "MACHINE_ARCH",
+                    "SDKMACHINE",
+                    "SDK_DEPLOY",
+                    "SPDX_VERSION",
+                    "TOOLCHAIN_OUTPUTNAME",
+                ],
+                target_name,
+            )
+        )
+
+        return self.check_spdx_file(filename)
+
+    def check_objset_missing_ids(self, objset):
+        if objset.missing_ids:
+            self.assertTrue(
+                False,
+                "The following SPDXIDs are unresolved:\n  "
+                + "\n  ".join(objset.missing_ids),
+            )
+
+
+class SPDX30Check(SPDX3CheckBase, OESelftestTestCase):
+    SPDX_CLASS = "create-spdx-3.0"
+    SCHEMA_URL = "https://spdx.org/schema/3.0.0/spdx-json-schema.json"
+
+    def test_base_files(self):
+        self.check_recipe_spdx(
+            "base-files",
+            "{DEPLOY_DIR_SPDX}/{MACHINE_ARCH}/packages/base-files.spdx.json",
+        )
+
+    def test_core_image_minimal(self):
+        objset = self.check_recipe_spdx(
+            "core-image-minimal",
+            "{DEPLOY_DIR_IMAGE}/core-image-minimal-{MACHINE}.rootfs.spdx.json",
+        )
+
+        # Document should be fully linked
+        self.check_objset_missing_ids(objset)
+
+    def test_core_image_minimal_sdk(self):
+        objset = self.check_recipe_spdx(
+            "core-image-minimal",
+            "{SDK_DEPLOY}/{TOOLCHAIN_OUTPUTNAME}.spdx.json",
+            task="populate_sdk",
+        )
+
+        # Document should be fully linked
+        self.check_objset_missing_ids(objset)
Signature · 1 line
2.43.2
#21 lore raw

PATCH v2 5/7 classes-recipe: nospdx: Add class

In reply to Joshua Watt

Adds a class that allows recipes to opt out of generating SPDX

---
 meta/classes-recipe/nospdx.bbclass           | 13 +++++++++++++
 meta/recipes-core/meta/build-sysroots.bb     |  5 +----
 meta/recipes-core/meta/meta-world-pkgdata.bb |  3 +--
 3 files changed, 15 insertions(+), 6 deletions(-)
 create mode 100644 meta/classes-recipe/nospdx.bbclass
meta/classes-recipe/nospdx.bbclass new file +13 -0
new file mode 100644
index 00000000000..b20e28218be
@@ -0,0 +1,13 @@
+#
+# Copyright OpenEmbedded Contributors
+#
+# SPDX-License-Identifier: MIT
+#
+
+deltask do_collect_spdx_deps
+deltask do_create_spdx
+deltask do_create_spdx_runtime
+deltask do_create_package_spdx
+deltask do_create_rootfs_spdx
+deltask do_create_image_spdx
+deltask do_create_image_sbom
meta/recipes-core/meta/build-sysroots.bb +1 -4
index db05c111ab2..b0b8fb3c79a 100644
@@ -7,7 +7,7 @@ STANDALONE_SYSROOT_NATIVE = "${STAGING_DIR}/${BUILD_ARCH}"
 PACKAGE_ARCH = "${MACHINE_ARCH}"
 EXCLUDE_FROM_WORLD = "1"
 
-inherit nopackages
+inherit nopackages nospdx
 deltask fetch
 deltask unpack
 deltask patch
@@ -17,9 +17,6 @@ deltask configure
 deltask compile
 deltask install
 deltask populate_sysroot
-deltask create_spdx
-deltask collect_spdx_deps
-deltask create_runtime_spdx
 deltask recipe_qa
 
 do_build_warn () {
meta/recipes-core/meta/meta-world-pkgdata.bb +1 -2
index 0438bf61387..244175ddd44 100644
@@ -27,14 +27,13 @@ python do_collect_packagedata() {
     oe.copy_buildsystem.generate_locked_sigs(sigfile, d)
 }
 
+inherit nospdx
 deltask do_fetch
 deltask do_unpack
 deltask do_patch
 deltask do_configure
 deltask do_compile
 deltask do_install
-deltask do_create_spdx
-deltask do_create_spdx_runtime
 
 do_prepare_recipe_sysroot[deptask] = ""
 
Signature · 1 line
2.43.2
#22 lore raw

PATCH v2 6/7 classes/spdx-common: Move SPDX_SUPPLIER

In reply to Joshua Watt

Move the SPDX_SUPPLIER variable to create-spdx-2.2 since it's format only has meaning in SPDX 2.2 (SPDX 3 uses SPDX_PACKAGE_SUPPLIER with a different format)

---
 meta/classes/create-spdx-2.2.bbclass | 8 ++++++++
 meta/classes/spdx-common.bbclass     | 7 -------
 2 files changed, 8 insertions(+), 7 deletions(-)
meta/classes/create-spdx-2.2.bbclass +8 -0
index 3ebf92b5e12..99061320e53 100644
@@ -8,6 +8,14 @@ inherit spdx-common
 
 SPDX_VERSION = "2.2"
 
+SPDX_ORG ??= "OpenEmbedded ()"
+SPDX_SUPPLIER ??= "Organization: ${SPDX_ORG}"
+SPDX_SUPPLIER[doc] = "The SPDX PackageSupplier field for SPDX packages created from \
+    this recipe. For SPDX documents create using this class during the build, this \
+    is the contact information for the person or organization who is doing the \
+    build."
+
+
 def get_namespace(d, name):
     import uuid
     namespace_uuid = uuid.uuid5(uuid.NAMESPACE_DNS, d.getVar("SPDX_UUID_NAMESPACE"))
meta/classes/spdx-common.bbclass +0 -7
index dc2ecb01009..48abdeb55eb 100644
@@ -40,13 +40,6 @@ SPDX_LICENSES ??= "${COREBASE}/meta/files/spdx-licenses.json"
 
 SPDX_CUSTOM_ANNOTATION_VARS ??= ""
 
-SPDX_ORG ??= "OpenEmbedded ()"
-SPDX_SUPPLIER ??= "Organization: ${SPDX_ORG}"
-SPDX_SUPPLIER[doc] = "The SPDX PackageSupplier field for SPDX packages created from \
-    this recipe. For SPDX documents create using this class during the build, this \
-    is the contact information for the person or organization who is doing the \
-    build."
-
 def extract_licenses(filename):
     import re
 
Signature · 1 line
2.43.2
#23 lore raw

PATCH v2 7/7 Switch default spdx version to 3.0

In reply to Joshua Watt

Changes the default SPDX version to 3.0

---
 meta/classes/create-spdx.bbclass | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
meta/classes/create-spdx.bbclass +1 -1
index 19c6c0ff0b9..b604973ae0a 100644
@@ -5,4 +5,4 @@
 #
 # Include this class when you don't care what version of SPDX you get; it will
 # be updated to the latest stable version that is supported
-inherit create-spdx-2.2
+inherit create-spdx-3.0
Signature · 1 line
2.43.2
#24 lore raw

Patchtest results for [OE-core][PATCH v2 7/7] Switch default spdx version to 3.0

Thank you for your submission. Patchtest identified one or more issues with the patch. Please see the log below for more information:

---
Testing patch /home/patchtest/share/mboxes/v2-7-7-Switch-default-spdx-version-to-3.0.patch

FAIL: test shortlog format: Commit shortlog (first line of commit message) should follow the format "<target>: <summary>" (test_mbox.TestMbox.test_shortlog_format)

PASS: test Signed-off-by presence (test_mbox.TestMbox.test_signed_off_by_presence)
PASS: test author valid (test_mbox.TestMbox.test_author_valid)
PASS: test commit message presence (test_mbox.TestMbox.test_commit_message_presence)
PASS: test mbox format (test_mbox.TestMbox.test_mbox_format)
PASS: test non-AUH upgrade (test_mbox.TestMbox.test_non_auh_upgrade)
PASS: test shortlog length (test_mbox.TestMbox.test_shortlog_length)

SKIP: pretest pylint: No python related patches, skipping test (test_python_pylint.PyLint.pretest_pylint)
SKIP: pretest src uri left files: No modified recipes, skipping pretest (test_metadata.TestMetadata.pretest_src_uri_left_files)
SKIP: test CVE tag format: No new CVE patches introduced (test_patch.TestPatch.test_cve_tag_format)
SKIP: test Signed-off-by presence: No new CVE patches introduced (test_patch.TestPatch.test_signed_off_by_presence)
SKIP: test Upstream-Status presence: No new CVE patches introduced (test_patch.TestPatch.test_upstream_status_presence_format)
SKIP: test bugzilla entry format: No bug ID found (test_mbox.TestMbox.test_bugzilla_entry_format)
SKIP: test pylint: No python related patches, skipping test (test_python_pylint.PyLint.test_pylint)
SKIP: test series merge on head: Merge test is disabled for now (test_mbox.TestMbox.test_series_merge_on_head)
SKIP: test target mailing list: Series merged, no reason to check other mailing lists (test_mbox.TestMbox.test_target_mailing_list)

---

Please address the issues identified and
submit a new revision of the patch, or alternatively, reply to this email with an explanation of why the patch should be accepted. If you believe these results are due to an error in patchtest, please submit a bug at https://bugzilla.yoctoproject.org/ (use the 'Patchtest' category under 'Yocto Project Subprojects'). For more information on specific failures, see: https://wiki.yoctoproject.org/wiki/Patchtest. Thank you!

#25 lore raw
In reply to Joshua Watt

Hey Joshua

https://errors.yoctoproject.org/Errors/Build/187139/

does this seem related to this changeset

Quoted 52 lines · Joshua Watt via lists.openembedded.org

This patch series add support for SPDX 3.0 and sets it as the default. Currently it is not possible to have SPDX 2.2 and SPDX 3.0 enabled at the same time; users may choose to specifically use SPDX 3.0 instead of 2.2 by doing INHERIT += "create-spdx-3.0"

v2: Added tests and addressed feedback

Joshua Watt (7):
classes-recipe/image: Add image file manifest classes/create-spdx-3.0: Add class classes-recipe/image_types: Add SPDX_IMAGE_PURPOSE to images
selftest: spdx: Add SPDX 3.0 test cases
classes-recipe: nospdx: Add class classes/spdx-common: Move SPDX_SUPPLIER
Switch default spdx version to 3.0

 meta/classes-recipe/image.bbclass            |   58 +
 meta/classes-recipe/image_types.bbclass      |    2 +
 meta/classes-recipe/image_types_wic.bbclass  |    1 +
 meta/classes-recipe/nospdx.bbclass           |   13 +
 meta/classes-recipe/packagegroup.bbclass     |    2 +
 meta/classes/create-spdx-2.2.bbclass         |    8 +
 meta/classes/create-spdx-3.0.bbclass         | 1116 ++++
 meta/classes/create-spdx-image-3.0.bbclass   |  203 +
 meta/classes/create-spdx.bbclass             |    2 +-
 meta/classes/spdx-common.bbclass             |   32 +-
 meta/lib/oe/sbom30.py                        | 1118 ++++
 meta/lib/oe/spdx30.py                        | 5996 ++++++++++++++++++
 meta/lib/oeqa/selftest/cases/spdx.py         |  119 +-
 meta/recipes-core/meta/build-sysroots.bb     |    5 +-
 meta/recipes-core/meta/meta-world-pkgdata.bb |    3 +-
 15 files changed, 8654 insertions(+), 24 deletions(-)
 create mode 100644 meta/classes-recipe/nospdx.bbclass
 create mode 100644 meta/classes/create-spdx-3.0.bbclass
 create mode 100644 meta/classes/create-spdx-image-3.0.bbclass
 create mode 100644 meta/lib/oe/sbom30.py
 create mode 100644 meta/lib/oe/spdx30.py

--
2.43.2

-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#200929): https://lists.openembedded.org/g/openembedded-core/message/200929
Mute This Topic: https://lists.openembedded.org/mt/106770352/1997914
Group Owner: [email protected]
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [[email protected]]
-=-=-=-=-=-=-=-=-=-=-=-

#26 lore raw
In reply to Joshua Watt

Unfortunately, this still causes failures:

https://autobuilder.yoctoproject.org/typhoon/#/builders/83/builds/7060/steps/23/logs/stdio

https://autobuilder.yoctoproject.org/typhoon/#/builders/23/builds/9481/steps/14/logs/stdio

https://autobuilder.yoctoproject.org/typhoon/#/builders/20/builds/9574/steps/11/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/89/builds/9149/steps/12/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/82/builds/6555/steps/12/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/117/builds/4969/steps/12/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/65/builds/9220/steps/11/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/166/builds/518/steps/11/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/165/builds/514/steps/11/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/48/builds/9186/steps/12/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/37/builds/9166/steps/11/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/104/builds/8044/steps/11/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/103/builds/8038/steps/11/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/100/builds/6112/steps/12/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/64/builds/9159/steps/12/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/40/builds/9145/steps/11/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/81/builds/6733/steps/12/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/52/builds/8994/steps/12/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/108/builds/6122/steps/12/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/108/builds/6122/steps/12/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/58/builds/8923/steps/15/logs/stdio

https://autobuilder.yoctoproject.org/typhoon/#/builders/80/builds/6837/steps/14/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/79/builds/6881/steps/14/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/86/builds/6906/steps/14/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/87/builds/6892/steps/14/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/127/builds/3501/steps/15/logs/stdio

https://autobuilder.yoctoproject.org/typhoon/#/builders/76/builds/9099/steps/23/logs/stdio

Quoted 52 lines · Joshua Watt

This patch series add support for SPDX 3.0 and sets it as the default. Currently it is not possible to have SPDX 2.2 and SPDX 3.0 enabled at the same time; users may choose to specifically use SPDX 3.0 instead of 2.2 by doing INHERIT += "create-spdx-3.0"

v2: Added tests and addressed feedback

Joshua Watt (7):
classes-recipe/image: Add image file manifest classes/create-spdx-3.0: Add class classes-recipe/image_types: Add SPDX_IMAGE_PURPOSE to images
selftest: spdx: Add SPDX 3.0 test cases
classes-recipe: nospdx: Add class classes/spdx-common: Move SPDX_SUPPLIER
Switch default spdx version to 3.0

 meta/classes-recipe/image.bbclass            |   58 +
 meta/classes-recipe/image_types.bbclass      |    2 +
 meta/classes-recipe/image_types_wic.bbclass  |    1 +
 meta/classes-recipe/nospdx.bbclass           |   13 +
 meta/classes-recipe/packagegroup.bbclass     |    2 +
 meta/classes/create-spdx-2.2.bbclass         |    8 +
 meta/classes/create-spdx-3.0.bbclass         | 1116 ++++
 meta/classes/create-spdx-image-3.0.bbclass   |  203 +
 meta/classes/create-spdx.bbclass             |    2 +-
 meta/classes/spdx-common.bbclass             |   32 +-
 meta/lib/oe/sbom30.py                        | 1118 ++++
 meta/lib/oe/spdx30.py                        | 5996 ++++++++++++++++++
 meta/lib/oeqa/selftest/cases/spdx.py         |  119 +-
 meta/recipes-core/meta/build-sysroots.bb     |    5 +-
 meta/recipes-core/meta/meta-world-pkgdata.bb |    3 +-
 15 files changed, 8654 insertions(+), 24 deletions(-)
 create mode 100644 meta/classes-recipe/nospdx.bbclass
 create mode 100644 meta/classes/create-spdx-3.0.bbclass
 create mode 100644 m: failures:eta/classes/create-spdx-image-3.0.bbclass
 create mode 100644 meta/lib/oe/sbom30.py
 create mode 100644 meta/lib/oe/spdx30.py
Signature · 11 lines
2.43.2



-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#200929): https://lists.openembedded.org/g/openembedded-core/message/200929
Mute This Topic: https://lists.openembedded.org/mt/106770352/3617179
Group Owner: [email protected]
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [[email protected]]
-=-=-=-=-=-=-=-=-=-=-=-
Signature · 3 lines
Alexandre Belloni, co-owner and COO, Bootlin
Embedded Linux and Kernel engineering
https://bootlin.com
#27 lore raw
Quoted 2 lines · Alexandre Belloni

Unfortunately, this still causes failures:

Thanks, I'll take a look

Quoted 93 lines

https://autobuilder.yoctoproject.org/typhoon/#/builders/83/builds/7060/steps/23/logs/stdio

https://autobuilder.yoctoproject.org/typhoon/#/builders/23/builds/9481/steps/14/logs/stdio

https://autobuilder.yoctoproject.org/typhoon/#/builders/20/builds/9574/steps/11/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/89/builds/9149/steps/12/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/82/builds/6555/steps/12/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/117/builds/4969/steps/12/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/65/builds/9220/steps/11/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/166/builds/518/steps/11/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/165/builds/514/steps/11/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/48/builds/9186/steps/12/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/37/builds/9166/steps/11/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/104/builds/8044/steps/11/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/103/builds/8038/steps/11/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/100/builds/6112/steps/12/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/64/builds/9159/steps/12/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/40/builds/9145/steps/11/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/81/builds/6733/steps/12/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/52/builds/8994/steps/12/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/108/builds/6122/steps/12/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/108/builds/6122/steps/12/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/58/builds/8923/steps/15/logs/stdio

https://autobuilder.yoctoproject.org/typhoon/#/builders/80/builds/6837/steps/14/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/79/builds/6881/steps/14/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/86/builds/6906/steps/14/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/87/builds/6892/steps/14/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/127/builds/3501/steps/15/logs/stdio

https://autobuilder.yoctoproject.org/typhoon/#/builders/76/builds/9099/steps/23/logs/stdio

Quoted 44 lines · Joshua Watt

This patch series add support for SPDX 3.0 and sets it as the default. Currently it is not possible to have SPDX 2.2 and SPDX 3.0 enabled at the same time; users may choose to specifically use SPDX 3.0 instead of 2.2 by doing INHERIT += "create-spdx-3.0"

v2: Added tests and addressed feedback

Joshua Watt (7):
classes-recipe/image: Add image file manifest classes/create-spdx-3.0: Add class classes-recipe/image_types: Add SPDX_IMAGE_PURPOSE to images
selftest: spdx: Add SPDX 3.0 test cases
classes-recipe: nospdx: Add class classes/spdx-common: Move SPDX_SUPPLIER
Switch default spdx version to 3.0

 meta/classes-recipe/image.bbclass            |   58 +
 meta/classes-recipe/image_types.bbclass      |    2 +
 meta/classes-recipe/image_types_wic.bbclass  |    1 +
 meta/classes-recipe/nospdx.bbclass           |   13 +
 meta/classes-recipe/packagegroup.bbclass     |    2 +
 meta/classes/create-spdx-2.2.bbclass         |    8 +
 meta/classes/create-spdx-3.0.bbclass         | 1116 ++++
 meta/classes/create-spdx-image-3.0.bbclass   |  203 +
 meta/classes/create-spdx.bbclass             |    2 +-
 meta/classes/spdx-common.bbclass             |   32 +-
 meta/lib/oe/sbom30.py                        | 1118 ++++
 meta/lib/oe/spdx30.py                        | 5996 ++++++++++++++++++
 meta/lib/oeqa/selftest/cases/spdx.py         |  119 +-
 meta/recipes-core/meta/build-sysroots.bb     |    5 +-
 meta/recipes-core/meta/meta-world-pkgdata.bb |    3 +-
 15 files changed, 8654 insertions(+), 24 deletions(-)
 create mode 100644 meta/classes-recipe/nospdx.bbclass
 create mode 100644 meta/classes/create-spdx-3.0.bbclass
 create mode 100644 m: failures:eta/classes/create-spdx-image-3.0.bbclass
 create mode 100644 meta/lib/oe/sbom30.py
 create mode 100644 meta/lib/oe/spdx30.py

--
2.43.2

Quoted 8 lines

-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#200929): https://lists.openembedded.org/g/openembedded-core/message/200929
Mute This Topic: https://lists.openembedded.org/mt/106770352/3617179
Group Owner: [email protected]
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [[email protected]]
-=-=-=-=-=-=-=-=-=-=-=-

--
Alexandre Belloni, co-owner and COO, Bootlin
Embedded Linux and Kernel engineering
https://bootlin.com

#28 lore raw
In reply to Alexandre Belloni

This jogged my memory on why I added the patch to allow tasks to opt out of running extend_recipe_sysroot. I'm sending this to get it written down before I forget, we can talk about it in the tech meeting on Tuesday if necessary. There are two build failure modes that having do_create_spdx run extend_recipe_sysroot causes.

The first failure mode is when a recipe explicitly does do_populate_sysroot[noexec] = "1" (an example here is glib-y2038-tests). In this case, when do_create_spdx tries to run extend_recipe_sysroot, it sees that the task depends on do_populate_sysroot, and this looks for the glibc-y2038-tests:populate_sysroot manifest file, which doesn't exist because the task was not run which results in a failure. Example here:
https://autobuilder.yoctoproject.org/typhoon/#/builders/81/builds/6733/steps/12/logs/stdio

The second failure mode is (I think) a bug in binutils-cross-testsuite. This recipe is odd because it doesn't actually inherit cross.bbclass, and thus isn't _actually_ a cross recipe, despite the name. This confuses find_sstate_manifest(), which assumes that if the recipe has "-cross-" in the name, the package arch must be ${BUILD_ARCH}, which is not the case here because it doesn't inherit cross.bbclass. When do_create_spdx tries to run extend_recipe_sysroot, it is looking for the binutils-cross-testsuite:populate_sysroot manifest file using the wrong architecture, and thus errors out. I _think_ this might be fixed by renaming the recipe so that it no longer contains the string "-cross-". Example here:
https://errors.yoctoproject.org/Errors/Details/787278/

do_create_spdx doesn't actually need the sysroot extended; it only actually depends on do_populate_sysroot do that it can inspect the sysroot produced by the current recipe to add the files to the SPDX output. IIUC, the automatic call to extend_recipe_sysroot when a task depends on do_populate_ssyroot is mostly a heuristic designed to ensure that tasks get the full recipe sysroot correctly without having to manually invoke the call everywhere. However, in the case of do_create_spdx() it is actually not necessary, thus I added the flag to disable that behavior on a per-task basis, since I thought that might be better than hard coding an exception for do_create_spdx.

The other solution would be to identify each individual recipe that is doing something weird and disable SPDX generation in that specific recipe (I've made a new class called nospdx.bbclass that makes this easy). I'm not sure how many recipes this would be, probably only a few, but it might be confusing to other layers why failures occur and what to do about it.... and again since the heuristic for when to call extend_recipe_sysroot is incorrect for do_create_spdx, I'd prefer to explore that route.

Quoted 95 lines · Alexandre Belloni

Unfortunately, this still causes failures:

https://autobuilder.yoctoproject.org/typhoon/#/builders/83/builds/7060/steps/23/logs/stdio

https://autobuilder.yoctoproject.org/typhoon/#/builders/23/builds/9481/steps/14/logs/stdio

https://autobuilder.yoctoproject.org/typhoon/#/builders/20/builds/9574/steps/11/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/89/builds/9149/steps/12/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/82/builds/6555/steps/12/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/117/builds/4969/steps/12/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/65/builds/9220/steps/11/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/166/builds/518/steps/11/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/165/builds/514/steps/11/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/48/builds/9186/steps/12/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/37/builds/9166/steps/11/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/104/builds/8044/steps/11/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/103/builds/8038/steps/11/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/100/builds/6112/steps/12/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/64/builds/9159/steps/12/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/40/builds/9145/steps/11/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/81/builds/6733/steps/12/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/52/builds/8994/steps/12/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/108/builds/6122/steps/12/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/108/builds/6122/steps/12/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/58/builds/8923/steps/15/logs/stdio

https://autobuilder.yoctoproject.org/typhoon/#/builders/80/builds/6837/steps/14/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/79/builds/6881/steps/14/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/86/builds/6906/steps/14/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/87/builds/6892/steps/14/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/127/builds/3501/steps/15/logs/stdio

https://autobuilder.yoctoproject.org/typhoon/#/builders/76/builds/9099/steps/23/logs/stdio

Quoted 44 lines · Joshua Watt

This patch series add support for SPDX 3.0 and sets it as the default. Currently it is not possible to have SPDX 2.2 and SPDX 3.0 enabled at the same time; users may choose to specifically use SPDX 3.0 instead of 2.2 by doing INHERIT += "create-spdx-3.0"

v2: Added tests and addressed feedback

Joshua Watt (7):
classes-recipe/image: Add image file manifest classes/create-spdx-3.0: Add class classes-recipe/image_types: Add SPDX_IMAGE_PURPOSE to images
selftest: spdx: Add SPDX 3.0 test cases
classes-recipe: nospdx: Add class classes/spdx-common: Move SPDX_SUPPLIER
Switch default spdx version to 3.0

 meta/classes-recipe/image.bbclass            |   58 +
 meta/classes-recipe/image_types.bbclass      |    2 +
 meta/classes-recipe/image_types_wic.bbclass  |    1 +
 meta/classes-recipe/nospdx.bbclass           |   13 +
 meta/classes-recipe/packagegroup.bbclass     |    2 +
 meta/classes/create-spdx-2.2.bbclass         |    8 +
 meta/classes/create-spdx-3.0.bbclass         | 1116 ++++
 meta/classes/create-spdx-image-3.0.bbclass   |  203 +
 meta/classes/create-spdx.bbclass             |    2 +-
 meta/classes/spdx-common.bbclass             |   32 +-
 meta/lib/oe/sbom30.py                        | 1118 ++++
 meta/lib/oe/spdx30.py                        | 5996 ++++++++++++++++++
 meta/lib/oeqa/selftest/cases/spdx.py         |  119 +-
 meta/recipes-core/meta/build-sysroots.bb     |    5 +-
 meta/recipes-core/meta/meta-world-pkgdata.bb |    3 +-
 15 files changed, 8654 insertions(+), 24 deletions(-)
 create mode 100644 meta/classes-recipe/nospdx.bbclass
 create mode 100644 meta/classes/create-spdx-3.0.bbclass
 create mode 100644 m: failures:eta/classes/create-spdx-image-3.0.bbclass
 create mode 100644 meta/lib/oe/sbom30.py
 create mode 100644 meta/lib/oe/spdx30.py

--
2.43.2

Quoted 8 lines

-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#200929): https://lists.openembedded.org/g/openembedded-core/message/200929
Mute This Topic: https://lists.openembedded.org/mt/106770352/3617179
Group Owner: [email protected]
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [[email protected]]
-=-=-=-=-=-=-=-=-=-=-=-

--
Alexandre Belloni, co-owner and COO, Bootlin
Embedded Linux and Kernel engineering
https://bootlin.com

#29 lore raw

v3 cover Add SPDX 3.0 support

In reply to Joshua Watt

This patch series add support for SPDX 3.0 and sets it as the default. Currently it is not possible to have SPDX 2.2 and SPDX 3.0 enabled at the same time

v2: Added tests and addressed feedback
v3: Fixed several oe-selftest and build failures

Joshua Watt (10):
classes-recipe/image: Add image file manifest classes-recipe/baremetal-image: Add image file manifest classes/create-spdx-3.0: Add classes classes-global/staging: Exclude do_create_spdx from automatic sysroot
extension
binutils-cross-testsuite: Rename to binutils-testsuite classes-recipe/image_types: Add SPDX_IMAGE_PURPOSE to images
selftest: spdx: Add SPDX 3.0 test cases
classes-recipe: nospdx: Add class
classes/spdx-common: Move SPDX_SUPPLIER
Switch default spdx version to 3.0

 meta/classes-global/mirrors.bbclass           |    2 +-
 meta/classes-global/staging.bbclass           |    6 +-
 meta/classes-recipe/baremetal-image.bbclass   |   30 +-
 meta/classes-recipe/image.bbclass             |   58 +
 meta/classes-recipe/image_types.bbclass       |    2 +
 meta/classes-recipe/image_types_wic.bbclass   |    1 +
 meta/classes-recipe/nospdx.bbclass            |   13 +
 meta/classes-recipe/packagegroup.bbclass      |    2 +
 meta/classes/create-spdx-2.2.bbclass          |    8 +
 meta/classes/create-spdx-3.0.bbclass          | 1179 ++++
 meta/classes/create-spdx-image-3.0.bbclass    |  224 +
 meta/classes/create-spdx.bbclass              |    2 +-
 meta/classes/spdx-common.bbclass              |   13 +-
 meta/conf/distro/include/maintainers.inc      |    2 +-
 meta/lib/oe/sbom30.py                         | 1122 +++
 meta/lib/oe/spdx30.py                         | 5996 +++++++++++++++++
 meta/lib/oeqa/selftest/cases/binutils.py      |    2 +-
 meta/lib/oeqa/selftest/cases/spdx.py          |  119 +-
 meta/recipes-core/meta/build-sysroots.bb      |    5 +-
 meta/recipes-core/meta/meta-world-pkgdata.bb  |    3 +-
 ...ite_2.42.bb => binutils-testsuite_2.42.bb} |    3 +
 21 files changed, 8762 insertions(+), 30 deletions(-)
 create mode 100644 meta/classes-recipe/nospdx.bbclass
 create mode 100644 meta/classes/create-spdx-3.0.bbclass
 create mode 100644 meta/classes/create-spdx-image-3.0.bbclass
 create mode 100644 meta/lib/oe/sbom30.py
 create mode 100644 meta/lib/oe/spdx30.py
 rename meta/recipes-devtools/binutils/{binutils-cross-testsuite_2.42.bb => binutils-testsuite_2.42.bb} (95%)
Signature · 1 line
2.43.2
#30 lore raw

PATCH v3 1/10 classes-recipe/image: Add image file manifest

Downstream tasks may want to know what image files were written by the do_image family of tasks (e.g. SPDX) so have each task write out a manifest file that describes the files it produced, then aggregate them in do_image_complete

---
 meta/classes-recipe/image.bbclass | 58 +++++++++++++++++++++++++++++++
 1 file changed, 58 insertions(+)
meta/classes-recipe/image.bbclass +58 -0
index 28be6c63623..fc9b6116650 100644
@@ -88,6 +88,11 @@ PACKAGE_INSTALL_ATTEMPTONLY ?= "${FEATURE_INSTALL_OPTIONAL}"
 
 IMGDEPLOYDIR = "${WORKDIR}/deploy-${PN}-image-complete"
 
+IMGMANIFESTDIR = "${WORKDIR}/image-task-manifest"
+
+IMAGE_FILE_MANIFEST_DIR = "${WORKDIR}/deploy-image-file-manifest"
+IMAGE_FILE_MANIFEST = "${IMAGE_FILE_MANIFEST_DIR}/manifest.json"
+
 # Images are generally built explicitly, do not need to be part of world.
 EXCLUDE_FROM_WORLD = "1"
 
@@ -277,14 +282,28 @@ fakeroot python do_image () {
     execute_pre_post_process(d, pre_process_cmds)
 }
 do_image[dirs] = "${TOPDIR}"
+do_image[cleandirs] += "${IMGMANIFESTDIR}"
 addtask do_image after do_rootfs
 
 fakeroot python do_image_complete () {
     from oe.utils import execute_pre_post_process
+    from pathlib import Path
+    import json
 
     post_process_cmds = d.getVar("IMAGE_POSTPROCESS_COMMAND")
 
     execute_pre_post_process(d, post_process_cmds)
+
+    image_manifest_dir = Path(d.getVar('IMGMANIFESTDIR'))
+
+    data = []
+
+    for manifest_path in image_manifest_dir.glob("*.json"):
+        with manifest_path.open("r") as f:
+            data.extend(json.load(f))
+
+    with open(d.getVar("IMAGE_FILE_MANIFEST"), "w") as f:
+        json.dump(data, f)
 }
 do_image_complete[dirs] = "${TOPDIR}"
 SSTATETASKS += "do_image_complete"
@@ -292,6 +311,8 @@ SSTATE_SKIP_CREATION:task-image-complete = '1'
 do_image_complete[sstate-inputdirs] = "${IMGDEPLOYDIR}"
 do_image_complete[sstate-outputdirs] = "${DEPLOY_DIR_IMAGE}"
 do_image_complete[stamp-extra-info] = "${MACHINE_ARCH}"
+do_image_complete[sstate-plaindirs] += "${IMAGE_FILE_MANIFEST_DIR}"
+do_image_complete[dirs] += "${IMAGE_FILE_MANIFEST_DIR}"
 addtask do_image_complete after do_image before do_build
 python do_image_complete_setscene () {
     sstate_setscene(d)
@@ -507,12 +528,14 @@ python () {
         d.setVar(task, '\n'.join(cmds))
         d.setVarFlag(task, 'func', '1')
         d.setVarFlag(task, 'fakeroot', '1')
+        d.setVarFlag(task, 'imagetype', t)
 
         d.appendVarFlag(task, 'prefuncs', ' ' + debug + ' set_image_size')
         d.prependVarFlag(task, 'postfuncs', 'create_symlinks ')
         d.appendVarFlag(task, 'subimages', ' ' + ' '.join(subimages))
         d.appendVarFlag(task, 'vardeps', ' ' + ' '.join(vardeps))
         d.appendVarFlag(task, 'vardepsexclude', ' DATETIME DATE ' + ' '.join(vardepsexclude))
+        d.appendVarFlag(task, 'postfuncs', ' write_image_file_manifest')
 
         bb.debug(2, "Adding task %s before %s, after %s" % (task, 'do_image_complete', after))
         bb.build.addtask(task, 'do_image_complete', after, d)
@@ -610,6 +633,41 @@ python create_symlinks() {
             bb.note("Skipping symlink, source does not exist: %s -> %s" % (dst, src))
 }
 
+python write_image_file_manifest() {
+    import json
+    from pathlib import Path
+
+    taskname = d.getVar("BB_CURRENTTASK")
+    image_deploy_dir = Path(d.getVar('IMGDEPLOYDIR'))
+    image_manifest_dir = Path(d.getVar('IMGMANIFESTDIR'))
+    manifest_path = image_manifest_dir / ("do_" + d.getVar("BB_CURRENTTASK") + ".json")
+
+    image_name = d.getVar("IMAGE_NAME")
+    image_basename = d.getVar("IMAGE_BASENAME")
+    machine = d.getVar("MACHINE")
+
+    subimages = (d.getVarFlag("do_" + taskname, 'subimages', False) or "").split()
+    imagetype = d.getVarFlag("do_" + taskname, 'imagetype', False)
+
+    data = {
+        "taskname": taskname,
+        "imagetype": imagetype,
+        "images": []
+    }
+
+    for type in subimages:
+        image_filename = image_name + "." + type
+        image_path = image_deploy_dir / image_filename
+        if not image_path.exists():
+            continue
+        data["images"].append({
+            "filename": image_filename,
+        })
+
+    with manifest_path.open("w") as f:
+        json.dump([data], f)
+}
+
 MULTILIBRE_ALLOW_REP += "${base_bindir} ${base_sbindir} ${bindir} ${sbindir} ${libexecdir} ${sysconfdir} ${nonarch_base_libdir}/udev /lib/modules/[^/]*/modules.*"
 MULTILIB_CHECK_FILE = "${WORKDIR}/multilib_check.py"
 MULTILIB_TEMP_ROOTFS = "${WORKDIR}/multilib"
Signature · 1 line
2.43.2
#31 lore raw

PATCH v3 2/10 classes-recipe/baremetal-image: Add image file manifest

In reply to Joshua Watt

Downstream tasks may want to know what image files were written so write out a manifest in do_image_complete. The format of the manifest is the same as the one in image.bbclass

---
 meta/classes-recipe/baremetal-image.bbclass | 30 +++++++++++++++++++--
 1 file changed, 28 insertions(+), 2 deletions(-)
meta/classes-recipe/baremetal-image.bbclass +28 -2
index 4e7d413626e..d7f53803e2a 100644
@@ -30,6 +30,9 @@ BAREMETAL_BINNAME ?= "hello_baremetal_${MACHINE}"
 IMAGE_LINK_NAME ?= "baremetal-helloworld-image-${MACHINE}"
 IMAGE_NAME_SUFFIX ?= ""
 
+IMAGE_FILE_MANIFEST_DIR = "${WORKDIR}/deploy-image-file-manifest"
+IMAGE_FILE_MANIFEST = "${IMAGE_FILE_MANIFEST_DIR}/manifest.json"
+
 do_rootfs[dirs] = "${IMGDEPLOYDIR} ${DEPLOY_DIR_IMAGE}"
 
 do_image(){
@@ -37,8 +40,28 @@ do_image(){
     install ${D}/${base_libdir}/firmware/${BAREMETAL_BINNAME}.elf ${IMGDEPLOYDIR}/${IMAGE_LINK_NAME}.elf
 }
 
-do_image_complete(){
-    :
+python do_image_complete(){
+    from pathlib import Path
+    import json
+
+    data = {
+        "taskname": "do_image",
+        "imagetype": "baremetal-image",
+        "images": []
+    }
+
+    img_deploy_dir = Path(d.getVar("IMGDEPLOYDIR"))
+
+    for child in img_deploy_dir.iterdir():
+        if not child.is_file() or child.is_symlink():
+            continue
+
+        data["images"].append({
+            "filename": child.name,
+        })
+
+    with open(d.getVar("IMAGE_FILE_MANIFEST"), "w") as f:
+        json.dump([data], f)
 }
 
 python do_rootfs(){
@@ -62,6 +85,7 @@ python do_rootfs(){
     bb.utils.mkdirhier(sysconfdir)
 
     execute_pre_post_process(d, d.getVar('ROOTFS_POSTPROCESS_COMMAND'))
+    execute_pre_post_process(d, d.getVar("ROOTFS_POSTUNINSTALL_COMMAND"))
 }
 
 
@@ -72,6 +96,8 @@ SSTATE_SKIP_CREATION:task-image-complete = '1'
 do_image_complete[sstate-inputdirs] = "${IMGDEPLOYDIR}"
 do_image_complete[sstate-outputdirs] = "${DEPLOY_DIR_IMAGE}"
 do_image_complete[stamp-extra-info] = "${MACHINE_ARCH}"
+do_image_complete[sstate-plaindirs] += "${IMAGE_FILE_MANIFEST_DIR}"
+do_image_complete[dirs] += "${IMAGE_FILE_MANIFEST_DIR}"
 addtask do_image_complete after do_image before do_build
 
 python do_image_complete_setscene () {
Signature · 1 line
2.43.2
#32 lore raw

PATCH v3 3/10 classes/create-spdx-3.0: Add classes

In reply to Joshua Watt

Adds a class to generate SPDX 3.0 output and an image class that is used when generating images

---
 meta/classes-recipe/packagegroup.bbclass   |    2 +
 meta/classes/create-spdx-3.0.bbclass       | 1179 ++++
 meta/classes/create-spdx-image-3.0.bbclass |  224 +
 meta/classes/spdx-common.bbclass           |    6 +-
 meta/lib/oe/sbom30.py                      | 1122 ++++
 meta/lib/oe/spdx30.py                      | 5996 ++++++++++++++++++++
 6 files changed, 8528 insertions(+), 1 deletion(-)
 create mode 100644 meta/classes/create-spdx-3.0.bbclass
 create mode 100644 meta/classes/create-spdx-image-3.0.bbclass
 create mode 100644 meta/lib/oe/sbom30.py
 create mode 100644 meta/lib/oe/spdx30.py
meta/classes-recipe/packagegroup.bbclass +2 -0
index cf6fc354a81..2099c8b37d6 100644
@@ -56,6 +56,8 @@ deltask do_populate_sysroot
 
 do_create_runtime_spdx[deptask] = "do_create_spdx"
 do_create_runtime_spdx[rdeptask] = ""
+do_create_package_spdx[deptask] = "do_create_spdx"
+do_create_package_spdx[rdeptask] = ""
 
 INHIBIT_DEFAULT_DEPS = "1"
 
meta/classes/create-spdx-3.0.bbclass new file +1179 -0

1,179 lines not on this page · raw message

meta/classes/create-spdx-image-3.0.bbclass new file +224 -0

224 lines not on this page · raw message

meta/classes/spdx-common.bbclass +5 -1
index 03f1d0cc278..9dee325ba6b 100644
@@ -17,6 +17,7 @@ SPDXDEPLOY = "${SPDXDIR}/deploy"
 SPDXWORK = "${SPDXDIR}/work"
 SPDXIMAGEWORK = "${SPDXDIR}/image-work"
 SPDXSDKWORK = "${SPDXDIR}/sdk-work"
+SPDXSDKEXTWORK = "${SPDXDIR}/sdk-ext-work"
 SPDXDEPS = "${SPDXDIR}/deps.json"
 
 SPDX_TOOL_NAME ??= "oe-spdx-creator"
@@ -68,7 +69,7 @@ def get_json_indent(d):
         return 2
     return None
 
-python() {
+def load_spdx_license_data(d):
     import json
     if d.getVar("SPDX_LICENSE_DATA"):
         return
@@ -78,6 +79,9 @@ python() {
         # Transform the license array to a dictionary
         data["licenses"] = {l["licenseId"]: l for l in data["licenses"]}
         d.setVar("SPDX_LICENSE_DATA", data)
+
+python() {
+    load_spdx_license_data(d)
 }
 
 def process_sources(d):
meta/lib/oe/sbom30.py new file +1122 -0

1,122 lines not on this page · raw message

meta/lib/oe/spdx30.py new file +5996 -0

443 lines not on this page · raw message

The rest of this diff is not shown. View the raw message.

Signature · 1 line
2.43.2

Part of the diff is not shown here. View the raw message.

#33 lore raw

PATCH v3 4/10 classes-global/staging: Exclude do_create_spdx from automatic sysroot extension

In reply to Joshua Watt

do_create_spdx is a outlier in that it doesn't need the RSS to be extended just because it depends on do_populate_sysroot. In fact, it only depends on do_populate_sysroot so it can see the actual recipes sysroot, and attempting to extend the sysroot can cause problems for some recipes (e.g. if a recipe does do_populate_sysroot[noexec] = "1")

As such, explicitly exclude do_create_spdx from extending the sysroot just because it depends on do_populate_sysroot.

---
 meta/classes-global/staging.bbclass | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)
meta/classes-global/staging.bbclass +5 -1
index 3678a1b4415..ecce68d532e 100644
@@ -652,10 +652,14 @@ python do_prepare_recipe_sysroot () {
 addtask do_prepare_recipe_sysroot before do_configure after do_fetch
 
 python staging_taskhandler() {
+    EXCLUDED_TASK = (
+        "do_prepare_recipe_sysroot",
+        "do_create_spdx",
+    )
     bbtasks = e.tasklist
     for task in bbtasks:
         deps = d.getVarFlag(task, "depends")
-        if task != 'do_prepare_recipe_sysroot' and (task == "do_configure" or (deps and "populate_sysroot" in deps)):
+        if task not in EXCLUDED_TASKS and (task == "do_configure" or (deps and "populate_sysroot" in deps)):
             d.prependVarFlag(task, "prefuncs", "extend_recipe_sysroot ")
 }
 staging_taskhandler[eventmask] = "bb.event.RecipeTaskPreProcess"
Signature · 1 line
2.43.2
#34 lore raw

PATCH v3 5/10 binutils-cross-testsuite: Rename to binutils-testsuite

In reply to Joshua Watt

This recipe needs to be renamed because the "-cross-" substring in the name triggers the cross architecture detection in sstate, but this recipe is not actually a cross recipe.

---
 meta/classes-global/mirrors.bbclass                            | 2 +-
 meta/conf/distro/include/maintainers.inc                       | 2 +-
 meta/lib/oeqa/selftest/cases/binutils.py                       | 2 +-
 ...tils-cross-testsuite_2.42.bb => binutils-testsuite_2.42.bb} | 3 +++
 4 files changed, 6 insertions(+), 3 deletions(-)
 rename meta/recipes-devtools/binutils/{binutils-cross-testsuite_2.42.bb => binutils-testsuite_2.42.bb} (95%)
meta/classes-global/mirrors.bbclass +1 -1
index 862648eec5f..d68d30b0f2a 100644
@@ -88,7 +88,7 @@ git://.*/.*                   git://HOST/git/PATH;protocol=https \
 BB_GIT_SHALLOW:pn-binutils = "1"
 BB_GIT_SHALLOW:pn-binutils-cross-${TARGET_ARCH} = "1"
 BB_GIT_SHALLOW:pn-binutils-cross-canadian-${TRANSLATED_TARGET_ARCH} = "1"
-BB_GIT_SHALLOW:pn-binutils-cross-testsuite = "1"
+BB_GIT_SHALLOW:pn-binutils-testsuite = "1"
 BB_GIT_SHALLOW:pn-binutils-crosssdk-${SDK_SYS} = "1"
 BB_GIT_SHALLOW:pn-binutils-native = "1"
 BB_GIT_SHALLOW:pn-nativesdk-binutils = "1"
meta/conf/distro/include/maintainers.inc +1 -1
index 14e0637c838..368d3de1583 100644
@@ -64,7 +64,7 @@ RECIPE_MAINTAINER:pn-bind = "Unassigned <[email protected]>"
 RECIPE_MAINTAINER:pn-binutils = "Khem Raj <[email protected]>"
 RECIPE_MAINTAINER:pn-binutils-cross-${TARGET_ARCH} = "Khem Raj <[email protected]>"
 RECIPE_MAINTAINER:pn-binutils-cross-canadian-${TRANSLATED_TARGET_ARCH} = "Khem Raj <[email protected]>"
-RECIPE_MAINTAINER:pn-binutils-cross-testsuite = "Khem Raj <[email protected]>"
+RECIPE_MAINTAINER:pn-binutils-testsuite = "Khem Raj <[email protected]>"
 RECIPE_MAINTAINER:pn-binutils-crosssdk-${SDK_SYS} = "Khem Raj <[email protected]>"
 RECIPE_MAINTAINER:pn-bison = "Chen Qi <[email protected]>"
 RECIPE_MAINTAINER:pn-blktool = "Yi Zhao <[email protected]>"
meta/lib/oeqa/selftest/cases/binutils.py +1 -1
index 1688eabe4ec..5ff263d342a 100644
@@ -33,7 +33,7 @@ class BinutilsCrossSelfTest(OESelftestTestCase, OEPTestResultTestCase):
         features.append('CHECK_TARGETS = "{0}"'.format(suite))
         self.write_config("\n".join(features))
 
-        recipe = "binutils-cross-testsuite"
+        recipe = "binutils-testsuite"
         bb_vars = get_bb_vars(["B", "TARGET_SYS", "T"], recipe)
         builddir, target_sys, tdir = bb_vars["B"], bb_vars["TARGET_SYS"], bb_vars["T"]
 
meta/recipes-devtools/binutils/binutils-testsuite_2.42.bb renamed from meta/recipes-devtools/binutils/binutils-cross-testsuite_2.42.bb +3 -0
similarity index 95%
rename from meta/recipes-devtools/binutils/binutils-cross-testsuite_2.42.bb
rename to meta/recipes-devtools/binutils/binutils-testsuite_2.42.bb
index 630815c7a3e..f2facd52c3a 100644
@@ -1,3 +1,6 @@
+# NOTE: This recipe cannot have -cross- in the file name because it triggers
+# the cross build detection in sstate which causes it to use the wrong
+# architecture
 require binutils.inc
 require binutils-${PV}.inc
 
Signature · 1 line
2.43.2
#35 lore raw

PATCH v3 6/10 classes-recipe/image_types: Add SPDX_IMAGE_PURPOSE to images

In reply to Joshua Watt

Adds the variable overrides to set the SPDX image purpose for various image types

---
 meta/classes-recipe/image_types.bbclass     | 2 ++
 meta/classes-recipe/image_types_wic.bbclass | 1 +
 2 files changed, 3 insertions(+)
meta/classes-recipe/image_types.bbclass +2 -0
index 2f948ecbf88..506b9934cb7 100644
@@ -146,6 +146,7 @@ IMAGE_CMD:vfat = "oe_mkvfatfs ${EXTRA_IMAGECMD}"
 IMAGE_CMD_TAR ?= "tar"
 # ignore return code 1 "file changed as we read it" as other tasks(e.g. do_image_wic) may be hardlinking rootfs
 IMAGE_CMD:tar = "${IMAGE_CMD_TAR} --sort=name --format=posix --numeric-owner -cf ${IMGDEPLOYDIR}/${IMAGE_NAME}.tar -C ${IMAGE_ROOTFS} . || [ $? -eq 1 ]"
+SPDX_IMAGE_PURPOSE:tar = "archive"
 
 do_image_cpio[cleandirs] += "${WORKDIR}/cpio_append"
 IMAGE_CMD:cpio () {
@@ -167,6 +168,7 @@ IMAGE_CMD:cpio () {
 		fi
 	fi
 }
+SPDX_IMAGE_PURPOSE:cpio = "archive"
 
 UBI_VOLNAME ?= "${MACHINE}-rootfs"
 UBI_VOLTYPE ?= "dynamic"
meta/classes-recipe/image_types_wic.bbclass +1 -0
index cf3be909b30..86f40633ebc 100644
@@ -91,6 +91,7 @@ IMAGE_CMD:wic () {
 	mv "$build_wic/$(basename "${wks%.wks}")"*.${IMAGER} "$out.wic"
 }
 IMAGE_CMD:wic[vardepsexclude] = "WKS_FULL_PATH WKS_FILES TOPDIR"
+SPDX_IMAGE_PURPOSE:wic = "diskImage"
 do_image_wic[cleandirs] = "${WORKDIR}/build-wic"
 
 PSEUDO_IGNORE_PATHS .= ",${WORKDIR}/build-wic"
Signature · 1 line
2.43.2
#36 lore raw

PATCH v3 7/10 selftest: spdx: Add SPDX 3.0 test cases

In reply to Joshua Watt

Adds test cases for SPDX 3.0. Reworks the SPDX 2.2 test setup so it can also be run even if the default is SPDX 3.0

---
 meta/lib/oeqa/selftest/cases/spdx.py | 119 +++++++++++++++++++++++++--
 1 file changed, 110 insertions(+), 9 deletions(-)
meta/lib/oeqa/selftest/cases/spdx.py +110 -9
index 7685a81e7fb..0ae49aec333 100644
@@ -6,21 +6,26 @@
 
 import json
 import os
+import textwrap
+from pathlib import Path
 from oeqa.selftest.case import OESelftestTestCase
-from oeqa.utils.commands import bitbake, get_bb_var, runCmd
+from oeqa.utils.commands import bitbake, get_bb_var, get_bb_vars, runCmd
 
-class SPDXCheck(OESelftestTestCase):
 
+class SPDX22Check(OESelftestTestCase):
     @classmethod
     def setUpClass(cls):
-        super(SPDXCheck, cls).setUpClass()
+        super().setUpClass()
         bitbake("python3-spdx-tools-native")
         bitbake("-c addto_recipe_sysroot python3-spdx-tools-native")
 
     def check_recipe_spdx(self, high_level_dir, spdx_file, target_name):
-        config = """
-INHERIT += "create-spdx"
-"""
+        config = textwrap.dedent(
+            """\
+            INHERIT:remove = "create-spdx"
+            INHERIT += "create-spdx-2.2"
+            """
+        )
         self.write_config(config)
 
         deploy_dir = get_bb_var("DEPLOY_DIR")
@@ -29,7 +34,9 @@ INHERIT += "create-spdx"
         # qemux86-64 creates the directory qemux86_64
         machine_dir = machine_var.replace("-", "_")
 
-        full_file_path = os.path.join(deploy_dir, "spdx", spdx_version, machine_dir, high_level_dir, spdx_file)
+        full_file_path = os.path.join(
+            deploy_dir, "spdx", spdx_version, machine_dir, high_level_dir, spdx_file
+        )
 
         try:
             os.remove(full_file_path)
@@ -44,8 +51,13 @@ INHERIT += "create-spdx"
                 self.assertNotEqual(report, None)
                 self.assertNotEqual(report["SPDXID"], None)
 
-            python = os.path.join(get_bb_var('STAGING_BINDIR', 'python3-spdx-tools-native'), 'nativepython3')
-            validator = os.path.join(get_bb_var('STAGING_BINDIR', 'python3-spdx-tools-native'), 'pyspdxtools')
+            python = os.path.join(
+                get_bb_var("STAGING_BINDIR", "python3-spdx-tools-native"),
+                "nativepython3",
+            )
+            validator = os.path.join(
+                get_bb_var("STAGING_BINDIR", "python3-spdx-tools-native"), "pyspdxtools"
+            )
             result = runCmd("{} {} -i {}".format(python, validator, filename))
 
         self.assertExists(full_file_path)
@@ -53,3 +65,92 @@ INHERIT += "create-spdx"
 
     def test_spdx_base_files(self):
         self.check_recipe_spdx("packages", "base-files.spdx.json", "base-files")
+
+
+class SPDX3CheckBase(object):
+    """
+    Base class for checking SPDX 3 based tests
+    """
+
+    def check_spdx_file(self, filename):
+        import oe.spdx30
+
+        self.assertExists(filename)
+
+        # Read the file
+        objset = oe.spdx30.SHACLObjectSet()
+        with open(filename, "r") as f:
+            d = oe.spdx30.JSONLDDeserializer()
+            d.read(f, objset)
+
+        return objset
+
+    def check_recipe_spdx(self, target_name, spdx_path, *, task=None):
+        config = textwrap.dedent(
+            f"""\
+            INHERIT:remove = "create-spdx"
+            INHERIT += "{self.SPDX_CLASS}"
+            """
+        )
+        self.write_config(config)
+
+        if task:
+            bitbake(f"-c {task} {target_name}")
+        else:
+            bitbake(target_name)
+
+        filename = spdx_path.format(
+            **get_bb_vars(
+                [
+                    "DEPLOY_DIR_IMAGE",
+                    "DEPLOY_DIR_SPDX",
+                    "MACHINE",
+                    "MACHINE_ARCH",
+                    "SDKMACHINE",
+                    "SDK_DEPLOY",
+                    "SPDX_VERSION",
+                    "TOOLCHAIN_OUTPUTNAME",
+                ],
+                target_name,
+            )
+        )
+
+        return self.check_spdx_file(filename)
+
+    def check_objset_missing_ids(self, objset):
+        if objset.missing_ids:
+            self.assertTrue(
+                False,
+                "The following SPDXIDs are unresolved:\n  "
+                + "\n  ".join(objset.missing_ids),
+            )
+
+
+class SPDX30Check(SPDX3CheckBase, OESelftestTestCase):
+    SPDX_CLASS = "create-spdx-3.0"
+    SCHEMA_URL = "https://spdx.org/schema/3.0.0/spdx-json-schema.json"
+
+    def test_base_files(self):
+        self.check_recipe_spdx(
+            "base-files",
+            "{DEPLOY_DIR_SPDX}/{MACHINE_ARCH}/packages/base-files.spdx.json",
+        )
+
+    def test_core_image_minimal(self):
+        objset = self.check_recipe_spdx(
+            "core-image-minimal",
+            "{DEPLOY_DIR_IMAGE}/core-image-minimal-{MACHINE}.rootfs.spdx.json",
+        )
+
+        # Document should be fully linked
+        self.check_objset_missing_ids(objset)
+
+    def test_core_image_minimal_sdk(self):
+        objset = self.check_recipe_spdx(
+            "core-image-minimal",
+            "{SDK_DEPLOY}/{TOOLCHAIN_OUTPUTNAME}.spdx.json",
+            task="populate_sdk",
+        )
+
+        # Document should be fully linked
+        self.check_objset_missing_ids(objset)
Signature · 1 line
2.43.2
#37 lore raw

PATCH v3 8/10 classes-recipe: nospdx: Add class

In reply to Joshua Watt

Adds a class that allows recipes to opt out of generating SPDX

---
 meta/classes-recipe/nospdx.bbclass           | 13 +++++++++++++
 meta/recipes-core/meta/build-sysroots.bb     |  5 +----
 meta/recipes-core/meta/meta-world-pkgdata.bb |  3 +--
 3 files changed, 15 insertions(+), 6 deletions(-)
 create mode 100644 meta/classes-recipe/nospdx.bbclass
meta/classes-recipe/nospdx.bbclass new file +13 -0
new file mode 100644
index 00000000000..b20e28218be
@@ -0,0 +1,13 @@
+#
+# Copyright OpenEmbedded Contributors
+#
+# SPDX-License-Identifier: MIT
+#
+
+deltask do_collect_spdx_deps
+deltask do_create_spdx
+deltask do_create_spdx_runtime
+deltask do_create_package_spdx
+deltask do_create_rootfs_spdx
+deltask do_create_image_spdx
+deltask do_create_image_sbom
meta/recipes-core/meta/build-sysroots.bb +1 -4
index db05c111ab2..b0b8fb3c79a 100644
@@ -7,7 +7,7 @@ STANDALONE_SYSROOT_NATIVE = "${STAGING_DIR}/${BUILD_ARCH}"
 PACKAGE_ARCH = "${MACHINE_ARCH}"
 EXCLUDE_FROM_WORLD = "1"
 
-inherit nopackages
+inherit nopackages nospdx
 deltask fetch
 deltask unpack
 deltask patch
@@ -17,9 +17,6 @@ deltask configure
 deltask compile
 deltask install
 deltask populate_sysroot
-deltask create_spdx
-deltask collect_spdx_deps
-deltask create_runtime_spdx
 deltask recipe_qa
 
 do_build_warn () {
meta/recipes-core/meta/meta-world-pkgdata.bb +1 -2
index 0438bf61387..244175ddd44 100644
@@ -27,14 +27,13 @@ python do_collect_packagedata() {
     oe.copy_buildsystem.generate_locked_sigs(sigfile, d)
 }
 
+inherit nospdx
 deltask do_fetch
 deltask do_unpack
 deltask do_patch
 deltask do_configure
 deltask do_compile
 deltask do_install
-deltask do_create_spdx
-deltask do_create_spdx_runtime
 
 do_prepare_recipe_sysroot[deptask] = ""
 
Signature · 1 line
2.43.2
#38 lore raw

PATCH v3 9/10 classes/spdx-common: Move SPDX_SUPPLIER

In reply to Joshua Watt

Move the SPDX_SUPPLIER variable to create-spdx-2.2 since it's format only has meaning in SPDX 2.2 (SPDX 3 uses SPDX_PACKAGE_SUPPLIER with a different format)

---
 meta/classes/create-spdx-2.2.bbclass | 8 ++++++++
 meta/classes/spdx-common.bbclass     | 7 -------
 2 files changed, 8 insertions(+), 7 deletions(-)
meta/classes/create-spdx-2.2.bbclass +8 -0
index 3ebf92b5e12..99061320e53 100644
@@ -8,6 +8,14 @@ inherit spdx-common
 
 SPDX_VERSION = "2.2"
 
+SPDX_ORG ??= "OpenEmbedded ()"
+SPDX_SUPPLIER ??= "Organization: ${SPDX_ORG}"
+SPDX_SUPPLIER[doc] = "The SPDX PackageSupplier field for SPDX packages created from \
+    this recipe. For SPDX documents create using this class during the build, this \
+    is the contact information for the person or organization who is doing the \
+    build."
+
+
 def get_namespace(d, name):
     import uuid
     namespace_uuid = uuid.uuid5(uuid.NAMESPACE_DNS, d.getVar("SPDX_UUID_NAMESPACE"))
meta/classes/spdx-common.bbclass +0 -7
index 9dee325ba6b..6dfc1fd9e4c 100644
@@ -37,13 +37,6 @@ SPDX_LICENSES ??= "${COREBASE}/meta/files/spdx-licenses.json"
 
 SPDX_CUSTOM_ANNOTATION_VARS ??= ""
 
-SPDX_ORG ??= "OpenEmbedded ()"
-SPDX_SUPPLIER ??= "Organization: ${SPDX_ORG}"
-SPDX_SUPPLIER[doc] = "The SPDX PackageSupplier field for SPDX packages created from \
-    this recipe. For SPDX documents create using this class during the build, this \
-    is the contact information for the person or organization who is doing the \
-    build."
-
 def extract_licenses(filename):
     import re
 
Signature · 1 line
2.43.2
#39 lore raw

PATCH v3 10/10 Switch default spdx version to 3.0

In reply to Joshua Watt

Changes the default SPDX version to 3.0

---
 meta/classes/create-spdx.bbclass | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
meta/classes/create-spdx.bbclass +1 -1
index 19c6c0ff0b9..b604973ae0a 100644
@@ -5,4 +5,4 @@
 #
 # Include this class when you don't care what version of SPDX you get; it will
 # be updated to the latest stable version that is supported
-inherit create-spdx-2.2
+inherit create-spdx-3.0
Signature · 1 line
2.43.2
#40 lore raw

v4 cover Add SPDX 3.0 support

In reply to Joshua Watt

This patch series add support for SPDX 3.0 and sets it as the default. Currently it is not possible to have SPDX 2.2 and SPDX 3.0 enabled at the same time

v2: Added tests and addressed feedback
v3: Fixed several oe-selftest and build failures
v4: Fixed silly typo mistake in staging.bbclass

Joshua Watt (10):
classes-recipe/image: Add image file manifest classes-recipe/baremetal-image: Add image file manifest classes/create-spdx-3.0: Add classes classes-global/staging: Exclude do_create_spdx from automatic sysroot
extension
binutils-cross-testsuite: Rename to binutils-testsuite classes-recipe/image_types: Add SPDX_IMAGE_PURPOSE to images
selftest: spdx: Add SPDX 3.0 test cases
classes-recipe: nospdx: Add class
classes/spdx-common: Move SPDX_SUPPLIER
Switch default spdx version to 3.0

 meta/classes-global/mirrors.bbclass           |    2 +-
 meta/classes-global/staging.bbclass           |    9 +-
 meta/classes-recipe/baremetal-image.bbclass   |   30 +-
 meta/classes-recipe/image.bbclass             |   58 +
 meta/classes-recipe/image_types.bbclass       |    2 +
 meta/classes-recipe/image_types_wic.bbclass   |    1 +
 meta/classes-recipe/nospdx.bbclass            |   13 +
 meta/classes-recipe/packagegroup.bbclass      |    2 +
 meta/classes/create-spdx-2.2.bbclass          |    8 +
 meta/classes/create-spdx-3.0.bbclass          | 1179 ++++
 meta/classes/create-spdx-image-3.0.bbclass    |  224 +
 meta/classes/create-spdx.bbclass              |    2 +-
 meta/classes/spdx-common.bbclass              |   13 +-
 meta/conf/distro/include/maintainers.inc      |    2 +-
 meta/lib/oe/sbom30.py                         | 1122 +++
 meta/lib/oe/spdx30.py                         | 5996 +++++++++++++++++
 meta/lib/oeqa/selftest/cases/binutils.py      |    2 +-
 meta/lib/oeqa/selftest/cases/spdx.py          |  119 +-
 meta/recipes-core/meta/build-sysroots.bb      |    5 +-
 meta/recipes-core/meta/meta-world-pkgdata.bb  |    3 +-
 ...ite_2.42.bb => binutils-testsuite_2.42.bb} |    3 +
 21 files changed, 8765 insertions(+), 30 deletions(-)
 create mode 100644 meta/classes-recipe/nospdx.bbclass
 create mode 100644 meta/classes/create-spdx-3.0.bbclass
 create mode 100644 meta/classes/create-spdx-image-3.0.bbclass
 create mode 100644 meta/lib/oe/sbom30.py
 create mode 100644 meta/lib/oe/spdx30.py
 rename meta/recipes-devtools/binutils/{binutils-cross-testsuite_2.42.bb => binutils-testsuite_2.42.bb} (95%)
Signature · 1 line
2.43.2
#41 lore raw

PATCH v4 1/10 classes-recipe/image: Add image file manifest

Downstream tasks may want to know what image files were written by the do_image family of tasks (e.g. SPDX) so have each task write out a manifest file that describes the files it produced, then aggregate them in do_image_complete

---
 meta/classes-recipe/image.bbclass | 58 +++++++++++++++++++++++++++++++
 1 file changed, 58 insertions(+)
meta/classes-recipe/image.bbclass +58 -0
index 28be6c63623..fc9b6116650 100644
@@ -88,6 +88,11 @@ PACKAGE_INSTALL_ATTEMPTONLY ?= "${FEATURE_INSTALL_OPTIONAL}"
 
 IMGDEPLOYDIR = "${WORKDIR}/deploy-${PN}-image-complete"
 
+IMGMANIFESTDIR = "${WORKDIR}/image-task-manifest"
+
+IMAGE_FILE_MANIFEST_DIR = "${WORKDIR}/deploy-image-file-manifest"
+IMAGE_FILE_MANIFEST = "${IMAGE_FILE_MANIFEST_DIR}/manifest.json"
+
 # Images are generally built explicitly, do not need to be part of world.
 EXCLUDE_FROM_WORLD = "1"
 
@@ -277,14 +282,28 @@ fakeroot python do_image () {
     execute_pre_post_process(d, pre_process_cmds)
 }
 do_image[dirs] = "${TOPDIR}"
+do_image[cleandirs] += "${IMGMANIFESTDIR}"
 addtask do_image after do_rootfs
 
 fakeroot python do_image_complete () {
     from oe.utils import execute_pre_post_process
+    from pathlib import Path
+    import json
 
     post_process_cmds = d.getVar("IMAGE_POSTPROCESS_COMMAND")
 
     execute_pre_post_process(d, post_process_cmds)
+
+    image_manifest_dir = Path(d.getVar('IMGMANIFESTDIR'))
+
+    data = []
+
+    for manifest_path in image_manifest_dir.glob("*.json"):
+        with manifest_path.open("r") as f:
+            data.extend(json.load(f))
+
+    with open(d.getVar("IMAGE_FILE_MANIFEST"), "w") as f:
+        json.dump(data, f)
 }
 do_image_complete[dirs] = "${TOPDIR}"
 SSTATETASKS += "do_image_complete"
@@ -292,6 +311,8 @@ SSTATE_SKIP_CREATION:task-image-complete = '1'
 do_image_complete[sstate-inputdirs] = "${IMGDEPLOYDIR}"
 do_image_complete[sstate-outputdirs] = "${DEPLOY_DIR_IMAGE}"
 do_image_complete[stamp-extra-info] = "${MACHINE_ARCH}"
+do_image_complete[sstate-plaindirs] += "${IMAGE_FILE_MANIFEST_DIR}"
+do_image_complete[dirs] += "${IMAGE_FILE_MANIFEST_DIR}"
 addtask do_image_complete after do_image before do_build
 python do_image_complete_setscene () {
     sstate_setscene(d)
@@ -507,12 +528,14 @@ python () {
         d.setVar(task, '\n'.join(cmds))
         d.setVarFlag(task, 'func', '1')
         d.setVarFlag(task, 'fakeroot', '1')
+        d.setVarFlag(task, 'imagetype', t)
 
         d.appendVarFlag(task, 'prefuncs', ' ' + debug + ' set_image_size')
         d.prependVarFlag(task, 'postfuncs', 'create_symlinks ')
         d.appendVarFlag(task, 'subimages', ' ' + ' '.join(subimages))
         d.appendVarFlag(task, 'vardeps', ' ' + ' '.join(vardeps))
         d.appendVarFlag(task, 'vardepsexclude', ' DATETIME DATE ' + ' '.join(vardepsexclude))
+        d.appendVarFlag(task, 'postfuncs', ' write_image_file_manifest')
 
         bb.debug(2, "Adding task %s before %s, after %s" % (task, 'do_image_complete', after))
         bb.build.addtask(task, 'do_image_complete', after, d)
@@ -610,6 +633,41 @@ python create_symlinks() {
             bb.note("Skipping symlink, source does not exist: %s -> %s" % (dst, src))
 }
 
+python write_image_file_manifest() {
+    import json
+    from pathlib import Path
+
+    taskname = d.getVar("BB_CURRENTTASK")
+    image_deploy_dir = Path(d.getVar('IMGDEPLOYDIR'))
+    image_manifest_dir = Path(d.getVar('IMGMANIFESTDIR'))
+    manifest_path = image_manifest_dir / ("do_" + d.getVar("BB_CURRENTTASK") + ".json")
+
+    image_name = d.getVar("IMAGE_NAME")
+    image_basename = d.getVar("IMAGE_BASENAME")
+    machine = d.getVar("MACHINE")
+
+    subimages = (d.getVarFlag("do_" + taskname, 'subimages', False) or "").split()
+    imagetype = d.getVarFlag("do_" + taskname, 'imagetype', False)
+
+    data = {
+        "taskname": taskname,
+        "imagetype": imagetype,
+        "images": []
+    }
+
+    for type in subimages:
+        image_filename = image_name + "." + type
+        image_path = image_deploy_dir / image_filename
+        if not image_path.exists():
+            continue
+        data["images"].append({
+            "filename": image_filename,
+        })
+
+    with manifest_path.open("w") as f:
+        json.dump([data], f)
+}
+
 MULTILIBRE_ALLOW_REP += "${base_bindir} ${base_sbindir} ${bindir} ${sbindir} ${libexecdir} ${sysconfdir} ${nonarch_base_libdir}/udev /lib/modules/[^/]*/modules.*"
 MULTILIB_CHECK_FILE = "${WORKDIR}/multilib_check.py"
 MULTILIB_TEMP_ROOTFS = "${WORKDIR}/multilib"
Signature · 1 line
2.43.2
#42 lore raw

PATCH v4 2/10 classes-recipe/baremetal-image: Add image file manifest

In reply to Joshua Watt

Downstream tasks may want to know what image files were written so write out a manifest in do_image_complete. The format of the manifest is the same as the one in image.bbclass

---
 meta/classes-recipe/baremetal-image.bbclass | 30 +++++++++++++++++++--
 1 file changed, 28 insertions(+), 2 deletions(-)
meta/classes-recipe/baremetal-image.bbclass +28 -2
index 4e7d413626e..d7f53803e2a 100644
@@ -30,6 +30,9 @@ BAREMETAL_BINNAME ?= "hello_baremetal_${MACHINE}"
 IMAGE_LINK_NAME ?= "baremetal-helloworld-image-${MACHINE}"
 IMAGE_NAME_SUFFIX ?= ""
 
+IMAGE_FILE_MANIFEST_DIR = "${WORKDIR}/deploy-image-file-manifest"
+IMAGE_FILE_MANIFEST = "${IMAGE_FILE_MANIFEST_DIR}/manifest.json"
+
 do_rootfs[dirs] = "${IMGDEPLOYDIR} ${DEPLOY_DIR_IMAGE}"
 
 do_image(){
@@ -37,8 +40,28 @@ do_image(){
     install ${D}/${base_libdir}/firmware/${BAREMETAL_BINNAME}.elf ${IMGDEPLOYDIR}/${IMAGE_LINK_NAME}.elf
 }
 
-do_image_complete(){
-    :
+python do_image_complete(){
+    from pathlib import Path
+    import json
+
+    data = {
+        "taskname": "do_image",
+        "imagetype": "baremetal-image",
+        "images": []
+    }
+
+    img_deploy_dir = Path(d.getVar("IMGDEPLOYDIR"))
+
+    for child in img_deploy_dir.iterdir():
+        if not child.is_file() or child.is_symlink():
+            continue
+
+        data["images"].append({
+            "filename": child.name,
+        })
+
+    with open(d.getVar("IMAGE_FILE_MANIFEST"), "w") as f:
+        json.dump([data], f)
 }
 
 python do_rootfs(){
@@ -62,6 +85,7 @@ python do_rootfs(){
     bb.utils.mkdirhier(sysconfdir)
 
     execute_pre_post_process(d, d.getVar('ROOTFS_POSTPROCESS_COMMAND'))
+    execute_pre_post_process(d, d.getVar("ROOTFS_POSTUNINSTALL_COMMAND"))
 }
 
 
@@ -72,6 +96,8 @@ SSTATE_SKIP_CREATION:task-image-complete = '1'
 do_image_complete[sstate-inputdirs] = "${IMGDEPLOYDIR}"
 do_image_complete[sstate-outputdirs] = "${DEPLOY_DIR_IMAGE}"
 do_image_complete[stamp-extra-info] = "${MACHINE_ARCH}"
+do_image_complete[sstate-plaindirs] += "${IMAGE_FILE_MANIFEST_DIR}"
+do_image_complete[dirs] += "${IMAGE_FILE_MANIFEST_DIR}"
 addtask do_image_complete after do_image before do_build
 
 python do_image_complete_setscene () {
Signature · 1 line
2.43.2
#43 lore raw

Re: [OE-core][PATCH v4 02/10] classes-recipe/baremetal-image: Add image file manifest

There's already an IMAGE_MANIFEST file that's quite different from this? tmp/deploy/core-image-minimal-qemux86-64.rootfs.manifest base-files qemux86_64 3.0.14
base-passwd core2_64 3.6.3
busybox core2_64 1.36.1

So I find the name very confusing. Perhaps call this something completely different, like "result" or do-image-task-output or similar...

Regards
//Ernst

Den mån 24 juni 2024 kl 21:32 skrev Joshua Watt via lists.openembedded.org <[email protected]>:

Quoted 88 lines

Downstream tasks may want to know what image files were written so write out a manifest in do_image_complete. The format of the manifest is the same as the one in image.bbclass

---
 meta/classes-recipe/baremetal-image.bbclass | 30 +++++++++++++++++++--
 1 file changed, 28 insertions(+), 2 deletions(-)
meta/classes-recipe/baremetal-image.bbclass +3 -0
index 4e7d413626e..d7f53803e2a 100644
@@ -30,6 +30,9 @@ BAREMETAL_BINNAME ?= "hello_baremetal_${MACHINE}"
 IMAGE_LINK_NAME ?= "baremetal-helloworld-image-${MACHINE}"
 IMAGE_NAME_SUFFIX ?= ""
 
+IMAGE_FILE_MANIFEST_DIR = "${WORKDIR}/deploy-image-file-manifest"
+IMAGE_FILE_MANIFEST = "${IMAGE_FILE_MANIFEST_DIR}/manifest.json"
+
 do_rootfs[dirs] = "${IMGDEPLOYDIR} ${DEPLOY_DIR_IMAGE}"
 
 do_image(){
@@ -37,8 +40,28 @@ do_image(){
     install ${D}/${base_libdir}/firmware/${BAREMETAL_BINNAME}.elf

${IMGDEPLOYDIR}/${IMAGE_LINK_NAME}.elf
}

meta/classes-recipe/baremetal-image.bbclass +25 -3
-do_image_complete(){
-    :
+python do_image_complete(){
+    from pathlib import Path
+    import json
+
+    data = {
+        "taskname": "do_image",
+        "imagetype": "baremetal-image",
+        "images": []
+    }
+
+    img_deploy_dir = Path(d.getVar("IMGDEPLOYDIR"))
+
+    for child in img_deploy_dir.iterdir():
+        if not child.is_file() or child.is_symlink():
+            continue
+
+        data["images"].append({
+            "filename": child.name,
+        })
+
+    with open(d.getVar("IMAGE_FILE_MANIFEST"), "w") as f:
+        json.dump([data], f)
 }
 
 python do_rootfs(){
@@ -62,6 +85,7 @@ python do_rootfs(){
     bb.utils.mkdirhier(sysconfdir)
 
     execute_pre_post_process(d, d.getVar('ROOTFS_POSTPROCESS_COMMAND'))
+    execute_pre_post_process(d, d.getVar("ROOTFS_POSTUNINSTALL_COMMAND"))
 }
 
 
@@ -72,6 +96,8 @@ SSTATE_SKIP_CREATION:task-image-complete = '1'
 do_image_complete[sstate-inputdirs] = "${IMGDEPLOYDIR}"
 do_image_complete[sstate-outputdirs] = "${DEPLOY_DIR_IMAGE}"
 do_image_complete[stamp-extra-info] = "${MACHINE_ARCH}"
+do_image_complete[sstate-plaindirs] += "${IMAGE_FILE_MANIFEST_DIR}"
+do_image_complete[dirs] += "${IMAGE_FILE_MANIFEST_DIR}"
 addtask do_image_complete after do_image before do_build
 
 python do_image_complete_setscene () {
--

2.43.2

-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group. View/Reply Online (#201104):
https://lists.openembedded.org/g/openembedded-core/message/201104
Mute This Topic: https://lists.openembedded.org/mt/106856870/4947266
Group Owner: [email protected]
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [ [email protected]]
-=-=-=-=-=-=-=-=-=-=-=-

#44 lore raw

PATCH v4 3/10 classes/create-spdx-3.0: Add classes

In reply to Joshua Watt

Adds a class to generate SPDX 3.0 output and an image class that is used when generating images

---
 meta/classes-recipe/packagegroup.bbclass   |    2 +
 meta/classes/create-spdx-3.0.bbclass       | 1179 ++++
 meta/classes/create-spdx-image-3.0.bbclass |  224 +
 meta/classes/spdx-common.bbclass           |    6 +-
 meta/lib/oe/sbom30.py                      | 1122 ++++
 meta/lib/oe/spdx30.py                      | 5996 ++++++++++++++++++++
 6 files changed, 8528 insertions(+), 1 deletion(-)
 create mode 100644 meta/classes/create-spdx-3.0.bbclass
 create mode 100644 meta/classes/create-spdx-image-3.0.bbclass
 create mode 100644 meta/lib/oe/sbom30.py
 create mode 100644 meta/lib/oe/spdx30.py
meta/classes-recipe/packagegroup.bbclass +2 -0
index cf6fc354a81..2099c8b37d6 100644
@@ -56,6 +56,8 @@ deltask do_populate_sysroot
 
 do_create_runtime_spdx[deptask] = "do_create_spdx"
 do_create_runtime_spdx[rdeptask] = ""
+do_create_package_spdx[deptask] = "do_create_spdx"
+do_create_package_spdx[rdeptask] = ""
 
 INHIBIT_DEFAULT_DEPS = "1"
 
meta/classes/create-spdx-3.0.bbclass new file +1179 -0

1,179 lines not on this page · raw message

meta/classes/create-spdx-image-3.0.bbclass new file +224 -0

224 lines not on this page · raw message

meta/classes/spdx-common.bbclass +5 -1
index 03f1d0cc278..9dee325ba6b 100644
@@ -17,6 +17,7 @@ SPDXDEPLOY = "${SPDXDIR}/deploy"
 SPDXWORK = "${SPDXDIR}/work"
 SPDXIMAGEWORK = "${SPDXDIR}/image-work"
 SPDXSDKWORK = "${SPDXDIR}/sdk-work"
+SPDXSDKEXTWORK = "${SPDXDIR}/sdk-ext-work"
 SPDXDEPS = "${SPDXDIR}/deps.json"
 
 SPDX_TOOL_NAME ??= "oe-spdx-creator"
@@ -68,7 +69,7 @@ def get_json_indent(d):
         return 2
     return None
 
-python() {
+def load_spdx_license_data(d):
     import json
     if d.getVar("SPDX_LICENSE_DATA"):
         return
@@ -78,6 +79,9 @@ python() {
         # Transform the license array to a dictionary
         data["licenses"] = {l["licenseId"]: l for l in data["licenses"]}
         d.setVar("SPDX_LICENSE_DATA", data)
+
+python() {
+    load_spdx_license_data(d)
 }
 
 def process_sources(d):
meta/lib/oe/sbom30.py new file +1122 -0

1,122 lines not on this page · raw message

meta/lib/oe/spdx30.py new file +5996 -0

443 lines not on this page · raw message

The rest of this diff is not shown. View the raw message.

Signature · 1 line
2.43.2

Part of the diff is not shown here. View the raw message.

#45 lore raw

Re: [OE-core][PATCH v4 03/10] classes/create-spdx-3.0: Add classes

Quoted 52 lines · Joshua Watt via lists.openembedded.org
meta/classes/create-spdx-3.0.bbclass new file +52 -0
+python spdx30_build_started_handler () {
+    import oe.spdx30
+    import oe.sbom30
+    import os
+    from pathlib import Path
+
+    # Create a copy of the datastore. Set PN to "bitbake" so that SPDX IDs can
+    # be generated
+    d = e.data.createCopy()
+    d.setVar("PN", "bitbake")
+    d.setVar("BB_TASKHASH", "bitbake")
+    load_spdx_license_data(d)
+
+    deploy_dir_spdx = Path(e.data.getVar("DEPLOY_DIR_SPDX"))
+
+    nonce = os.urandom(16).hex()
+
+    objset = oe.sbom30.ObjectSet.new_objset(d, "bitbake", False)
+
+    build = objset.add_root(oe.spdx30.build_Build(
+        _id=objset.new_spdxid(nonce, include_unihash=False),
+        creationInfo=objset.doc.creationInfo,
+        build_buildType=oe.sbom30.SPDX_BUILD_TYPE,
+        build_buildStartTime=oe.sbom30.spdx_now()
+    ))
+
+    host_import_key = d.getVar("SPDX_BUILD_HOST")
+    if host_import_key:
+        objset.new_scoped_relationship(
+            [build],
+            oe.spdx30.RelationshipType.hasHost,
+            oe.spdx30.LifecycleScopeType.build,
+            [objset.new_import("SPDX_BUILD_HOST")],
+        )
+
+    invoked_by = objset.new_agent("SPDX_INVOKED_BY")
+    if invoked_by:
+        invoked_by_spdx = objset.new_scoped_relationship(
+            [build],
+            oe.spdx30.RelationshipType.invokedBy,
+            oe.spdx30.LifecycleScopeType.build,
+            [invoked_by],
+        )
+
+        on_behalf_of = objset.new_agent("SPDX_ON_BEHALF_OF")
+        if on_behalf_of:
+            objset.new_scoped_relationship(
+                [on_behalf_of],
+                oe.spdx30.RelationshipType.delegatedTo,
+                oe.spdx30.LifecycleScopeType.build,
+                invoked_by_spdx,
+            )

meta/classes/create-spdx-3.0.bbclass line 1178

Quoted 10 lines
meta/classes/create-spdx-3.0.bbclass new file +10 -0
+
+    for obj in objset.foreach_type(oe.spdx30.Element):
+        obj.extension.append(oe.sbom30.OELinkExtension(link_spdx_id=False))
+        obj.extension.append(oe.sbom30.OEIdAliasExtension())
+
+    oe.sbom30.write_jsonld_doc(d, objset, deploy_dir_spdx / "bitbake.spdx.json")
+}
+
+addhandler spdx30_build_started_handler
+spdx30_build_started_handler[eventmask] = "bb.event.ConfigParsed"

Reading through, this caught my eye. Do we really need to do that at ConfigParsed? I suspect you'll hit this a lot more than you expect since each new execution thread could parse the config and trigger this. Would BuildStarted not be more appropriate?

I'm also worried about determinism here. I didn't look into what data this is writing out but it worries me...

Cheers,

Richard

#46 lore raw

Re: [OE-core][PATCH v4 03/10] classes/create-spdx-3.0: Add classes

In reply to Joshua Watt

Comments inline below

Quoted 91 lines · Joshua Watt

Adds a class to generate SPDX 3.0 output and an image class that is used when generating images

---
  meta/classes-recipe/packagegroup.bbclass   |    2 +
  meta/classes/create-spdx-3.0.bbclass       | 1179 ++++
  meta/classes/create-spdx-image-3.0.bbclass |  224 +
  meta/classes/spdx-common.bbclass           |    6 +-
  meta/lib/oe/sbom30.py                      | 1122 ++++
  meta/lib/oe/spdx30.py                      | 5996 ++++++++++++++++++++
  6 files changed, 8528 insertions(+), 1 deletion(-)

create mode 100644 meta/classes/create-spdx-3.0.bbclass create mode 100644 meta/classes/create-spdx-image-3.0.bbclass create mode 100644 meta/lib/oe/sbom30.py
create mode 100644 meta/lib/oe/spdx30.py

meta/classes-recipe/packagegroup.bbclass +2 -0
index cf6fc354a81..2099c8b37d6 100644
@@ -56,6 +56,8 @@ deltask do_populate_sysroot
 
  do_create_runtime_spdx[deptask] = "do_create_spdx"
  do_create_runtime_spdx[rdeptask] = ""
+do_create_package_spdx[deptask] = "do_create_spdx"
+do_create_package_spdx[rdeptask] = ""
 
  INHIBIT_DEFAULT_DEPS = "1"
 
meta/classes/create-spdx-3.0.bbclass new file +55 -0
new file mode 100644
index 00000000000..ac84bb0ca26
@@ -0,0 +1,1179 @@
+#
+# Copyright OpenEmbedded Contributors
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+
+inherit spdx-common
+
+SPDX_VERSION = "3.0.0"
+
+# The list of SPDX profiles generated documents will conform to
+SPDX_PROFILES ?= "core build software simpleLicensing security"
+
+SPDX_INCLUDE_BUILD_VARIABLES ??= "0"
+SPDX_INCLUDE_BUILD_VARIABLES[doc] = "If set to '1', the bitbake variables for a \
+    recipe will be included in the Build object"
+
+SPDX_IMPORTS ??= ""
+SPDX_IMPORTS[doc] = "SPDX_IMPORTS is the base variable that describes how to \
+    reference external SPDX ids. Each import is defined as a key in this \
+    variable with a suffix to describe to as a suffix to look up more \
+    information about the import. Each key can have the following variables: \
+        SPDX_IMPORTS_<key>_spdxid: The Fully qualified SPDX ID of the object \
+        SPDX_IMPORTS_<key>_uri: The URI where the SPDX Document that contains \
+            the external object can be found. Optional but recommended \
+        SPDX_IMPORTS_<key>_hash_<hash>: The Checksum of the SPDX Document that \
+            contains the External ID. <hash> must be one the valid SPDX hashing \
+            algorithms, as described by the HashAlgorithm vocabulary in the\
+            SPDX 3 spec. Optional but recommended \
+"
+
+# Agents
+#   Bitbake variables can be used to describe an SPDX Agent that may be used
+#   during the build. An Agent is specified using a set of variables which all
+#   start with some common base name:
+#
+#   <BASE>_name: The name of the Agent (required)
+#   <BASE>_type: The type of Agent. Must be one of "person", "organization",
+#       "software", or "agent" (the default if not specified)
+#   <BASE>_comment: The comment for the Agent (optional)
+#   <BASE>_id_<ID>: And External Identifier for the Agent. <ID> must be a valid
+#       ExternalIdentifierType from the SPDX 3 spec. Commonly, an E-mail address
+#       can be specified with <BASE>_id_email
+#
+#   Alternatively, an Agent can be an external reference by referencing a key
+#   in SPDX_IMPORTS like so:
+#
+#   <BASE>_import = "<key>"
+#
+#   Finally, the same agent described by another set of agent variables can be
+#   referenced by specifying the basename of the variable that should be
+#   referenced:
+#
+#   SPDX_PACKAGE_SUPPLIER_ref = "SPDX_AUTHORS_openembedded"
+

meta/classes/create-spdx-3.0.bbclass line 65

Quoted 10 lines
meta/classes/create-spdx-3.0.bbclass new file +10 -0
+SPDX_AUTHORS ??= "openembedded"
+SPDX_AUTHORS[doc] = "A space separated list of the document authors. Each item \
+    is used to name a base variable like SPDX_AUTHORS_<AUTHOR> that \
+    describes the author."
+
+SPDX_AUTHORS_openembedded_name = "OpenEmbedded"
+SPDX_AUTHORS_openembedded_type = "organization"
+
+SPDX_BUILD_HOST[doc] = "The base variable name to describe the build host on \
+    which a build is running. Must be an SPDX_IMPORTS key"

Is there any sort of documentation or external reference for the variable above (as well as the SPDX_ below) that explains what the SPDX standard is expecting to be put in there?

I.e. machine name, host type, an arbitrary string that means something to the agent construction the SPDX, etc. I.e. how would I know what is valid in these various things?

This then leads to a second question, deterministic behavior/reproducibility. I believe the purpose of this is reproducible builds, but we should have a more deterministic approach in the Yocto Project where we provide (and/or check for host capabilities) to help allow this to be a more generic, many different host process.

Which leads to a third question, when a build uses sstate-cache, each host could be different then the build host that actually combines the builds into an image (SBOM). Is this a concern?

Quoted 8,515 lines
meta/classes/create-spdx-3.0.bbclass new file +1113 -0
+SPDX_INVOKED_BY[doc] = "The base variable name to describe the Agent that \
+    invoked the build, which builds will link to if specified."
+
+SPDX_ON_BEHALF_OF[doc] = "The base variable name to describe the Agent on who's \
+    behalf the invoking Agent (SPDX_INVOKED_BY) is running the build."
+
+SPDX_PACKAGE_SUPPLIER[doc] = "The base variable name to describe the Agent who is supplying \
+    artifacts produced by the build"
+
+SPDX_PACKAGE_ADDITIONAL_PURPOSE ?= ""
+SPDX_PACKAGE_ADDITIONAL_PURPOSE[doc] = "The list of additional purposes to assign to \
+    the generated packages for a recipe. The primary purpose is always `install`. \
+    Packages overrides are allowed to override the additional purposes for \
+    individual packages."
+
+SPDX_IMAGE_PURPOSE ?= "filesystemImage"
+SPDX_IMAGE_PURPOSE[doc] = "The list of purposes to assign to the generated images. \
+    The first listed item will be the Primary Purpose and all additional items will \
+    be added as additional purposes"
+
+SPDX_SDK_PURPOSE ?= "install"
+SPDX_SDK_PURPOSE[doc] = "The list of purposes to assign to the generate SDK installer. \
+    The first listed item will be the Primary Purpose and all additional items will \
+    be added as additional purposes"
+
+SPDX_INCLUDE_VEX ??= "current"
+SPDX_INCLUDE_VEX[doc] = "Controls what VEX information is in the output. Set to \
+    'none' to disable all VEX data. Set to 'current' to only include VEX data \
+    for vulnerabilities not already fixed in the upstream source code \
+    (recommended). Set  to 'all' to get all known historical vulnerabilities, \
+    including those already fixed upstream (warning: This can be large and \
+    slow)."
+
+IMAGE_CLASSES:append = " create-spdx-image-3.0"
+
+def add_license_expression(d, objset, license_expression):
+    from pathlib import Path
+    import oe.spdx30
+    import oe.sbom30
+
+    license_data = d.getVar("SPDX_LICENSE_DATA")
+    simple_license_text = {}
+    license_text_map = {}
+    license_ref_idx = 0
+
+    def add_license_text(name):
+        nonlocal objset
+        nonlocal simple_license_text
+
+        if name in simple_license_text:
+            return simple_license_text[name]
+
+        lic = objset.find_filter(
+            oe.spdx30.simplelicensing_SimpleLicensingText,
+            name=name,
+        )
+
+        if lic is not None:
+            simple_license_text[name] = lic
+            return lic
+
+        lic = objset.add(oe.spdx30.simplelicensing_SimpleLicensingText(
+            _id=objset.new_spdxid("license-text", name),
+            creationInfo=objset.doc.creationInfo,
+            name=name,
+        ))
+        simple_license_text[name] = lic
+
+        if name == "PD":
+            lic.simplelicensing_licenseText = "Software released to the public domain"
+            return lic
+
+        # Seach for the license in COMMON_LICENSE_DIR and LICENSE_PATH
+        for directory in [d.getVar('COMMON_LICENSE_DIR')] + (d.getVar('LICENSE_PATH') or '').split():
+            try:
+                with (Path(directory) / name).open(errors="replace") as f:
+                    lic.simplelicensing_licenseText = f.read()
+                    return lic
+
+            except FileNotFoundError:
+                pass
+
+        # If it's not SPDX or PD, then NO_GENERIC_LICENSE must be set
+        filename = d.getVarFlag('NO_GENERIC_LICENSE', name)
+        if filename:
+            filename = d.expand("${S}/" + filename)
+            with open(filename, errors="replace") as f:
+                lic.simplelicensing_licenseText = f.read()
+                return lic
+        else:
+            bb.fatal("Cannot find any text for license %s" % name)
+
+    def convert(l):
+        nonlocal license_text_map
+        nonlocal license_ref_idx
+
+        if l == "(" or l == ")":
+            return l
+
+        if l == "&":
+            return "AND"
+
+        if l == "|":
+            return "OR"
+
+        if l == "CLOSED":
+            return "NONE"
+
+        spdx_license = d.getVarFlag("SPDXLICENSEMAP", l) or l
+        if spdx_license in license_data["licenses"]:
+            return spdx_license
+
+        spdx_license = "LicenseRef-" + l
+        if spdx_license not in license_text_map:
+            license_text_map[spdx_license] = add_license_text(l)._id
+
+        return spdx_license
+
+    lic_split = license_expression.replace("(", " ( ").replace(")", " ) ").replace("|", " | ").replace("&", " & ").split()
+    spdx_license_expression = ' '.join(convert(l) for l in lic_split)
+
+    return objset.new_license_expression(spdx_license_expression, license_text_map)
+
+
+def add_package_files(d, objset, topdir, get_spdxid, get_purposes, *, archive=None, ignore_dirs=[], ignore_top_level_dirs=[]):
+    from pathlib import Path
+    import oe.spdx30
+    import oe.sbom30
+
+    source_date_epoch = d.getVar("SOURCE_DATE_EPOCH")
+    if source_date_epoch:
+        source_date_epoch = int(source_date_epoch)
+
+    spdx_files = set()
+
+    file_counter = 1
+    for subdir, dirs, files in os.walk(topdir):
+        dirs[:] = [d for d in dirs if d not in ignore_dirs]
+        if subdir == str(topdir):
+            dirs[:] = [d for d in dirs if d not in ignore_top_level_dirs]
+
+        for file in files:
+            filepath = Path(subdir) / file
+            if filepath.is_symlink() or not filepath.is_file():
+                continue
+
+            bb.debug(1, "Adding file %s to %s" % (filepath, objset.doc._id))
+
+            filename = str(filepath.relative_to(topdir))
+            file_purposes = get_purposes(filepath)
+
+            spdx_file = objset.new_file(
+                get_spdxid(file_counter),
+                filename,
+                filepath,
+                purposes=file_purposes,
+            )
+            spdx_files.add(spdx_file)
+
+            if oe.spdx30.software_SoftwarePurpose.source in file_purposes:
+                objset.scan_declared_licenses(spdx_file, filepath)
+
+            if archive is not None:
+                with filepath.open("rb") as f:
+                    info = archive.gettarinfo(fileobj=f)
+                    info.name = filename
+                    info.uid = 0
+                    info.gid = 0
+                    info.uname = "root"
+                    info.gname = "root"
+
+                    if source_date_epoch is not None and info.mtime > source_date_epoch:
+                        info.mtime = source_date_epoch
+
+                    archive.addfile(info, f)
+
+            file_counter += 1
+
+    return spdx_files
+
+
+def get_package_sources_from_debug(d, package, package_files, sources, source_hash_cache):
+    from pathlib import Path
+    import oe.packagedata
+
+    def file_path_match(file_path, pkg_file):
+        if file_path.lstrip("/") == pkg_file.name.lstrip("/"):
+            return True
+
+        for e in pkg_file.extension:
+            if isinstance(e, oe.sbom30.OEFileNameAliasExtension):
+                for a in e.aliases:
+                    if file_path.lstrip("/") == a.lstrip("/"):
+                        return True
+
+        return False
+
+    debug_search_paths = [
+        Path(d.getVar('PKGD')),
+        Path(d.getVar('STAGING_DIR_TARGET')),
+        Path(d.getVar('STAGING_DIR_NATIVE')),
+        Path(d.getVar('STAGING_KERNEL_DIR')),
+    ]
+
+    pkg_data = oe.packagedata.read_subpkgdata_extended(package, d)
+
+    if pkg_data is None:
+        return
+
+    dep_source_files = set()
+
+    for file_path, file_data in pkg_data["files_info"].items():
+        if not "debugsrc" in file_data:
+            continue
+
+        if not any(file_path_match(file_path, pkg_file) for pkg_file in package_files):
+            bb.fatal("No package file found for %s in %s; SPDX found: %s" % (str(file_path), package,
+                " ".join(p.name for p in package_files)))
+            continue
+
+        for debugsrc in file_data["debugsrc"]:
+            for search in debug_search_paths:
+                if debugsrc.startswith("/usr/src/kernel"):
+                    debugsrc_path = search / debugsrc.replace('/usr/src/kernel/', '')
+                else:
+                    debugsrc_path = search / debugsrc.lstrip("/")
+
+                if debugsrc_path in source_hash_cache:
+                    file_sha256 = source_hash_cache[debugsrc_path]
+                    if file_sha256 is None:
+                        continue
+                else:
+                    if not debugsrc_path.exists():
+                        source_hash_cache[debugsrc_path] = None
+                        continue
+
+                    file_sha256 = bb.utils.sha256_file(debugsrc_path)
+                    source_hash_cache[debugsrc_path] = file_sha256
+
+                if file_sha256 in sources:
+                    source_file = sources[file_sha256]
+                    dep_source_files.add(source_file)
+                else:
+                    bb.debug(1, "Debug source %s with SHA256 %s not found in any dependency" % (str(debugsrc_path), file_sha256))
+                break
+            else:
+                bb.debug(1, "Debug source %s not found" % debugsrc)
+
+    return dep_source_files
+
+get_package_sources_from_debug[vardepsexclude] += "STAGING_KERNEL_DIR"
+
+def collect_dep_objsets(d, build):
+    import json
+    from pathlib import Path
+    import oe.sbom30
+    import oe.spdx30
+
+    deps = get_spdx_deps(d)
+
+    dep_objsets = []
+    dep_builds = set()
+
+    dep_build_spdxids = set()
+    for dep_pn, _, in_taskhash  in deps:
+        bb.debug(1, "Fetching SPDX for dependency %s" % (dep_pn))
+        dep_build, dep_objset = oe.sbom30.find_root_obj_in_jsonld(d, "recipes", dep_pn, oe.spdx30.build_Build)
+        # If the dependency is part of the taskhash, return it to be linked
+        # against. Otherwise, it cannot be linked against because this recipe
+        # will not rebuilt if dependency changes
+        if in_taskhash:
+            dep_objsets.append(dep_objset)
+
+        # The build _can_ be linked against (by alias)
+        dep_builds.add(dep_build)
+
+    return dep_objsets, dep_builds
+
+collect_dep_objsets[vardepsexclude] = "SSTATE_ARCHS"
+
+def collect_dep_sources(dep_objsets):
+    import oe.spdx30
+    import oe.sbom30
+
+    sources = {}
+    for objset in dep_objsets:
+        # Don't collect sources from native recipes as they
+        # match non-native sources also.
+        if objset.is_native():
+            continue
+
+        bb.debug(1, "Fetching Sources for dependency %s" % (objset.doc.name))
+
+        dep_build = objset.find_root(oe.spdx30.build_Build)
+        if not dep_build:
+            bb.fatal("Unable to find a build")
+
+        for e in objset.foreach_type(oe.spdx30.Relationship):
+            if dep_build is not e.from_:
+                continue
+
+            if e.relationshipType != oe.spdx30.RelationshipType.hasInputs:
+                continue
+
+            for to in e.to:
+                if not isinstance(to, oe.spdx30.software_File):
+                    continue
+
+                if to.software_primaryPurpose != oe.spdx30.software_SoftwarePurpose.source:
+                    continue
+
+                for v in to.verifiedUsing:
+                    if v.algorithm == oe.spdx30.HashAlgorithm.sha256:
+                        sources[v.hashValue] = to
+                        break
+                else:
+                    bb.fatal("No SHA256 found for %s in %s" % (to.name, objset.doc.name))
+
+    return sources
+
+def add_download_files(d, objset):
+    import oe.patch
+    import oe.spdx30
+    import os
+
+    inputs = set()
+
+    urls = d.getVar("SRC_URI").split()
+    fetch = bb.fetch2.Fetch(urls, d)
+
+    for download_idx, src_uri in enumerate(urls):
+        fd = fetch.ud[src_uri]
+
+        for name in fd.names:
+            file_name = os.path.basename(fetch.localpath(src_uri))
+            if oe.patch.patch_path(src_uri, fetch, '', expand=False):
+                primary_purpose = oe.spdx30.software_SoftwarePurpose.patch
+            else:
+                primary_purpose = oe.spdx30.software_SoftwarePurpose.source
+
+            if fd.type == "file":
+                if os.path.isdir(fd.localpath):
+                    walk_idx = 1
+                    for root, dirs, files in os.walk(fd.localpath):
+                        for f in files:
+                            f_path = os.path.join(root, f)
+                            if os.path.islink(f_path):
+                                # TODO: SPDX doesn't support symlinks yet
+                                continue
+
+                            file = objset.new_file(
+                                objset.new_spdxid("source", str(download_idx + 1), str(walk_idx)),
+                                os.path.join(file_name, os.path.relpath(f_path, fd.localpath)),
+                                f_path,
+                                purposes=[primary_purpose],
+                            )
+
+                            inputs.add(file)
+                            walk_idx += 1
+
+                else:
+                    file = objset.new_file(
+                        objset.new_spdxid("source", str(download_idx + 1)),
+                        file_name,
+                        fd.localpath,
+                        purposes=[primary_purpose],
+                    )
+                    inputs.add(file)
+
+            else:
+                uri = fd.type
+                proto = getattr(fd, "proto", None)
+                if proto is not None:
+                    uri = uri + "+" + proto
+                uri = uri + "://" + fd.host + fd.path
+
+                if fd.method.supports_srcrev():
+                    uri = uri + "@" + fd.revisions[name]
+
+                dl = objset.add(oe.spdx30.software_Package(
+                    _id=objset.new_spdxid("source", str(download_idx + 1)),
+                    creationInfo=objset.doc.creationInfo,
+                    name=file_name,
+                    software_primaryPurpose=primary_purpose,
+                    software_downloadLocation=uri,
+                ))
+
+                if fd.method.supports_checksum(fd):
+                    # TODO Need something better than hard coding this
+                    for checksum_id in ["sha256", "sha1"]:
+                        expected_checksum = getattr(fd, "%s_expected" % checksum_id, None)
+                        if expected_checksum is None:
+                            continue
+
+                        dl.verifiedUsing.append(
+                            oe.spdx30.Hash(
+                                algorithm=getattr(oe.spdx30.HashAlgorithm, checksum_id),
+                                hashValue=expected_checksum,
+                            )
+                        )
+
+                inputs.add(dl)
+
+    return inputs
+
+
+def collect_build_package_inputs(d, objset, build, packages):
+    providers = collect_package_providers(d)
+
+    build_deps = set()
+
+    for name in sorted(packages.keys()):
+        if name not in providers:
+            bb.fatal("Unable to find SPDX provider for '%s'" % name)
+
+        pkg_name, pkg_hashfn = providers[name]
+
+        # Copy all of the package SPDX files into the Sbom elements
+        pkg_spdx, _ = oe.sbom30.find_root_obj_in_jsonld(
+            d,
+            "packages",
+            pkg_name,
+            oe.spdx30.software_Package,
+            software_primaryPurpose=oe.spdx30.software_SoftwarePurpose.install,
+        )
+        build_deps.add(pkg_spdx._id)
+
+    if build_deps:
+        objset.new_scoped_relationship(
+            [build],
+            oe.spdx30.RelationshipType.hasInputs,
+            oe.spdx30.LifecycleScopeType.build,
+            sorted(list(build_deps)),
+        )
+
+def set_purposes(d, element, *var_names, force_purposes=[]):
+    purposes = force_purposes[:]
+
+    for var_name in var_names:
+        val = d.getVar(var_name)
+        if val:
+            purposes.extend(val.split())
+            break
+
+    if not purposes:
+        bb.warn("No SPDX purposes found in %s" % " ".join(var_names))
+        return
+
+    element.software_primaryPurpose = getattr(oe.spdx30.software_SoftwarePurpose, purposes[0])
+    element.software_additionalPurpose = [getattr(oe.spdx30.software_SoftwarePurpose, p) for p in purposes[1:]]
+
+
+python do_create_spdx() {
+    import oe.sbom30
+    import oe.spdx30
+    from pathlib import Path
+    from contextlib import contextmanager
+    import oe.cve_check
+    from datetime import datetime
+
+    def set_var_field(var, obj, name, package=None):
+        val = None
+        if package:
+            val = d.getVar("%s:%s" % (var, package))
+
+        if not val:
+            val = d.getVar(var)
+
+        if val:
+            setattr(obj, name, val)
+
+    deploydir = Path(d.getVar("SPDXDEPLOY"))
+    deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX"))
+    spdx_workdir = Path(d.getVar("SPDXWORK"))
+    include_sources = d.getVar("SPDX_INCLUDE_SOURCES") == "1"
+    pkg_arch = d.getVar("SSTATE_PKGARCH")
+    is_native = bb.data.inherits_class("native", d) or bb.data.inherits_class("cross", d)
+    include_vex = d.getVar("SPDX_INCLUDE_VEX")
+    if not include_vex in ("none", "current", "all"):
+        bb.fatal("SPDX_INCLUDE_VEX must be one of 'none', 'current', 'all'")
+
+    build_objset = oe.sbom30.ObjectSet.new_objset(d, d.getVar("PN"))
+
+    build = build_objset.new_sub_build("recipe", "recipe")
+    build_objset.doc.rootElement.append(build)
+
+    build_objset.set_is_native(is_native)
+
+    for var in (d.getVar('SPDX_CUSTOM_ANNOTATION_VARS') or "").split():
+        new_annotation(
+            d,
+            build_objset,
+            build,
+            "%s=%s" % (var, d.getVar(var)),
+            oe.spdx30.AnnotationType.other
+        )
+
+    build_inputs = set()
+
+    # Add CVEs
+    cve_by_status = {}
+    if include_vex != "none":
+        for cve in (d.getVarFlags("CVE_STATUS") or {}):
+            status, detail, description = oe.cve_check.decode_cve_status(d, cve)
+
+            # If this CVE is fixed upstream, skip it unless all CVEs are
+            # specified.
+            if include_vex != "all" and detail in ("fixed-version", "cpe-stable-backport"):
+                bb.debug(1, "Skipping %s since it is already fixed upstream" % cve)
+                continue
+
+            cve_by_status.setdefault(status, {})[cve] = (
+                build_objset.new_cve_vuln(cve),
+                detail,
+                description,
+            )
+
+    cpe_ids = oe.cve_check.get_cpe_ids(d.getVar("CVE_PRODUCT"), d.getVar("CVE_VERSION"))
+
+    source_files = add_download_files(d, build_objset)
+    build_inputs |= source_files
+
+    recipe_spdx_license = add_license_expression(d, build_objset, d.getVar("LICENSE"))
+    build_objset.new_relationship(
+        source_files,
+        oe.spdx30.RelationshipType.hasConcludedLicense,
+        [recipe_spdx_license],
+    )
+
+    if process_sources(d) and include_sources:
+        bb.debug(1, "Adding source files to SPDX")
+        spdx_get_src(d)
+
+        build_inputs |= add_package_files(
+            d,
+            build_objset,
+            spdx_workdir,
+            lambda file_counter: build_objset.new_spdxid("sourcefile", str(file_counter)),
+            lambda filepath: [oe.spdx30.software_SoftwarePurpose.source],
+            ignore_dirs=[".git"],
+            ignore_top_level_dirs=["temp"],
+            archive=None,
+        )
+
+
+    dep_objsets, dep_builds = collect_dep_objsets(d, build)
+    if dep_builds:
+        build_objset.new_scoped_relationship(
+            [build],
+            oe.spdx30.RelationshipType.dependsOn,
+            oe.spdx30.LifecycleScopeType.build,
+            sorted(oe.sbom30.get_element_link_id(b) for b in dep_builds),
+        )
+
+    debug_source_ids = set()
+    source_hash_cache = {}
+
+    # Write out the package SPDX data now. It is not complete as we cannot
+    # write the runtime data, so write it to a staging area and a later task
+    # will write out the final collection
+
+    # TODO: Handle native recipe output
+    if not is_native:
+        bb.debug(1, "Collecting Dependency sources files")
+        sources = collect_dep_sources(dep_objsets)
+
+        bb.build.exec_func("read_subpackage_metadata", d)
+
+        pkgdest = Path(d.getVar("PKGDEST"))
+        for package in d.getVar("PACKAGES").split():
+            if not oe.packagedata.packaged(package, d):
+                continue
+
+            pkg_name = d.getVar("PKG:%s" % package) or package
+
+            bb.debug(1, "Creating SPDX for package %s" % pkg_name)
+
+            pkg_objset = oe.sbom30.ObjectSet.new_objset(d, pkg_name)
+
+            spdx_package = pkg_objset.add_root(oe.spdx30.software_Package(
+                _id=pkg_objset.new_spdxid("package", pkg_name),
+                creationInfo=pkg_objset.doc.creationInfo,
+                name=pkg_name,
+                software_packageVersion=d.getVar("PV"),
+                builtTime=oe.sbom30.spdx_now(),
+            ))
+
+            set_purposes(
+                d,
+                spdx_package,
+                "SPDX_PACKAGE_ADDITIONAL_PURPOSE:%s" % package,
+                "SPDX_PACKAGE_ADDITIONAL_PURPOSE",
+                force_purposes=["install"],
+            )
+
+
+            supplier = build_objset.new_agent("SPDX_PACKAGE_SUPPLIER")
+            if supplier is not None:
+                spdx_package.supplier = supplier if isinstance(supplier, str) else supplier._id
+
+            set_var_field("HOMEPAGE", spdx_package, "software_homePage", package=package)
+            set_var_field("SUMMARY", spdx_package, "summary", package=package)
+            set_var_field("DESCRIPTION", spdx_package, "description", package=package)
+
+            pkg_objset.new_scoped_relationship(
+                [build._id],
+                oe.spdx30.RelationshipType.hasOutputs,
+                oe.spdx30.LifecycleScopeType.build,
+                [spdx_package],
+            )
+
+            for cpe_id in cpe_ids:
+                spdx_package.externalIdentifier.append(
+                    oe.spdx30.ExternalIdentifier(
+                        externalIdentifierType=oe.spdx30.ExternalIdentifierType.cpe23,
+                        identifier=cpe_id,
+                ))
+
+            # TODO: Generate a file for each actual IPK/DEB/RPM/TGZ file
+            # generated and link it to the package
+            #spdx_package_file = pkg_objset.add(oe.spdx30.software_File(
+            #    _id=pkg_objset.new_spdxid("distribution", pkg_name),
+            #    creationInfo=pkg_objset.doc.creationInfo,
+            #    name=pkg_name,
+            #    builtTime=oe.sbom30.spdx_now(),
+            #    software_primaryPurpose=spdx_package.software_primaryPurpose,
+            #    software_additionalPurpose=spdx_package.software_additionalPurpose,
+            #))
+
+            ## TODO add hashes
+            #pkg_objset.new_relationship(
+            #    [spdx_package],
+            #    oe.spdx30.RelationshipType.hasDistributionArtifact,
+            #    [spdx_package_file],
+            #)
+
+            # NOTE: licenses live in the recipe collection and are referenced
+            # by ID in the package collection(s). This helps reduce duplication
+            # (since a lot of packages will have the same license), and also
+            # prevents duplicate license SPDX IDs in the packages
+            package_license = d.getVar("LICENSE:%s" % package)
+            if package_license and package_license != d.getVar("LICENSE"):
+                package_spdx_license = add_license_expression(d, build_objset, package_license)
+            else:
+                package_spdx_license = recipe_spdx_license
+
+            pkg_objset.new_relationship(
+                [spdx_package],
+                oe.spdx30.RelationshipType.hasConcludedLicense,
+                [package_spdx_license._id],
+            )
+
+            # NOTE: CVE Elements live in the recipe collection
+            all_cves = set()
+            for status, cves in cve_by_status.items():
+                for cve, items in cves.items():
+                    spdx_cve, detail, description = items
+
+                    all_cves.add(spdx_cve._id)
+
+                    if status == "Patched":
+                        pkg_objset.new_vex_patched_relationship([spdx_cve._id], [spdx_package])
+                    elif status == "Unpatched":
+                        pkg_objset.new_vex_unpatched_relationship([spdx_cve._id], [spdx_package])
+                    elif status == "Ignored":
+                        spdx_vex = pkg_objset.new_vex_ignored_relationship(
+                            [spdx_cve._id],
+                            [spdx_package],
+                            impact_statement=description,
+                        )
+
+                        if detail in ("ignored", "cpe-incorrect", "disputed", "upstream-wontfix"):
+                            # VEX doesn't have justifications for this
+                            pass
+                        elif detail in ("not-applicable-config", "not-applicable-platform"):
+                            for v in spdx_vex:
+                                v.security_justificationType = oe.spdx30.security_VexJustificationType.vulnerableCodeNotPresent
+                        else:
+                            bb.fatal(f"Unknown detail '{detail}' for ignored {cve}")
+                    else:
+                        bb.fatal(f"Unknown CVE status {status}")
+
+            if all_cves:
+                pkg_objset.new_relationship(
+                    [spdx_package],
+                    oe.spdx30.RelationshipType.hasAssociatedVulnerability,
+                    sorted(list(all_cves)),
+                )
+
+            bb.debug(1, "Adding package files to SPDX for package %s" % pkg_name)
+            package_files = add_package_files(
+                d,
+                pkg_objset,
+                pkgdest / package,
+                lambda file_counter: pkg_objset.new_spdxid("package", pkg_name, "file", str(file_counter)),
+                # TODO: Can we know the purpose here?
+                lambda filepath: [],
+                ignore_top_level_dirs=['CONTROL', 'DEBIAN'],
+                archive=None,
+            )
+
+            if package_files:
+                pkg_objset.new_relationship(
+                    [spdx_package],
+                    oe.spdx30.RelationshipType.contains,
+                    sorted(list(package_files)),
+                )
+
+            if include_sources:
+                debug_sources = get_package_sources_from_debug(d, package, package_files, sources, source_hash_cache)
+                debug_source_ids |= set(oe.sbom30.get_element_link_id(d) for d in debug_sources)
+
+            oe.sbom30.write_recipe_jsonld_doc(d, pkg_objset, "packages-staging", deploydir, create_spdx_id_links=False)
+
+    if include_sources:
+        bb.debug(1, "Adding sysroot files to SPDX")
+        sysroot_files = add_package_files(
+            d,
+            build_objset,
+            d.expand("${COMPONENTS_DIR}/${PACKAGE_ARCH}/${PN}"),
+            lambda file_counter: build_objset.new_spdxid("sysroot", str(file_counter)),
+            lambda filepath: [],
+            archive=None,
+        )
+
+        if sysroot_files:
+            build_objset.new_scoped_relationship(
+                [build],
+                oe.spdx30.RelationshipType.hasOutputs,
+                oe.spdx30.LifecycleScopeType.build,
+                sorted(list(sysroot_files)),
+            )
+
+    if build_inputs or debug_source_ids:
+        build_objset.new_scoped_relationship(
+            [build],
+            oe.spdx30.RelationshipType.hasInputs,
+            oe.spdx30.LifecycleScopeType.build,
+            sorted(list(build_inputs)) + sorted(list(debug_source_ids)),
+        )
+
+    oe.sbom30.write_recipe_jsonld_doc(d, build_objset, "recipes", deploydir)
+}
+do_create_spdx[vardepsexclude] += "BB_NUMBER_THREADS"
+addtask do_create_spdx after \
+    do_collect_spdx_deps \
+    do_populate_sysroot do_package do_packagedata \
+    ${create_spdx_source_deps(d)} \
+    do_package_write_ipk do_package_write_deb do_package_write_rpm do_package_write_tar \
+    before do_populate_sdk do_populate_sdk_ext do_build do_rm_work
+
+def create_spdx_source_deps(d):
+    deps = []
+    if d.getVar("SPDX_INCLUDE_SOURCES") == "1":
+        deps.extend([
+            # do_unpack is a hack for now; we only need it to get the
+            # dependencies do_unpack already has so we can extract the source
+            # ourselves
+            "do_unpack",
+            # For kernel source code
+            "do_shared_workdir",
+        ])
+    return " ".join(deps)
+
+SSTATETASKS += "do_create_spdx"
+do_create_spdx[sstate-inputdirs] = "${SPDXDEPLOY}"
+do_create_spdx[sstate-outputdirs] = "${DEPLOY_DIR_SPDX}"
+
+python do_create_spdx_setscene () {
+    sstate_setscene(d)
+}
+addtask do_create_spdx_setscene
+
+do_create_spdx[dirs] = "${SPDXWORK}"
+do_create_spdx[cleandirs] = "${SPDXDEPLOY} ${SPDXWORK}"
+do_create_spdx[depends] += "${PATCHDEPENDENCY}"
+
+python do_create_package_spdx() {
+    import oe.sbom30
+    import oe.spdx30
+    import oe.packagedata
+    from pathlib import Path
+
+    deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX"))
+    deploydir = Path(d.getVar("SPDXRUNTIMEDEPLOY"))
+    is_native = bb.data.inherits_class("native", d) or bb.data.inherits_class("cross", d)
+
+    providers = collect_package_providers(d)
+    pkg_arch = d.getVar("SSTATE_PKGARCH")
+
+    if not is_native:
+        bb.build.exec_func("read_subpackage_metadata", d)
+
+        dep_package_cache = {}
+
+        # Any element common to all packages that need to be referenced by ID
+        # should be written into this objset set
+        common_objset = oe.sbom30.ObjectSet.new_objset(d, "%s-package-common" % d.getVar("PN"))
+
+        pkgdest = Path(d.getVar("PKGDEST"))
+        for package in d.getVar("PACKAGES").split():
+            localdata = bb.data.createCopy(d)
+            pkg_name = d.getVar("PKG:%s" % package) or package
+            localdata.setVar("PKG", pkg_name)
+            localdata.setVar('OVERRIDES', d.getVar("OVERRIDES", False) + ":" + package)
+
+            if not oe.packagedata.packaged(package, localdata):
+                continue
+
+            spdx_package, pkg_objset = oe.sbom30.load_obj_in_jsonld(
+                d,
+                pkg_arch,
+                "packages-staging",
+                pkg_name,
+                oe.spdx30.software_Package,
+                software_primaryPurpose=oe.spdx30.software_SoftwarePurpose.install,
+            )
+
+            # We will write out a new collection, so link it to the new
+            # creation info in the common package data. The old creation info
+            # should still exist and be referenced by all the existing elements
+            # in the package
+            pkg_objset.creationInfo = pkg_objset.copy_creation_info(common_objset.doc.creationInfo)
+
+            runtime_spdx_deps = set()
+
+            deps = bb.utils.explode_dep_versions2(localdata.getVar("RDEPENDS") or "")
+            seen_deps = set()
+            for dep, _ in deps.items():
+                if dep in seen_deps:
+                    continue
+
+                if dep not in providers:
+                    continue
+
+                (dep, _) = providers[dep]
+
+                if not oe.packagedata.packaged(dep, localdata):
+                    continue
+
+                dep_pkg_data = oe.packagedata.read_subpkgdata_dict(dep, d)
+                dep_pkg = dep_pkg_data["PKG"]
+
+                if dep in dep_package_cache:
+                    dep_spdx_package = dep_package_cache[dep]
+                else:
+                    bb.debug(1, "Searching for %s" % dep_pkg)
+                    dep_spdx_package, _ = oe.sbom30.find_root_obj_in_jsonld(
+                        d,
+                        "packages-staging",
+                        dep_pkg,
+                        oe.spdx30.software_Package,
+                        software_primaryPurpose=oe.spdx30.software_SoftwarePurpose.install,
+                    )
+                    dep_package_cache[dep] = dep_spdx_package
+
+                runtime_spdx_deps.add(dep_spdx_package)
+                seen_deps.add(dep)
+
+            if runtime_spdx_deps:
+                pkg_objset.new_scoped_relationship(
+                    [spdx_package],
+                    oe.spdx30.RelationshipType.dependsOn,
+                    oe.spdx30.LifecycleScopeType.runtime,
+                    [oe.sbom30.get_element_link_id(dep) for dep in runtime_spdx_deps],
+                )
+
+            oe.sbom30.write_recipe_jsonld_doc(d, pkg_objset, "packages", deploydir)
+
+        oe.sbom30.write_recipe_jsonld_doc(d, common_objset, "common-package", deploydir)
+}
+
+do_create_package_spdx[vardepsexclude] += "OVERRIDES SSTATE_ARCHS"
+
+addtask do_create_package_spdx after do_create_spdx before do_build do_rm_work
+SSTATETASKS += "do_create_package_spdx"
+do_create_package_spdx[sstate-inputdirs] = "${SPDXRUNTIMEDEPLOY}"
+do_create_package_spdx[sstate-outputdirs] = "${DEPLOY_DIR_SPDX}"
+
+python do_create_package_spdx_setscene () {
+    sstate_setscene(d)
+}
+addtask do_create_package_spdx_setscene
+
+do_create_package_spdx[dirs] = "${SPDXRUNTIMEDEPLOY}"
+do_create_package_spdx[cleandirs] = "${SPDXRUNTIMEDEPLOY}"
+do_create_package_spdx[rdeptask] = "do_create_spdx"
+
+do_populate_sdk[recrdeptask] += "do_create_spdx do_create_package_spdx"
+do_populate_sdk[cleandirs] += "${SPDXSDKWORK}"
+do_populate_sdk[postfuncs] += "sdk_create_sbom"
+POPULATE_SDK_POST_HOST_COMMAND:append:task-populate-sdk = " sdk_host_create_spdx"
+POPULATE_SDK_POST_TARGET_COMMAND:append:task-populate-sdk = " sdk_target_create_spdx"
+
+do_populate_sdk_ext[recrdeptask] += "do_create_spdx do_create_package_spdx"
+do_populate_sdk_ext[cleandirs] += "${SPDXSDKEXTWORK}"
+do_populate_sdk_ext[postfuncs] += "sdk_ext_create_sbom"
+POPULATE_SDK_POST_HOST_COMMAND:append:task-populate-sdk-ext = " sdk_ext_host_create_spdx"
+POPULATE_SDK_POST_TARGET_COMMAND:append:task-populate-sdk-ext = " sdk_ext_target_create_spdx"
+
+python sdk_host_create_spdx() {
+    from pathlib import Path
+    spdx_work_dir = Path(d.getVar('SPDXSDKWORK'))
+
+    sdk_create_spdx(d, "host", spdx_work_dir, d.getVar("TOOLCHAIN_OUTPUTNAME"))
+}
+
+python sdk_target_create_spdx() {
+    from pathlib import Path
+    spdx_work_dir = Path(d.getVar('SPDXSDKWORK'))
+
+    sdk_create_spdx(d, "target", spdx_work_dir, d.getVar("TOOLCHAIN_OUTPUTNAME"))
+}
+
+python sdk_ext_host_create_spdx() {
+    from pathlib import Path
+    spdx_work_dir = Path(d.getVar('SPDXSDKEXTWORK'))
+
+    # TODO: This doesn't seem to work
+    sdk_create_spdx(d, "host", spdx_work_dir, d.getVar("TOOLCHAINEXT_OUTPUTNAME"))
+}
+
+python sdk_ext_target_create_spdx() {
+    from pathlib import Path
+    spdx_work_dir = Path(d.getVar('SPDXSDKEXTWORK'))
+
+    # TODO: This doesn't seem to work
+    sdk_create_spdx(d, "target", spdx_work_dir, d.getVar("TOOLCHAINEXT_OUTPUTNAME"))
+}
+
+def sdk_create_spdx(d, sdk_type, spdx_work_dir, toolchain_outputname):
+    from pathlib import Path
+    from oe.sdk import sdk_list_installed_packages
+    import oe.spdx30
+    import oe.sbom30
+    from datetime import datetime
+
+    sdk_name = toolchain_outputname + "-" + sdk_type
+    sdk_packages = sdk_list_installed_packages(d, sdk_type == "target")
+
+    objset = oe.sbom30.ObjectSet.new_objset(d, sdk_name)
+
+    sdk_rootfs = objset.add_root(oe.spdx30.software_Package(
+        _id=objset.new_spdxid("sdk-rootfs", sdk_name),
+        creationInfo=objset.doc.creationInfo,
+        name=sdk_name,
+        software_primaryPurpose=oe.spdx30.software_SoftwarePurpose.archive,
+        builtTime=oe.sbom30.spdx_now(),
+    ))
+
+    sdk_build = objset.add_root(objset.new_sub_build("sdk-rootfs", "sdk-rootfs"))
+    sdk_build.build_buildEndTime = oe.sbom30.spdx_now()
+
+    objset.new_scoped_relationship(
+        [sdk_build],
+        oe.spdx30.RelationshipType.hasOutputs,
+        oe.spdx30.LifecycleScopeType.build,
+        [sdk_rootfs],
+    )
+
+    collect_build_package_inputs(d, objset, sdk_build, sdk_packages)
+
+    objset.add_aliases()
+    oe.sbom30.write_jsonld_doc(d, objset, spdx_work_dir / "sdk-rootfs.spdx.json")
+
+python sdk_create_sbom() {
+    from pathlib import Path
+    sdk_deploydir = Path(d.getVar("SDKDEPLOYDIR"))
+    spdx_work_dir = Path(d.getVar('SPDXSDKWORK'))
+
+    create_sdk_sbom(d, sdk_deploydir, spdx_work_dir, d.getVar("TOOLCHAIN_OUTPUTNAME"))
+}
+
+python sdk_ext_create_sbom() {
+    from pathlib import Path
+    sdk_deploydir = Path(d.getVar("SDKEXTDEPLOYDIR"))
+    spdx_work_dir = Path(d.getVar('SPDXSDKEXTWORK'))
+
+    create_sdk_sbom(d, sdk_deploydir, spdx_work_dir, d.getVar("TOOLCHAINEXT_OUTPUTNAME"))
+}
+
+def create_sdk_sbom(d, sdk_deploydir, spdx_work_dir, toolchain_outputname):
+    import oe.spdx30
+    import oe.sbom30
+    from pathlib import Path
+    from datetime import datetime
+
+    now = oe.sbom30.spdx_now()
+
+    # Load the document written earlier
+    rootfs_objset = oe.sbom30.load_jsonld(d, spdx_work_dir / "sdk-rootfs.spdx.json", required=True)
+
+    # Create a new build for the SDK installer
+    sdk_build = rootfs_objset.new_sub_build("sdk-populate", "sdk-populate")
+    sdk_build.build_buildEndTime = oe.sbom30.spdx_now()
+
+    rootfs = rootfs_objset.find_root(oe.spdx30.software_Package)
+    if rootfs is None:
+        bb.fatal("Unable to find rootfs artifact")
+
+    rootfs_objset.new_scoped_relationship(
+        [sdk_build],
+        oe.spdx30.RelationshipType.hasInputs,
+        oe.spdx30.LifecycleScopeType.build,
+        [rootfs]
+    )
+
+    files = set()
+    root_files = []
+
+    # NOTE: os.walk() doesn't return symlinks
+    for dirpath, dirnames, filenames in os.walk(sdk_deploydir):
+        for fn in filenames:
+            fpath = Path(dirpath) / fn
+            if not fpath.is_file() or fpath.is_symlink():
+                continue
+
+            relpath = str(fpath.relative_to(sdk_deploydir))
+
+            f = rootfs_objset.new_file(
+                rootfs_objset.new_spdxid("sdk-installer", relpath),
+                relpath,
+                fpath,
+            )
+            f.builtTime = now
+
+            if fn.endswith(".manifest"):
+                f.software_primaryPurpose = oe.spdx30.software_SoftwarePurpose.manifest
+            elif fn.endswith(".testdata.json"):
+                f.software_primaryPurpose = oe.spdx30.software_SoftwarePurpose.configuration
+            else:
+                set_purposes(d, f, "SPDX_SDK_PURPOSE")
+                root_files.append(f)
+
+            files.add(f)
+
+    if files:
+        rootfs_objset.new_scoped_relationship(
+            [sdk_build],
+            oe.spdx30.RelationshipType.hasOutputs,
+            oe.spdx30.LifecycleScopeType.build,
+            files,
+        )
+    else:
+        bb.warn(f"No SDK output files found in {sdk_deploydir}")
+
+    objset, sbom = oe.sbom30.create_sbom(d, toolchain_outputname, sorted(list(files)), [rootfs_objset])
+
+    oe.sbom30.write_jsonld_doc(d, objset, sdk_deploydir / (toolchain_outputname + ".spdx.json"))
+
+python spdx30_build_started_handler () {
+    import oe.spdx30
+    import oe.sbom30
+    import os
+    from pathlib import Path
+
+    # Create a copy of the datastore. Set PN to "bitbake" so that SPDX IDs can
+    # be generated
+    d = e.data.createCopy()
+    d.setVar("PN", "bitbake")
+    d.setVar("BB_TASKHASH", "bitbake")
+    load_spdx_license_data(d)
+
+    deploy_dir_spdx = Path(e.data.getVar("DEPLOY_DIR_SPDX"))
+
+    nonce = os.urandom(16).hex()
+
+    objset = oe.sbom30.ObjectSet.new_objset(d, "bitbake", False)
+
+    build = objset.add_root(oe.spdx30.build_Build(
+        _id=objset.new_spdxid(nonce, include_unihash=False),
+        creationInfo=objset.doc.creationInfo,
+        build_buildType=oe.sbom30.SPDX_BUILD_TYPE,
+        build_buildStartTime=oe.sbom30.spdx_now()
+    ))
+
+    host_import_key = d.getVar("SPDX_BUILD_HOST")
+    if host_import_key:
+        objset.new_scoped_relationship(
+            [build],
+            oe.spdx30.RelationshipType.hasHost,
+            oe.spdx30.LifecycleScopeType.build,
+            [objset.new_import("SPDX_BUILD_HOST")],
+        )
+
+    invoked_by = objset.new_agent("SPDX_INVOKED_BY")
+    if invoked_by:
+        invoked_by_spdx = objset.new_scoped_relationship(
+            [build],
+            oe.spdx30.RelationshipType.invokedBy,
+            oe.spdx30.LifecycleScopeType.build,
+            [invoked_by],
+        )
+
+        on_behalf_of = objset.new_agent("SPDX_ON_BEHALF_OF")
+        if on_behalf_of:
+            objset.new_scoped_relationship(
+                [on_behalf_of],
+                oe.spdx30.RelationshipType.delegatedTo,
+                oe.spdx30.LifecycleScopeType.build,
+                invoked_by_spdx,
+            )
+
+    for obj in objset.foreach_type(oe.spdx30.Element):
+        obj.extension.append(oe.sbom30.OELinkExtension(link_spdx_id=False))
+        obj.extension.append(oe.sbom30.OEIdAliasExtension())
+
+    oe.sbom30.write_jsonld_doc(d, objset, deploy_dir_spdx / "bitbake.spdx.json")
+}
+
+addhandler spdx30_build_started_handler
+spdx30_build_started_handler[eventmask] = "bb.event.ConfigParsed"
+
meta/classes/create-spdx-image-3.0.bbclass new file +224 -0
new file mode 100644
index 00000000000..31e6f25d29a
@@ -0,0 +1,224 @@
+#
+# Copyright OpenEmbedded Contributors
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# SPDX image tasks
+
+SPDX_ROOTFS_PACKAGES = "${SPDXDIR}/rootfs-packages.json"
+SPDXIMAGEDEPLOYDIR = "${SPDXDIR}/image-deploy"
+SPDXROOTFSDEPLOY = "${SPDXDIR}/rootfs-deploy"
+
+python spdx_collect_rootfs_packages() {
+    import json
+    from pathlib import Path
+    from oe.rootfs import image_list_installed_packages
+
+    root_packages_file = Path(d.getVar("SPDX_ROOTFS_PACKAGES"))
+
+    packages = image_list_installed_packages(d)
+    if not packages:
+        packages = {}
+
+    root_packages_file.parent.mkdir(parents=True, exist_ok=True)
+    with root_packages_file.open("w") as f:
+        json.dump(packages, f)
+}
+ROOTFS_POSTUNINSTALL_COMMAND =+ "spdx_collect_rootfs_packages"
+
+python do_create_rootfs_spdx() {
+    import json
+    from pathlib import Path
+    import oe.spdx30
+    import oe.sbom30
+    from datetime import datetime
+
+    deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX"))
+    deploydir = Path(d.getVar("SPDXROOTFSDEPLOY"))
+    root_packages_file = Path(d.getVar("SPDX_ROOTFS_PACKAGES"))
+    image_basename = d.getVar("IMAGE_BASENAME")
+    machine = d.getVar("MACHINE")
+
+    with root_packages_file.open("r") as f:
+        packages = json.load(f)
+
+    objset = oe.sbom30.ObjectSet.new_objset(d, "%s-%s" % (image_basename, machine))
+
+    rootfs = objset.add_root(oe.spdx30.software_Package(
+        _id=objset.new_spdxid("rootfs", image_basename),
+        creationInfo=objset.doc.creationInfo,
+        name=image_basename,
+        software_primaryPurpose=oe.spdx30.software_SoftwarePurpose.archive,
+        builtTime=oe.sbom30.spdx_now()
+    ))
+
+    rootfs_build = objset.add_root(objset.new_sub_build("rootfs", "rootfs"))
+    rootfs_build.build_buildEndTime = oe.sbom30.spdx_now()
+
+    objset.new_scoped_relationship(
+        [rootfs_build],
+        oe.spdx30.RelationshipType.hasOutputs,
+        oe.spdx30.LifecycleScopeType.build,
+        [rootfs],
+    )
+
+    collect_build_package_inputs(d, objset, rootfs_build, packages)
+
+    oe.sbom30.write_recipe_jsonld_doc(d, objset, "rootfs", deploydir)
+}
+addtask do_create_rootfs_spdx after do_rootfs before do_image
+SSTATETASKS += "do_create_rootfs_spdx"
+do_create_rootfs_spdx[sstate-inputdirs] = "${SPDXROOTFSDEPLOY}"
+do_create_rootfs_spdx[sstate-outputdirs] = "${DEPLOY_DIR_SPDX}"
+do_create_rootfs_spdx[recrdeptask] += "do_create_spdx do_create_package_spdx"
+do_create_rootfs_spdx[cleandirs] += "${SPDXROOTFSDEPLOY}"
+
+python do_create_rootfs_spdx_setscene() {
+    sstate_setscene(d)
+}
+addtask do_create_rootfs_spdx_setscene
+
+python do_create_image_spdx() {
+    import oe.spdx30
+    import oe.sbom30
+    import json
+    from pathlib import Path
+
+    image_deploy_dir = Path(d.getVar('IMGDEPLOYDIR'))
+    manifest_path = Path(d.getVar("IMAGE_FILE_MANIFEST"))
+    spdx_work_dir = Path(d.getVar('SPDXIMAGEWORK'))
+
+    image_basename = d.getVar('IMAGE_BASENAME')
+    machine = d.getVar("MACHINE")
+
+    objset = oe.sbom30.ObjectSet.new_objset(d, "%s-%s" % (image_basename, machine))
+
+    with manifest_path.open("r") as f:
+        manifest = json.load(f)
+
+    builds = []
+    for task in manifest:
+        imagetype = task["imagetype"]
+        taskname = task["taskname"]
+
+        image_build = objset.add_root(objset.new_sub_build(taskname, "image/%s" % imagetype))
+        image_build.build_buildEndTime = oe.sbom30.spdx_now()
+        builds.append(image_build)
+
+        artifacts = []
+
+        for image in task["images"]:
+            image_filename = image["filename"]
+            image_path = image_deploy_dir / image_filename
+            a = objset.add_root(oe.spdx30.software_File(
+                _id=objset.new_spdxid("image", image_filename),
+                creationInfo=objset.doc.creationInfo,
+                name=image_filename,
+                builtTime=oe.sbom30.spdx_now(),
+                verifiedUsing=[
+                    oe.spdx30.Hash(
+                        algorithm=oe.spdx30.HashAlgorithm.sha256,
+                        hashValue=bb.utils.sha256_file(image_path),
+                    )
+                ]
+            ))
+            set_purposes(d, a, "SPDX_IMAGE_PURPOSE:%s" % imagetype, "SPDX_IMAGE_PURPOSE")
+
+            artifacts.append(a)
+
+        if artifacts:
+            objset.new_scoped_relationship(
+                [image_build],
+                oe.spdx30.RelationshipType.hasOutputs,
+                oe.spdx30.LifecycleScopeType.build,
+                artifacts,
+            )
+
+    if builds:
+        rootfs_image, _ = oe.sbom30.find_root_obj_in_jsonld(
+            d,
+            "rootfs",
+            "%s-%s" % (image_basename, machine),
+            oe.spdx30.software_Package,
+            # TODO: Should use a purpose to filter here?
+        )
+        objset.new_scoped_relationship(
+            builds,
+            oe.spdx30.RelationshipType.hasInputs,
+            oe.spdx30.LifecycleScopeType.build,
+            [rootfs_image._id],
+        )
+
+    objset.add_aliases()
+    objset.link()
+    oe.sbom30.write_recipe_jsonld_doc(d, objset, "image", spdx_work_dir)
+}
+addtask do_create_image_spdx after do_image_complete do_create_rootfs_spdx before do_build
+SSTATETASKS += "do_create_image_spdx"
+SSTATE_SKIP_CREATION:task-combine-image-type-spdx = "1"
+do_create_image_spdx[sstate-inputdirs] = "${SPDXIMAGEWORK}"
+do_create_image_spdx[sstate-outputdirs] = "${DEPLOY_DIR_SPDX}"
+do_create_image_spdx[cleandirs] = "${SPDXIMAGEWORK}"
+do_create_image_spdx[dirs] = "${SPDXIMAGEWORK}"
+
+python do_create_image_spdx_setscene() {
+    sstate_setscene(d)
+}
+addtask do_create_image_spdx_setscene
+
+
+python do_create_image_sbom() {
+    import os
+    from pathlib import Path
+    import oe.spdx30
+    import oe.sbom30
+
+    image_name = d.getVar("IMAGE_NAME")
+    image_basename = d.getVar("IMAGE_BASENAME")
+    image_link_name = d.getVar("IMAGE_LINK_NAME")
+    imgdeploydir = Path(d.getVar("SPDXIMAGEDEPLOYDIR"))
+    machine = d.getVar("MACHINE")
+
+    spdx_path = imgdeploydir / (image_name + ".spdx.json")
+
+    root_elements = []
+
+    # TODO: Do we need to add the rootfs or are the image files sufficient?
+    rootfs_image, _ = oe.sbom30.find_root_obj_in_jsonld(
+        d,
+        "rootfs",
+        "%s-%s" % (image_basename, machine),
+        oe.spdx30.software_Package,
+        # TODO: Should use a purpose here?
+    )
+    root_elements.append(rootfs_image._id)
+
+    image_objset, _ = oe.sbom30.find_jsonld(d, "image", "%s-%s" % (image_basename, machine), required=True)
+    for o in image_objset.foreach_root(oe.spdx30.software_File):
+        root_elements.append(o._id)
+
+    objset, sbom = oe.sbom30.create_sbom(d, image_name, root_elements)
+
+    oe.sbom30.write_jsonld_doc(d, objset, spdx_path)
+
+    def make_image_link(target_path, suffix):
+        if image_link_name:
+            link = imgdeploydir / (image_link_name + suffix)
+            if link != target_path:
+                link.symlink_to(os.path.relpath(target_path, link.parent))
+
+    make_image_link(spdx_path, ".spdx.json")
+}
+addtask do_create_image_sbom after do_create_rootfs_spdx do_create_image_spdx before do_build
+SSTATETASKS += "do_create_image_sbom"
+SSTATE_SKIP_CREATION:task-create-image-sbom = "1"
+do_create_image_sbom[sstate-inputdirs] = "${SPDXIMAGEDEPLOYDIR}"
+do_create_image_sbom[sstate-outputdirs] = "${DEPLOY_DIR_IMAGE}"
+do_create_image_sbom[stamp-extra-info] = "${MACHINE_ARCH}"
+do_create_image_sbom[cleandirs] = "${SPDXIMAGEDEPLOYDIR}"
+do_create_image_sbom[recrdeptask] += "do_create_spdx do_create_package_spdx"
+
+python do_create_image_sbom_setscene() {
+    sstate_setscene(d)
+}
+addtask do_create_image_sbom_setscene
meta/classes/spdx-common.bbclass +5 -1
index 03f1d0cc278..9dee325ba6b 100644
@@ -17,6 +17,7 @@ SPDXDEPLOY = "${SPDXDIR}/deploy"
  SPDXWORK = "${SPDXDIR}/work"
  SPDXIMAGEWORK = "${SPDXDIR}/image-work"
  SPDXSDKWORK = "${SPDXDIR}/sdk-work"
+SPDXSDKEXTWORK = "${SPDXDIR}/sdk-ext-work"
  SPDXDEPS = "${SPDXDIR}/deps.json"
 
  SPDX_TOOL_NAME ??= "oe-spdx-creator"
@@ -68,7 +69,7 @@ def get_json_indent(d):
          return 2
      return None
 
-python() {
+def load_spdx_license_data(d):
      import json
      if d.getVar("SPDX_LICENSE_DATA"):
          return
@@ -78,6 +79,9 @@ python() {
          # Transform the license array to a dictionary
          data["licenses"] = {l["licenseId"]: l for l in data["licenses"]}
          d.setVar("SPDX_LICENSE_DATA", data)
+
+python() {
+    load_spdx_license_data(d)
  }
 
  def process_sources(d):
meta/lib/oe/sbom30.py new file +1122 -0
new file mode 100644
index 00000000000..2180438ce4c
@@ -0,0 +1,1122 @@
+#
+# Copyright OpenEmbedded Contributors
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+
+from pathlib import Path
+
+import oe.spdx30
+import bb
+import re
+import hashlib
+import uuid
+import os
+from datetime import datetime, timezone
+
+OE_SPDX_BASE = "https://rdf.openembedded.org/spdx/3.0/"
+
+VEX_VERSION = "1.0.0"
+
+SPDX_BUILD_TYPE = "http://openembedded.org/bitbake"
+
+
[email protected](OE_SPDX_BASE + "link-extension")
+class OELinkExtension(oe.spdx30.extension_Extension):
+    """
+    This custom extension controls if an Element creates a symlink based on
+    its SPDX ID in the deploy directory. Some elements may not be able to be
+    linked because they are duplicated in multiple documents (e.g. the bitbake
+    Build Element). Those elements can add this extension and set link_spdx_id
+    to False
+
+    It is in internal extension that should be removed when writing out a final
+    SBoM
+    """
+
+    CLOSED = True
+    INTERNAL = True
+
+    @classmethod
+    def _register_props(cls):
+        super()._register_props()
+        cls._add_property(
+            "link_spdx_id",
+            oe.spdx30.BooleanProp(),
+            OE_SPDX_BASE + "link-spdx-id",
+            min_count=1,
+            max_count=1,
+        )
+
+        # The symlinks written to the deploy directory are based on the hash of
+        # the SPDX ID. While this makes it easy to look them up, it can be
+        # difficult to trace a Element to the hashed symlink name. As a
+        # debugging aid, this property is set to the basename of the symlink
+        # when the symlink is created to make it easier to trace
+        cls._add_property(
+            "link_name",
+            oe.spdx30.StringProp(),
+            OE_SPDX_BASE + "link-name",
+            max_count=1,
+        )
+
+
[email protected](OE_SPDX_BASE + "id-alias")
+class OEIdAliasExtension(oe.spdx30.extension_Extension):
+    """
+    This extension allows an Element to provide an internal alias for the SPDX
+    ID. Since SPDX requires unique URIs for each SPDX ID, most of the objects
+    created have a unique UUID namespace and the unihash of the task encoded in
+    their SPDX ID. However, this causes a problem for referencing documents
+    across recipes, since the taskhash of a dependency may not factor into the
+    taskhash of the current task and thus the current task won't rebuild and
+    see the new SPDX ID when the dependency changes (e.g. ABI safe recipes and
+    tasks).
+
+    To help work around this, this extension provides a non-unique alias for an
+    Element by which it can be referenced from other tasks/recipes. When a
+    final SBoM is created, references to these aliases will be replaced with
+    the actual unique SPDX ID.
+
+    Most Elements will automatically get an alias created when they are written
+    out if they do not already have one. To suppress the creation of an alias,
+    add an extension with a blank `alias` property.
+
+
+    It is in internal extension that should be removed when writing out a final
+    SBoM
+    """
+
+    CLOSED = True
+    INTERNAL = True
+
+    @classmethod
+    def _register_props(cls):
+        super()._register_props()
+        cls._add_property(
+            "alias",
+            oe.spdx30.StringProp(),
+            OE_SPDX_BASE + "alias",
+            max_count=1,
+        )
+
+        cls._add_property(
+            "link_name",
+            oe.spdx30.StringProp(),
+            OE_SPDX_BASE + "link-name",
+            max_count=1,
+        )
+
+
[email protected](OE_SPDX_BASE + "file-name-alias")
+class OEFileNameAliasExtension(oe.spdx30.extension_Extension):
+    CLOSED = True
+    INTERNAL = True
+
+    @classmethod
+    def _register_props(cls):
+        super()._register_props()
+        cls._add_property(
+            "aliases",
+            oe.spdx30.ListProp(oe.spdx30.StringProp()),
+            OE_SPDX_BASE + "filename-alias",
+        )
+
+
[email protected](OE_SPDX_BASE + "license-scanned")
+class OELicenseScannedExtension(oe.spdx30.extension_Extension):
+    """
+    The presence of this extension means the file has already been scanned for
+    license information
+    """
+
+    CLOSED = True
+    INTERNAL = True
+
+
[email protected](OE_SPDX_BASE + "document-extension")
+class OEDocumentExtension(oe.spdx30.extension_Extension):
+    """
+    This extension is added to a SpdxDocument to indicate various useful bits
+    of information about its contents
+    """
+
+    CLOSED = True
+
+    @classmethod
+    def _register_props(cls):
+        super()._register_props()
+        cls._add_property(
+            "is_native",
+            oe.spdx30.BooleanProp(),
+            OE_SPDX_BASE + "is-native",
+            max_count=1,
+        )
+
+
+def spdxid_hash(*items):
+    h = hashlib.md5()
+    for i in items:
+        if isinstance(i, oe.spdx30.Element):
+            h.update(i._id.encode("utf-8"))
+        else:
+            h.update(i.encode("utf-8"))
+    return h.hexdigest()
+
+
+def spdx_now():
+    return datetime.now(timezone.utc)
+
+
+def get_element_link_id(e):
+    """
+    Get the string ID which should be used to link to an Element. If the
+    element has an alias, that will be preferred, otherwise its SPDX ID will be
+    used.
+    """
+    ext = get_alias(e)
+    if ext is not None and ext.alias:
+        return ext.alias
+    return e._id
+
+
+def set_alias(obj, alias):
+    for ext in obj.extension:
+        if not isinstance(ext, OEIdAliasExtension):
+            continue
+        ext.alias = alias
+        return ext
+
+    ext = OEIdAliasExtension(alias=alias)
+    obj.extension.append(ext)
+    return ext
+
+
+def get_alias(obj):
+    for ext in obj.extension:
+        if not isinstance(ext, OEIdAliasExtension):
+            continue
+        return ext
+
+    return None
+
+
+def extract_licenses(filename):
+    lic_regex = re.compile(
+        rb"^\W*SPDX-License-Identifier:\s*([ \w\d.()+-]+?)(?:\s+\W*)?$", re.MULTILINE
+    )
+
+    try:
+        with open(filename, "rb") as f:
+            size = min(15000, os.stat(filename).st_size)
+            txt = f.read(size)
+            licenses = re.findall(lic_regex, txt)
+            if licenses:
+                ascii_licenses = [lic.decode("ascii") for lic in licenses]
+                return ascii_licenses
+    except Exception as e:
+        bb.warn(f"Exception reading {filename}: {e}")
+    return []
+
+
+def to_list(l):
+    if isinstance(l, set):
+        l = sorted(list(l))
+
+    if not isinstance(l, (list, tuple)):
+        raise TypeError("Must be a list or tuple. Got %s" % type(l))
+
+    return l
+
+
+class ObjectSet(oe.spdx30.SHACLObjectSet):
+    def __init__(self, d):
+        super().__init__()
+        self.d = d
+
+    def create_index(self):
+        self.by_sha256_hash = {}
+        super().create_index()
+
+    def add_index(self, obj):
+        # Check that all elements are given an ID before being inserted
+        if isinstance(obj, oe.spdx30.Element):
+            if not obj._id:
+                raise ValueError("Element missing ID")
+            for ext in obj.extension:
+                if not isinstance(ext, OEIdAliasExtension):
+                    continue
+                if ext.alias:
+                    self.obj_by_id[ext.alias] = obj
+
+            for v in obj.verifiedUsing:
+                if not isinstance(v, oe.spdx30.Hash):
+                    continue
+
+                if v.algorithm == oe.spdx30.HashAlgorithm.sha256:
+                    continue
+
+                self.by_sha256_hash.setdefault(v.hashValue, set()).add(obj)
+
+        super().add_index(obj)
+        if isinstance(obj, oe.spdx30.SpdxDocument):
+            self.doc = obj
+
+    def __filter_obj(self, obj, attr_filter):
+        return all(getattr(obj, k) == v for k, v in attr_filter.items())
+
+    def foreach_filter(self, typ, *, match_subclass=True, **attr_filter):
+        for obj in self.foreach_type(typ, match_subclass=match_subclass):
+            if self.__filter_obj(obj, attr_filter):
+                yield obj
+
+    def find_filter(self, typ, *, match_subclass=True, **attr_filter):
+        for obj in self.foreach_filter(
+            typ, match_subclass=match_subclass, **attr_filter
+        ):
+            return obj
+        return None
+
+    def foreach_root(self, typ, **attr_filter):
+        for obj in self.doc.rootElement:
+            if not isinstance(obj, typ):
+                continue
+
+            if self.__filter_obj(obj, attr_filter):
+                yield obj
+
+    def find_root(self, typ, **attr_filter):
+        for obj in self.foreach_root(typ, **attr_filter):
+            return obj
+        return None
+
+    def add_root(self, obj):
+        self.add(obj)
+        self.doc.rootElement.append(obj)
+        return obj
+
+    def is_native(self):
+        for e in self.doc.extension:
+            if not isinstance(e, oe.sbom30.OEDocumentExtension):
+                continue
+
+            if e.is_native is not None:
+                return e.is_native
+
+        return False
+
+    def set_is_native(self, is_native):
+        for e in self.doc.extension:
+            if not isinstance(e, oe.sbom30.OEDocumentExtension):
+                continue
+
+            e.is_native = is_native
+            return
+
+        if is_native:
+            self.doc.extension.append(oe.sbom30.OEDocumentExtension(is_native=True))
+
+    def add_aliases(self):
+        for o in self.foreach_type(oe.spdx30.Element):
+            if not o._id or o._id.startswith("_:"):
+                continue
+
+            alias_ext = get_alias(o)
+            if alias_ext is None:
+                unihash = self.d.getVar("BB_UNIHASH")
+                namespace = self.get_namespace()
+                if unihash not in o._id:
+                    bb.warn(f"Unihash {unihash} not found in {o._id}")
+                elif namespace not in o._id:
+                    bb.warn(f"Namespace {namespace} not found in {o._id}")
+                else:
+                    alias_ext = set_alias(
+                        o,
+                        o._id.replace(unihash, "UNIHASH").replace(
+                            namespace, self.d.getVar("PN")
+                        ),
+                    )
+
+    def remove_internal_extensions(self):
+        def remove(o):
+            o.extension = [e for e in o.extension if not getattr(e, "INTERNAL", False)]
+
+        for o in self.foreach_type(oe.spdx30.Element):
+            remove(o)
+
+        if self.doc:
+            remove(self.doc)
+
+    def get_namespace(self):
+        namespace_uuid = uuid.uuid5(
+            uuid.NAMESPACE_DNS, self.d.getVar("SPDX_UUID_NAMESPACE")
+        )
+        pn = self.d.getVar("PN")
+        return "%s/%s-%s" % (
+            self.d.getVar("SPDX_NAMESPACE_PREFIX"),
+            pn,
+            str(uuid.uuid5(namespace_uuid, pn)),
+        )
+
+    def new_spdxid(self, *suffix, include_unihash=True):
+        items = [self.get_namespace()]
+        if include_unihash:
+            unihash = self.d.getVar("BB_UNIHASH")
+            items.append(unihash)
+        items.extend(re.sub(r"[^a-zA-Z0-9_-]", "_", s) for s in suffix)
+        return "/".join(items)
+
+    def new_import(self, key):
+        base = f"SPDX_IMPORTS_{key}"
+        spdxid = self.d.getVar(f"{base}_spdxid")
+        if not spdxid:
+            bb.fatal(f"{key} is not a valid SPDX_IMPORTS key")
+
+        for i in self.docs.imports:
+            if i.externalSpdxId == spdxid:
+                # Already imported
+                return spdxid
+
+        m = oe.spdx30.ExternalMap(externalSpdxId=spdxid)
+
+        uri = self.d.getVar(f"{base}_uri")
+        if uri:
+            m.locationHint = uri
+
+        for pyname, algorithm in oe.spdx30.HashAlgorithm.NAMED_INDIVIDUALS.items():
+            value = self.d.getVar(f"{base}_hash_{pyname}")
+            if value:
+                m.verifiedUsing.append(
+                    oe.spdx30.Hash(
+                        algorithm=algorithm,
+                        hashValue=value,
+                    )
+                )
+
+        self.doc.imports.append(m)
+        return spdxid
+
+    def new_agent(self, varname, *, creation_info=None):
+        ref_varname = self.d.getVar(f"{varname}_ref")
+        if ref_varname:
+            if ref_varname == varname:
+                bb.fatal(f"{varname} cannot reference itself")
+            return new_agent(varname, creation_info=creation_info)
+
+        import_key = self.d.getVar(f"{varname}_import")
+        if import_key:
+            return self.new_import(import_key)
+
+        name = self.d.getVar(f"{varname}_name")
+        if not name:
+            return None
+
+        spdxid = self.new_spdxid("agent", name)
+        agent = self.find_by_id(spdxid)
+        if agent is not None:
+            return agent
+
+        agent_type = self.d.getVar("%s_type" % varname)
+        if agent_type == "person":
+            agent = oe.spdx30.Person()
+        elif agent_type == "software":
+            agent = oe.spdx30.SoftwareAgent()
+        elif agent_type == "organization":
+            agent = oe.spdx30.Organization()
+        elif not agent_type or agent_type == "agent":
+            agent = oe.spdx30.Agent()
+        else:
+            bb.fatal("Unknown agent type '%s' in %s_type" % (agent_type, varname))
+
+        agent._id = spdxid
+        agent.creationInfo = creation_info or self.doc.creationInfo
+        agent.name = name
+
+        comment = self.d.getVar("%s_comment" % varname)
+        if comment:
+            agent.comment = comment
+
+        for (
+            pyname,
+            idtype,
+        ) in oe.spdx30.ExternalIdentifierType.NAMED_INDIVIDUALS.items():
+            value = self.d.getVar("%s_id_%s" % (varname, pyname))
+            if value:
+                agent.externalIdentifier.append(
+                    oe.spdx30.ExternalIdentifier(
+                        externalIdentifierType=idtype,
+                        identifier=value,
+                    )
+                )
+
+        return self.add(agent)
+
+    def new_creation_info(self):
+        creation_info = oe.spdx30.CreationInfo()
+
+        name = "%s %s" % (
+            self.d.getVar("SPDX_TOOL_NAME"),
+            self.d.getVar("SPDX_TOOL_VERSION"),
+        )
+        tool = self.add(
+            oe.spdx30.Tool(
+                _id=self.new_spdxid("tool", name),
+                creationInfo=creation_info,
+                name=name,
+            )
+        )
+
+        authors = []
+        for a in self.d.getVar("SPDX_AUTHORS").split():
+            varname = "SPDX_AUTHORS_%s" % a
+            author = self.new_agent(varname, creation_info=creation_info)
+
+            if not author:
+                bb.fatal("Unable to find or create author %s" % a)
+
+            authors.append(author)
+
+        creation_info.created = spdx_now()
+        creation_info.specVersion = self.d.getVar("SPDX_VERSION")
+        creation_info.createdBy = authors
+        creation_info.createdUsing = [tool]
+
+        return creation_info
+
+    def copy_creation_info(self, copy):
+        c = oe.spdx30.CreationInfo(
+            created=spdx_now(),
+            specVersion=self.d.getVar("SPDX_VERSION"),
+        )
+
+        for author in copy.createdBy:
+            if isinstance(author, str):
+                c.createdBy.append(author)
+            else:
+                c.createdBy.append(author._id)
+
+        for tool in copy.createdUsing:
+            if isinstance(tool, str):
+                c.createdUsing.append(tool)
+            else:
+                c.createdUsing.append(tool._id)
+
+        return c
+
+    def new_annotation(self, subject, comment, typ):
+        return self.add(
+            oe.spdx30.Annotation(
+                _id=self.new_spdxid("annotation", spdxid_hash(comment, typ)),
+                creationInfo=self.doc.creationInfo,
+                annotationType=typ,
+                subject=subject,
+                statement=comment,
+            )
+        )
+
+    def _new_relationship(
+        self,
+        cls,
+        from_,
+        typ,
+        to,
+        *,
+        spdxid_name="relationship",
+        **props,
+    ):
+        from_ = to_list(from_)
+        to = to_list(to)
+
+        if not from_:
+            return []
+
+        if not to:
+            # TODO: Switch to the code constant once SPDX 3.0.1 is released
+            to = ["https://spdx.org/rdf/3.0.0/terms/Core/NoneElement"]
+
+        ret = []
+
+        for f in from_:
+            hash_args = [typ, f]
+            for k in sorted(props.keys()):
+                hash_args.append(props[k])
+            hash_args.extend(to)
+
+            relationship = self.add(
+                cls(
+                    _id=self.new_spdxid(spdxid_name, spdxid_hash(*hash_args)),
+                    creationInfo=self.doc.creationInfo,
+                    from_=f,
+                    relationshipType=typ,
+                    to=to,
+                    **props,
+                )
+            )
+            ret.append(relationship)
+
+        return ret
+
+    def new_relationship(self, from_, typ, to):
+        return self._new_relationship(oe.spdx30.Relationship, from_, typ, to)
+
+    def new_scoped_relationship(self, from_, typ, scope, to):
+        return self._new_relationship(
+            oe.spdx30.LifecycleScopedRelationship,
+            from_,
+            typ,
+            to,
+            scope=scope,
+        )
+
+    def new_license_expression(self, license_expression, license_text_map={}):
+        license_list_version = self.d.getVar("SPDX_LICENSE_DATA")["licenseListVersion"]
+        # SPDX 3 requires that the license list version be a semver
+        # MAJOR.MINOR.MICRO, but the actual license version is only MAJOR.MINOR
+        # (as of this writing). As such, manually append a .0 micro version if
+        # its missing to keep SPDX happy
+        if license_list_version.count(".") < 3:
+            license_list_version += ".0"
+
+        spdxid = [
+            "license",
+            license_list_version,
+            re.sub(r"[^a-zA-Z0-9_-]", "_", license_expression),
+        ]
+
+        license_text = (
+            (k, license_text_map[k]) for k in sorted(license_text_map.keys())
+        )
+
+        if not license_text:
+            lic = self.find_filter(
+                oe.spdx30.simplelicensing_LicenseExpression,
+                simplelicensing_licenseExpression=license_expression,
+                simplelicensing_licenseListVersion=license_list_version,
+            )
+            if lic is not None:
+                return lic
+        else:
+            spdxid.append(spdxid_hash(*(v for _, v in license_text)))
+            lic = self.find_by_id(self.new_spdxid(*spdxid))
+            if lic is not None:
+                return lic
+
+        lic = self.add(
+            oe.spdx30.simplelicensing_LicenseExpression(
+                _id=self.new_spdxid(*spdxid),
+                creationInfo=self.doc.creationInfo,
+                simplelicensing_licenseExpression=license_expression,
+                simplelicensing_licenseListVersion=license_list_version,
+            )
+        )
+
+        for key, value in license_text:
+            lic.simplelicensing_customIdToUri.append(
+                oe.spdx30.DictionaryEntry(key=key, value=value)
+            )
+
+        return lic
+
+    def scan_declared_licenses(self, spdx_file, filepath):
+        for e in spdx_file.extension:
+            if isinstance(e, OELicenseScannedExtension):
+                return
+
+        file_licenses = set()
+        for extracted_lic in extract_licenses(filepath):
+            file_licenses.add(self.new_license_expression(extracted_lic))
+
+        self.new_relationship(
+            [spdx_file],
+            oe.spdx30.RelationshipType.hasDeclaredLicense,
+            file_licenses,
+        )
+        spdx_file.extension.append(OELicenseScannedExtension())
+
+    def new_file(self, _id, name, path, *, purposes=[]):
+        sha256_hash = bb.utils.sha256_file(path)
+
+        for f in self.by_sha256_hash.get(sha256_hash, []):
+            if not isinstance(oe.spdx30.software_File):
+                continue
+
+            if purposes:
+                new_primary = purposes[0]
+                new_additional = []
+
+                if f.software_primaryPurpose:
+                    new_additional.append(f.software_primaryPurpose)
+                new_additional.extend(f.software_additionalPurpose)
+
+                new_additional = sorted(
+                    list(set(p for p in new_additional if p != new_primary))
+                )
+
+                f.software_primaryPurpose = new_primary
+                f.software_additionalPurpose = new_additional
+
+            if f.name != name:
+                for e in f.extension:
+                    if isinstance(e, OEFileNameAliasExtension):
+                        e.aliases.append(name)
+                        break
+                else:
+                    f.extension.append(OEFileNameAliasExtension(aliases=[name]))
+
+            return f
+
+        spdx_file = oe.spdx30.software_File(
+            _id=_id,
+            creationInfo=self.doc.creationInfo,
+            name=name,
+        )
+        if purposes:
+            spdx_file.software_primaryPurpose = purposes[0]
+            spdx_file.software_additionalPurpose = purposes[1:]
+
+        spdx_file.verifiedUsing.append(
+            oe.spdx30.Hash(
+                algorithm=oe.spdx30.HashAlgorithm.sha256,
+                hashValue=sha256_hash,
+            )
+        )
+
+        return self.add(spdx_file)
+
+    def new_cve_vuln(self, cve):
+        v = oe.spdx30.security_Vulnerability()
+        v._id = self.new_spdxid("vulnerability", cve)
+        v.creationInfo = self.doc.creationInfo
+
+        v.externalIdentifier.append(
+            oe.spdx30.ExternalIdentifier(
+                externalIdentifierType=oe.spdx30.ExternalIdentifierType.cve,
+                identifier=cve,
+                identifierLocator=[
+                    f"https://cveawg.mitre.org/api/cve/{cve}",
+                    f"https://www.cve.org/CVERecord?id={cve}",
+                ],
+            )
+        )
+        return self.add(v)
+
+    def new_vex_patched_relationship(self, from_, to):
+        return self._new_relationship(
+            oe.spdx30.security_VexFixedVulnAssessmentRelationship,
+            from_,
+            oe.spdx30.RelationshipType.fixedIn,
+            to,
+            spdxid_name="vex-fixed",
+            security_vexVersion=VEX_VERSION,
+        )
+
+    def new_vex_unpatched_relationship(self, from_, to):
+        return self._new_relationship(
+            oe.spdx30.security_VexAffectedVulnAssessmentRelationship,
+            from_,
+            oe.spdx30.RelationshipType.affects,
+            to,
+            spdxid_name="vex-affected",
+            security_vexVersion=VEX_VERSION,
+        )
+
+    def new_vex_ignored_relationship(self, from_, to, *, impact_statement):
+        return self._new_relationship(
+            oe.spdx30.security_VexNotAffectedVulnAssessmentRelationship,
+            from_,
+            oe.spdx30.RelationshipType.doesNotAffect,
+            to,
+            spdxid_name="vex-not-affected",
+            security_vexVersion=VEX_VERSION,
+            security_impactStatement=impact_statement,
+        )
+
+    def import_bitbake_build(self):
+        def find_bitbake_build(objset):
+            return objset.find_filter(
+                oe.spdx30.build_Build,
+                build_buildType=SPDX_BUILD_TYPE,
+            )
+
+        build = find_bitbake_build(self)
+        if build:
+            return build
+
+        deploy_dir_spdx = Path(self.d.getVar("DEPLOY_DIR_SPDX"))
+        bb_objset = load_jsonld(
+            self.d, deploy_dir_spdx / "bitbake.spdx.json", required=True
+        )
+
+        build = find_bitbake_build(bb_objset)
+        if build is None:
+            bb.fatal(f"No build found in {deploy_dir_spdx}")
+
+        self.doc.imports.extend(bb_objset.doc.imports)
+        self.update(bb_objset.objects)
+
+        return build
+
+    def new_sub_build(self, name, typ):
+        build = self.add(
+            oe.spdx30.build_Build(
+                _id=self.new_spdxid("build", name),
+                creationInfo=self.doc.creationInfo,
+                name="%s-%s" % (self.d.getVar("PN"), name),
+                build_buildType="%s/%s" % (SPDX_BUILD_TYPE, typ),
+            )
+        )
+
+        bitbake_build = self.import_bitbake_build()
+
+        self.new_relationship(
+            [bitbake_build],
+            oe.spdx30.RelationshipType.ancestorOf,
+            [build],
+        )
+
+        if self.d.getVar("SPDX_INCLUDE_BUILD_VARIABLES") == "1":
+            for varname in sorted(self.d.keys()):
+                if varname.startswith("__"):
+                    continue
+
+                value = self.d.getVar(varname, expand=False)
+
+                # TODO: Deal with non-string values
+                if not isinstance(value, str):
+                    continue
+
+                build.parameters.append(
+                    oe.spdx30.DictionaryEntry(key=varname, value=value)
+                )
+
+        return build
+
+    def new_archive(self, archive_name):
+        return self.add(
+            oe.spdx30.software_File(
+                _id=self.new_spdxid("archive", str(archive_name)),
+                creationInfo=self.doc.creationInfo,
+                name=str(archive_name),
+                software_primaryPurpose=oe.spdx30.software_SoftwarePurpose.archive,
+            )
+        )
+
+    @classmethod
+    def new_objset(cls, d, name, copy_from_bitbake_doc=True):
+        objset = cls(d)
+
+        document = oe.spdx30.SpdxDocument(
+            _id=objset.new_spdxid("document", name),
+            name=name,
+        )
+        document.extension.append(OEIdAliasExtension())
+        document.extension.append(OELinkExtension(link_spdx_id=False))
+        objset.doc = document
+
+        if copy_from_bitbake_doc:
+            build = objset.import_bitbake_build()
+            document.creationInfo = objset.copy_creation_info(build.creationInfo)
+        else:
+            document.creationInfo = objset.new_creation_info()
+
+        return objset
+
+    def expand_collection(self, *, add_objectsets=[]):
+        """
+        Expands a collection to pull in all missing elements
+
+        Returns the set of ids that could not be found to link into the document
+        """
+        missing_spdxids = set()
+        imports = {e.externalSpdxId: e for e in self.doc.imports}
+
+        def merge_doc(other):
+            nonlocal imports
+
+            for e in other.doc.imports:
+                if not e.externalSpdxId in imports:
+                    imports[e.externalSpdxId] = e
+
+            self.objects |= other.objects
+
+        for o in add_objectsets:
+            merge_doc(o)
+
+        needed_spdxids = self.link()
+        provided_spdxids = set(self.obj_by_id.keys())
+
+        while True:
+            import_spdxids = set(imports.keys())
+            searching_spdxids = (
+                needed_spdxids - provided_spdxids - missing_spdxids - import_spdxids
+            )
+            if not searching_spdxids:
+                break
+
+            spdxid = searching_spdxids.pop()
+            bb.debug(
+                1,
+                f"Searching for {spdxid}. Remaining: {len(searching_spdxids)}, Total: {len(provided_spdxids)}, Missing: {len(missing_spdxids)}, Imports: {len(import_spdxids)}",
+            )
+            dep_objset, dep_path = find_by_spdxid(self.d, spdxid)
+
+            if dep_objset:
+                dep_provided = set(dep_objset.obj_by_id.keys())
+                if spdxid not in dep_provided:
+                    bb.fatal(f"{spdxid} not found in {dep_path}")
+                provided_spdxids |= dep_provided
+                needed_spdxids |= dep_objset.missing_ids
+                merge_doc(dep_objset)
+            else:
+                missing_spdxids.add(spdxid)
+
+        bb.debug(1, "Linking...")
+        missing = self.link()
+        if missing != missing_spdxids:
+            bb.fatal(
+                f"Linked document doesn't match missing SPDX ID list. Got: {missing}\nExpected: {missing_spdxids}"
+            )
+
+        self.doc.imports = sorted(imports.values(), key=lambda e: e.externalSpdxId)
+
+        return missing_spdxids
+
+
+def load_jsonld(d, path, required=False):
+    deserializer = oe.spdx30.JSONLDDeserializer()
+    objset = ObjectSet(d)
+    try:
+        with path.open("rb") as f:
+            deserializer.read(f, objset)
+    except FileNotFoundError:
+        if required:
+            bb.fatal("No SPDX document named %s found" % path)
+        return None
+
+    if not objset.doc:
+        bb.fatal("SPDX Document %s has no SPDXDocument element" % path)
+        return None
+
+    objset.objects.remove(objset.doc)
+    return objset
+
+
+def jsonld_arch_path(d, arch, subdir, name, deploydir=None):
+    if deploydir is None:
+        deploydir = Path(d.getVar("DEPLOY_DIR_SPDX"))
+    return deploydir / arch / subdir / (name + ".spdx.json")
+
+
+def jsonld_hash_path(_id):
+    h = hashlib.sha256(_id.encode("utf-8")).hexdigest()
+
+    return Path("by-spdxid-hash") / h[:2], h
+
+
+def load_jsonld_by_arch(d, arch, subdir, name, *, required=False):
+    path = jsonld_arch_path(d, arch, subdir, name)
+    objset = load_jsonld(d, path, required=required)
+    if objset is not None:
+        return (objset, path)
+    return (None, None)
+
+
+def find_jsonld(d, subdir, name, *, required=False):
+    package_archs = d.getVar("SSTATE_ARCHS").split()
+    package_archs.reverse()
+
+    for arch in package_archs:
+        objset, path = load_jsonld_by_arch(d, arch, subdir, name)
+        if objset is not None:
+            return (objset, path)
+
+    if required:
+        bb.fatal("Could not find a %s SPDX document named %s" % (subdir, name))
+
+    return (None, None)
+
+
+def write_jsonld_doc(d, objset, dest):
+    if not isinstance(objset, ObjectSet):
+        bb.fatal("Only an ObjsetSet can be serialized")
+        return
+
+    if not objset.doc:
+        bb.fatal("ObjectSet is missing a SpdxDocument")
+        return
+
+    objset.doc.rootElement = sorted(list(set(objset.doc.rootElement)))
+    objset.doc.profileConformance = sorted(
+        list(
+            getattr(oe.spdx30.ProfileIdentifierType, p)
+            for p in d.getVar("SPDX_PROFILES").split()
+        )
+    )
+
+    dest.parent.mkdir(exist_ok=True, parents=True)
+
+    if d.getVar("SPDX_PRETTY") == "1":
+        serializer = oe.spdx30.JSONLDSerializer(
+            indent=2,
+        )
+    else:
+        serializer = oe.spdx30.JSONLDInlineSerializer()
+
+    objset.objects.add(objset.doc)
+    with dest.open("wb") as f:
+        serializer.write(objset, f, force_at_graph=True)
+    objset.objects.remove(objset.doc)
+
+
+def write_recipe_jsonld_doc(
+    d,
+    objset,
+    subdir,
+    deploydir,
+    *,
+    create_spdx_id_links=True,
+):
+    pkg_arch = d.getVar("SSTATE_PKGARCH")
+
+    dest = jsonld_arch_path(d, pkg_arch, subdir, objset.doc.name, deploydir=deploydir)
+
+    def link_id(_id):
+        hash_path = jsonld_hash_path(_id)
+
+        link_name = jsonld_arch_path(
+            d,
+            pkg_arch,
+            *hash_path,
+            deploydir=deploydir,
+        )
+        try:
+            link_name.parent.mkdir(exist_ok=True, parents=True)
+            link_name.symlink_to(os.path.relpath(dest, link_name.parent))
+        except:
+            target = link_name.readlink()
+            bb.warn(
+                f"Unable to link {_id} in {dest} as {link_name}. Already points to {target}"
+            )
+            raise
+
+        return hash_path[-1]
+
+    objset.add_aliases()
+
+    try:
+        if create_spdx_id_links:
+            for o in objset.foreach_type(oe.spdx30.Element):
+                if not o._id or o._id.startswith("_:"):
+                    continue
+
+                ext = None
+                for e in o.extension:
+                    if not isinstance(e, OELinkExtension):
+                        continue
+
+                    ext = e
+                    break
+
+                if ext is None:
+                    ext = OELinkExtension(link_spdx_id=True)
+                    o.extension.append(ext)
+
+                if ext.link_spdx_id:
+                    ext.link_name = link_id(o._id)
+
+                    alias_ext = get_alias(o)
+                    if alias_ext is not None and alias_ext.alias:
+                        alias_ext.link_name = link_id(alias_ext.alias)
+
+    finally:
+        # It is really helpful for debugging if the JSON document is written
+        # out, so always do that even if there is an error making the links
+        write_jsonld_doc(d, objset, dest)
+
+
+def find_root_obj_in_jsonld(d, subdir, fn_name, obj_type, **attr_filter):
+    objset, fn = find_jsonld(d, subdir, fn_name, required=True)
+
+    spdx_obj = objset.find_root(obj_type, **attr_filter)
+    if not spdx_obj:
+        bb.fatal("No root %s found in %s" % (obj_type.__name__, fn))
+
+    return spdx_obj, objset
+
+
+def load_obj_in_jsonld(d, arch, subdir, fn_name, obj_type, **attr_filter):
+    objset, fn = load_jsonld_by_arch(d, arch, subdir, fn_name, required=True)
+
+    spdx_obj = objset.find_filter(obj_type, **attr_filter)
+    if not spdx_obj:
+        bb.fatal("No %s found in %s" % (obj_type.__name__, fn))
+
+    return spdx_obj, objset
+
+
+def find_by_spdxid(d, spdxid, *, required=False):
+    return find_jsonld(d, *jsonld_hash_path(spdxid), required=required)
+
+
+def create_sbom(d, name, root_elements, add_objectsets=[]):
+    objset = ObjectSet.new_objset(d, name)
+
+    sbom = objset.add(
+        oe.spdx30.software_Sbom(
+            _id=objset.new_spdxid("sbom", name),
+            name=name,
+            creationInfo=objset.doc.creationInfo,
+            software_sbomType=[oe.spdx30.software_SbomType.build],
+            rootElement=root_elements,
+        )
+    )
+
+    missing_spdxids = objset.expand_collection(add_objectsets=add_objectsets)
+    if missing_spdxids:
+        bb.warn(
+            "The following SPDX IDs were unable to be resolved:\n  "
+            + "\n  ".join(sorted(list(missing_spdxids)))
+        )
+
+    # Filter out internal extensions from final SBoMs
+    objset.remove_internal_extensions()
+
+    # SBoM should be the only root element of the document
+    objset.doc.rootElement = [sbom]
+
+    # De-duplicate licenses
+    unique = set()
+    dedup = {}
+    for lic in objset.foreach_type(oe.spdx30.simplelicensing_LicenseExpression):
+        for u in unique:
+            if (
+                u.simplelicensing_licenseExpression
+                == lic.simplelicensing_licenseExpression
+                and u.simplelicensing_licenseListVersion
+                == lic.simplelicensing_licenseListVersion
+            ):
+                dedup[lic] = u
+                break
+        else:
+            unique.add(lic)
+
+    if dedup:
+        for rel in objset.foreach_filter(
+            oe.spdx30.Relationship,
+            relationshipType=oe.spdx30.RelationshipType.hasDeclaredLicense,
+        ):
+            rel.to = [dedup.get(to, to) for to in rel.to]
+
+        for rel in objset.foreach_filter(
+            oe.spdx30.Relationship,
+            relationshipType=oe.spdx30.RelationshipType.hasConcludedLicense,
+        ):
+            rel.to = [dedup.get(to, to) for to in rel.to]
+
+        for k, v in dedup.items():
+            bb.debug(1, f"Removing duplicate License {k._id} -> {v._id}")
+            objset.objects.remove(k)
+
+        objset.create_index()
+
+    return objset, sbom
meta/lib/oe/spdx30.py new file +5996 -1
new file mode 100644
index 00000000000..3d24692f96e
@@ -0,0 +1,5996 @@
+#! /usr/bin/env python3
+#
+# Generated Python bindings from a SHACL model
+#
+# This file was automatically generated by shacl2code. DO NOT MANUALLY MODIFY IT
+#
+# SPDX-License-Identifier: MIT
+
+import functools
+import hashlib
+import json
+import re
+import sys
+import threading
+import time
+from contextlib import contextmanager
+from datetime import datetime, timezone, timedelta
+from enum import Enum
+from abc import ABC, abstractmethod
+
+
+def check_type(obj, types):
+    if not isinstance(obj, types):
+        if isinstance(types, (list, tuple)):
+            raise TypeError(
+                f"Value must be one of type: {', '.join(t.__name__ for t in types)}. Got {type(obj)}"
+            )
+        raise TypeError(f"Value must be of type {types.__name__}. Got {type(obj)}")
+
+
+class Property(ABC):
+    """
+    A generic SHACL object property. The different types will derive from this
+    class
+    """
+
+    def __init__(self, *, pattern=None):
+        self.pattern = pattern
+
+    def init(self):
+        return None
+
+    def validate(self, value):
+        check_type(value, self.VALID_TYPES)
+        if self.pattern is not None and not re.search(
+            self.pattern, self.to_string(value)
+        ):
+            raise ValueError(
+                f"Value is not correctly formatted. Got '{self.to_string(value)}'"
+            )
+
+    def set(self, value):
+        return value
+
+    def check_min_count(self, value, min_count):
+        return min_count == 1
+
+    def check_max_count(self, value, max_count):
+        return max_count == 1
+
+    def elide(self, value):
+        return value is None
+
+    def walk(self, value, callback, path):
+        callback(value, path)
+
+    def iter_objects(self, value, recursive, visited):
+        return []
+
+    def link_prop(self, value, objectset, missing, visited):
+        return value
+
+    def to_string(self, value):
+        return str(value)
+
+    @abstractmethod
+    def encode(self, encoder, value, state):
+        pass
+
+    @abstractmethod
+    def decode(self, decoder, *, objectset=None):
+        pass
+
+
+class StringProp(Property):
+    """
+    A scalar string property for an SHACL object
+    """
+
+    VALID_TYPES = str
+
+    def set(self, value):
+        return str(value)
+
+    def encode(self, encoder, value, state):
+        encoder.write_string(value)
+
+    def decode(self, decoder, *, objectset=None):
+        return decoder.read_string()
+
+
+class AnyURIProp(StringProp):
+    def encode(self, encoder, value, state):
+        encoder.write_iri(value)
+
+    def decode(self, decoder, *, objectset=None):
+        return decoder.read_iri()
+
+
+class DateTimeProp(Property):
+    """
+    A Date/Time Object with optional timezone
+    """
+
+    VALID_TYPES = datetime
+    UTC_FORMAT_STR = "%Y-%m-%dT%H:%M:%SZ"
+    REGEX = r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(Z|[+-]\d{2}:\d{2})?$"
+
+    def set(self, value):
+        return self._normalize(value)
+
+    def encode(self, encoder, value, state):
+        encoder.write_datetime(self.to_string(value))
+
+    def decode(self, decoder, *, objectset=None):
+        s = decoder.read_datetime()
+        if s is None:
+            return None
+        v = self.from_string(s)
+        return self._normalize(v)
+
+    def _normalize(self, value):
+        if value.utcoffset() is None:
+            value = value.astimezone()
+        offset = value.utcoffset()
+        if offset % timedelta(minutes=1):
+            offset = offset - (offset % timedelta(minutes=1))
+            value = value.replace(tzinfo=timezone(offset))
+        value = value.replace(microsecond=0)
+        return value
+
+    def to_string(self, value):
+        value = self._normalize(value)
+        if value.tzinfo == timezone.utc:
+            return value.strftime(self.UTC_FORMAT_STR)
+        return value.isoformat()
+
+    def from_string(self, value):
+        if not re.match(self.REGEX, value):
+            raise ValueError(f"'{value}' is not a correctly formatted datetime")
+        if "Z" in value:
+            d = datetime(
+                *(time.strptime(value, self.UTC_FORMAT_STR)[0:6]),
+                tzinfo=timezone.utc,
+            )
+        else:
+            d = datetime.fromisoformat(value)
+
+        return self._normalize(d)
+
+
+class DateTimeStampProp(DateTimeProp):
+    """
+    A Date/Time Object with required timestamp
+    """
+
+    REGEX = r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(Z|[+-]\d{2}:\d{2})$"
+
+
+class IntegerProp(Property):
+    VALID_TYPES = int
+
+    def set(self, value):
+        return int(value)
+
+    def encode(self, encoder, value, state):
+        encoder.write_integer(value)
+
+    def decode(self, decoder, *, objectset=None):
+        return decoder.read_integer()
+
+
+class PositiveIntegerProp(IntegerProp):
+    def validate(self, value):
+        super().validate(value)
+        if value < 1:
+            raise ValueError(f"Value must be >=1. Got {value}")
+
+
+class NonNegativeIntegerProp(IntegerProp):
+    def validate(self, value):
+        super().validate(value)
+        if value < 0:
+            raise ValueError(f"Value must be >= 0. Got {value}")
+
+
+class BooleanProp(Property):
+    VALID_TYPES = bool
+
+    def set(self, value):
+        return bool(value)
+
+    def encode(self, encoder, value, state):
+        encoder.write_bool(value)
+
+    def decode(self, decoder, *, objectset=None):
+        return decoder.read_bool()
+
+
+class FloatProp(Property):
+    VALID_TYPES = (float, int)
+
+    def set(self, value):
+        return float(value)
+
+    def encode(self, encoder, value, state):
+        encoder.write_float(value)
+
+    def decode(self, decoder, *, objectset=None):
+        return decoder.read_float()
+
+
+class ObjectProp(Property):
+    """
+    A scalar SHACL object property of a SHACL object
+    """
+
+    def __init__(self, cls, required):
+        super().__init__()
+        self.cls = cls
+        self.required = required
+
+    def init(self):
+        if self.required and not self.cls.IS_ABSTRACT:
+            return self.cls()
+        return None
+
+    def validate(self, value):
+        check_type(value, (self.cls, str))
+
+    def walk(self, value, callback, path):
+        if value is None:
+            return
+
+        if not isinstance(value, str):
+            value.walk(callback, path)
+        else:
+            callback(value, path)
+
+    def iter_objects(self, value, recursive, visited):
+        if value is None or isinstance(value, str):
+            return
+
+        if value not in visited:
+            visited.add(value)
+            yield value
+
+            if recursive:
+                for c in value.iter_objects(recursive=True, visited=visited):
+                    yield c
+
+    def encode(self, encoder, value, state):
+        if value is None:
+            raise ValueError("Object cannot be None")
+
+        if isinstance(value, str):
+            value = _NI_ENCODE_CONTEXT.get(value, value)
+            encoder.write_iri(value)
+            return
+
+        return value.encode(encoder, state)
+
+    def decode(self, decoder, *, objectset=None):
+        iri = decoder.read_iri()
+        if iri is None:
+            return self.cls.decode(decoder, objectset=objectset)
+
+        iri = _NI_DECODE_CONTEXT.get(iri, iri)
+
+        if objectset is None:
+            return iri
+
+        obj = objectset.find_by_id(iri)
+        if obj is None:
+            return iri
+
+        self.validate(obj)
+        return obj
+
+    def link_prop(self, value, objectset, missing, visited):
+        if value is None:
+            return value
+
+        if isinstance(value, str):
+            o = objectset.find_by_id(value)
+            if o is not None:
+                self.validate(o)
+                return o
+
+            if missing is not None:
+                missing.add(value)
+
+            return value
+
+        # De-duplicate IDs
+        if value._id:
+            value = objectset.find_by_id(value._id, value)
+            self.validate(value)
+
+        value.link_helper(objectset, missing, visited)
+        return value
+
+
+class ListProxy(object):
+    def __init__(self, prop, data=None):
+        if data is None:
+            self.__data = []
+        else:
+            self.__data = data
+        self.__prop = prop
+
+    def append(self, value):
+        self.__prop.validate(value)
+        self.__data.append(self.__prop.set(value))
+
+    def insert(self, idx, value):
+        self.__prop.validate(value)
+        self.__data.insert(idx, self.__prop.set(value))
+
+    def extend(self, items):
+        for i in items:
+            self.append(i)
+
+    def sort(self, *args, **kwargs):
+        self.__data.sort(*args, **kwargs)
+
+    def __getitem__(self, key):
+        return self.__data[key]
+
+    def __setitem__(self, key, value):
+        if isinstance(key, slice):
+            for v in value:
+                self.__prop.validate(v)
+            self.__data[key] = [self.__prop.set(v) for v in value]
+        else:
+            self.__prop.validate(value)
+            self.__data[key] = self.__prop.set(value)
+
+    def __delitem__(self, key):
+        del self.__data[key]
+
+    def __contains__(self, item):
+        return item in self.__data
+
+    def __iter__(self):
+        return iter(self.__data)
+
+    def __len__(self):
+        return len(self.__data)
+
+    def __str__(self):
+        return str(self.__data)
+
+    def __repr__(self):
+        return repr(self.__data)
+
+    def __eq__(self, other):
+        if isinstance(other, ListProxy):
+            return self.__data == other.__data
+
+        return self.__data == other
+
+
+class ListProp(Property):
+    """
+    A list of SHACL properties
+    """
+
+    VALID_TYPES = (list, ListProxy)
+
+    def __init__(self, prop):
+        super().__init__()
+        self.prop = prop
+
+    def init(self):
+        return ListProxy(self.prop)
+
+    def validate(self, value):
+        super().validate(value)
+
+        for i in value:
+            self.prop.validate(i)
+
+    def set(self, value):
+        if isinstance(value, ListProxy):
+            return value
+
+        return ListProxy(self.prop, [self.prop.set(d) for d in value])
+
+    def check_min_count(self, value, min_count):
+        check_type(value, ListProxy)
+        return len(value) >= min_count
+
+    def check_max_count(self, value, max_count):
+        check_type(value, ListProxy)
+        return len(value) <= max_count
+
+    def elide(self, value):
+        check_type(value, ListProxy)
+        return len(value) == 0
+
+    def walk(self, value, callback, path):
+        callback(value, path)
+        for idx, v in enumerate(value):
+            self.prop.walk(v, callback, path + [f"[{idx}]"])
+
+    def iter_objects(self, value, recursive, visited):
+        for v in value:
+            for c in self.prop.iter_objects(v, recursive, visited):
+                yield c
+
+    def link_prop(self, value, objectset, missing, visited):
+        if isinstance(value, ListProxy):
+            data = [self.prop.link_prop(v, objectset, missing, visited) for v in value]
+        else:
+            data = [self.prop.link_prop(v, objectset, missing, visited) for v in value]
+
+        return ListProxy(self.prop, data=data)
+
+    def encode(self, encoder, value, state):
+        check_type(value, ListProxy)
+
+        with encoder.write_list() as list_s:
+            for v in value:
+                with list_s.write_list_item() as item_s:
+                    self.prop.encode(item_s, v, state)
+
+    def decode(self, decoder, *, objectset=None):
+        data = []
+        for val_d in decoder.read_list():
+            v = self.prop.decode(val_d, objectset=objectset)
+            self.prop.validate(v)
+            data.append(v)
+
[… more lines in the raw message]

Lines of this file are not shown. View the raw message.

The rest of this diff is not shown. View the raw message.

Links: You receive all messages sent to this group.
View/Reply Online (#201106): https://lists.openembedded.org/g/openembedded-core/message/201106
Mute This Topic: https://lists.openembedded.org/mt/106856878/3616948
Group Owner: [email protected]
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [[email protected]]
-=-=-=-=-=-=-=-=-=-=-=-

Part of the diff is not shown here. View the raw message.

#47 lore raw

Re: [OE-core][PATCH v4 03/10] classes/create-spdx-3.0: Add classes

Quoted 4 lines · Mark Hatle

Comments inline below

On 6/24/24 2:10 PM, Joshua Watt wrote:

-- snip --

Quoted 7 lines

meta/classes/create-spdx-3.0.bbclass line 65

Quoted 3 lines
meta/classes/create-spdx-3.0.bbclass new file +3 -0
+
+SPDX_BUILD_HOST[doc] = "The base variable name to describe the build host on \
+    which a build is running. Must be an SPDX_IMPORTS key"

Is there any sort of documentation or external reference for the variable above (as well as the SPDX_ below) that explains what the SPDX standard is expecting to be put in there?

Not specifically for this, but for the SPDX 3.0 spec in general, the web docs are pretty comprehensive:
https://spdx.github.io/spdx-spec/v3.0/ . Although, be aware the navigation sidebar is really annoying ATM, but that's supposed to get fixed soon.

For a starter on how SPDX documents are written, see:
https://github.com/spdx/spdx-spec/blob/development/v3.0.1/docs/annexes/getting-started.md

It's a little tricky to encode the SPDX 3 structured data in bitbake variables; this is what I could come up with so far but if you have suggestions on improvements, let me know.

Specifically for this variable, it's referencing a "key" in SPDX_IMPORTS. SPDX_IMPORTS in turn is encoding entries in the "imports" property of
https://spdx.github.io/spdx-spec/v3.0/model/Core/Classes/SpdxDocument/
. The indirection is necessary so that you can tell users where the SPDX ID lives, since it's external to this document. It's pretty much impossible for us to validate that the SPDX ID you put in is real, short of downloading the referenced document, parsing it, and seeing if the SPDX ID is present.

For a walk through of how to cross-link SPDX 3 documents, look at:
https://github.com/spdx/spdx-spec/blob/development/v3.0.1/docs/annexes/cross-reference.md

Quoted 4 lines

I.e. machine name, host type, an arbitrary string that means something to the agent construction the SPDX, etc. I.e. how would I know what is valid in these various things?

There are pretty good tools to validate SPDX documents (offline
even!). https://github.com/spdx/spdx-3-model/blob/main/serialization/json_ld/validation.md
gives an overview on how to do this. It still won't do the validation
that the external SPDX ID is valid for the same reasons as above, but
it's pretty good otherwise.

Quoted 6 lines

This then leads to a second question, deterministic behavior/reproducibility. I believe the purpose of this is reproducible builds, but we should have a more deterministic approach in the Yocto Project where we provide (and/or check for host capabilities) to help allow this to be a more generic, many different host process.

I've attempted to make this process as deterministic as SPDX 3 allows. As an example, SPDX IDs are generated by hashing deterministic data (where as random SPDX IDs would be decidedly simpler!). However, there are parts of SPDX 3 that are simply not deterministic for various reasons (please read to the end of this section). They generally fall under a few categories:

The first category is probably best classified as "not very useful if deterministic" and omitted by default; SPDX_BUILD_HOST would be one such examples, since if you are going to set this to the same value all the time and not reference the _actual_ host, you may as well not include it at all. For these, I've set no default value (there isn't one that would make sense anyway), so their omission keeps things deterministic. However users do actually want this, it will necessarily result in non-deterministic builds unless you always do your builds on the same host. I think it might be helpful to annotate in the doc string which variables will introduce such non-determinism, so I'll do that.

The second category is "not very useful if deterministic", but are included in the output by this patch. Examples of this would be build timestamps and the bitbake parent build tracking (which basically tracks the invocation of bitbake itself as the "parent" build, so you can tell which tasks ran in the same invocation). These are useful pieces of information, and consumers do actually care about these things, so if push comes to shove we could add a flag to enable them, but I'm also leery of having too many configuration options for SPDX.

The last category are the require non-deterministic fields in SPDX 3. The primary offender here is the SPDX creation info "created"
datestamp: https://spdx.github.io/spdx-spec/v3.0/model/Core/Classes/CreationInfo/
. This is a mandatory field that is the timestamp of when the SPDX data itself was created, and every SPDX object you create links to this so you can track exactly when each object in a merged document was created. I did attempt to make the argument to SPDX that it was mandatory non-determinism, but it is very important to the SPDX community (for reasons I've not fully understood), and they _really_ want it to be the actual document creation date, not SOURCE_DATE_EPOCH or similar, so I really am not sure what to do about that one. I was more or less told to ignore these fields when calculating if output is "deterministic", which is a little annoying, but not an argument I could win.

I'm a little stuck between the SPDX side and the Yocto side on the determinism front. It's easy to say "SPDX must be deterministic", and "it's awful if non-deterministic" if your looking at it from just the point of view of you want to spit out some data, but equally, it's easy to say "determinism doesn't really matter" and "the non-deterministic data is important information" when you are consuming the data. This particular patch series errs on the side of making the data the most useful to the end consumers, in part because I really want Yocto to generate the most comprehensive and useful output it can; we are a pretty early adopter of SPDX 3, so being able to provide the most useful data we can early can means consumers writing downstream tools can use us as a reference which drastically improves their compatibility with our output. Yocto has a phenomenal supply chain story to tell, and I really want to tell it to the fullest extent that we can in our SPDX data. I can't reconcile that with "everything must be deterministic" though, so..... ?

Quoted 4 lines

Which leads to a third question, when a build uses sstate-cache, each host could be different then the build host that actually combines the builds into an image (SBOM). Is this a concern?

That is very much on purpose so you can track where your sstate came from, who built it, etc, but also why we don't set any of these by default (see above). This is pretty important from a supply chain tracking perspective, and part of the comprehensive story we can tell about the supply chain (IMHO anyway).

Currently, we are only tracking the actual do_create_spdx() task as the "build" (which isn't clear in the generated SPDX, but I'll make change to fix that), so you have to be a little bit careful about some of the conclusions you draw from that. Once we get the base SPDX 3 support in place, I want to look at having other sstate tasks generate SPDX fragements when they run which would allow us to trace those tasks more precisely. I don't really want to solve that now though as it is going to be quite a bit more complex to solve in a satisfactory manner.

Quoted 1 line

-- snip --

#48 lore raw

Re: [OE-core][PATCH v4 03/10] classes/create-spdx-3.0: Add classes

Quoted 88 lines · Joshua Watt
Quoted 4 lines · Mark Hatle

Comments inline below

On 6/24/24 2:10 PM, Joshua Watt wrote:

-- snip --

Quoted 7 lines

meta/classes/create-spdx-3.0.bbclass line 65

Quoted 3 lines
meta/classes/create-spdx-3.0.bbclass new file +3 -0
+
+SPDX_BUILD_HOST[doc] = "The base variable name to describe the build host on \
+    which a build is running. Must be an SPDX_IMPORTS key"

Is there any sort of documentation or external reference for the variable above (as well as the SPDX_ below) that explains what the SPDX standard is expecting to be put in there?

Not specifically for this, but for the SPDX 3.0 spec in general, the web docs are pretty comprehensive:
https://spdx.github.io/spdx-spec/v3.0/ . Although, be aware the navigation sidebar is really annoying ATM, but that's supposed to get fixed soon.

For a starter on how SPDX documents are written, see:
https://github.com/spdx/spdx-spec/blob/development/v3.0.1/docs/annexes/getting-started.md

It's a little tricky to encode the SPDX 3 structured data in bitbake variables; this is what I could come up with so far but if you have suggestions on improvements, let me know.

Specifically for this variable, it's referencing a "key" in SPDX_IMPORTS. SPDX_IMPORTS in turn is encoding entries in the "imports" property of
https://spdx.github.io/spdx-spec/v3.0/model/Core/Classes/SpdxDocument/
. The indirection is necessary so that you can tell users where the SPDX ID lives, since it's external to this document. It's pretty much impossible for us to validate that the SPDX ID you put in is real, short of downloading the referenced document, parsing it, and seeing if the SPDX ID is present.

For a walk through of how to cross-link SPDX 3 documents, look at:
https://github.com/spdx/spdx-spec/blob/development/v3.0.1/docs/annexes/cross-reference.md

Quoted 4 lines

I.e. machine name, host type, an arbitrary string that means something to the agent construction the SPDX, etc. I.e. how would I know what is valid in these various things?

There are pretty good tools to validate SPDX documents (offline
even!). https://github.com/spdx/spdx-3-model/blob/main/serialization/json_ld/validation.md
gives an overview on how to do this. It still won't do the validation
that the external SPDX ID is valid for the same reasons as above, but
it's pretty good otherwise.

Quoted 6 lines

This then leads to a second question, deterministic behavior/reproducibility. I believe the purpose of this is reproducible builds, but we should have a more deterministic approach in the Yocto Project where we provide (and/or check for host capabilities) to help allow this to be a more generic, many different host process.

I've attempted to make this process as deterministic as SPDX 3 allows. As an example, SPDX IDs are generated by hashing deterministic data (where as random SPDX IDs would be decidedly simpler!). However, there are parts of SPDX 3 that are simply not deterministic for various reasons (please read to the end of this section). They generally fall under a few categories:

The first category is probably best classified as "not very useful if deterministic" and omitted by default; SPDX_BUILD_HOST would be one such examples, since if you are going to set this to the same value all the time and not reference the _actual_ host, you may as well not include it at all. For these, I've set no default value (there isn't one that would make sense anyway), so their omission keeps things deterministic. However users do actually want this, it will necessarily result in non-deterministic builds unless you always do your builds on the same host. I think it might be helpful to annotate in the doc string which variables will introduce such non-determinism, so I'll do that.

The second category is "not very useful if deterministic", but are included in the output by this patch. Examples of this would be build timestamps and the bitbake parent build tracking (which basically tracks the invocation of bitbake itself as the "parent" build, so you can tell which tasks ran in the same invocation). These are useful pieces of information, and consumers do actually care about these things, so if push comes to shove we could add a flag to enable them, but I'm also leery of having too many configuration options for SPDX.

Quoted 14 lines

The last category are the require non-deterministic fields in SPDX 3. The primary offender here is the SPDX creation info "created"
datestamp: https://spdx.github.io/spdx-spec/v3.0/model/Core/Classes/CreationInfo/
. This is a mandatory field that is the timestamp of when the SPDX data itself was created, and every SPDX object you create links to this so you can track exactly when each object in a merged document was created. I did attempt to make the argument to SPDX that it was mandatory non-determinism, but it is very important to the SPDX community (for reasons I've not fully understood), and they _really_ want it to be the actual document creation date, not SOURCE_DATE_EPOCH or similar, so I really am not sure what to do about that one. I was more or less told to ignore these fields when calculating if output is "deterministic", which is a little annoying, but not an argument I could win.

Heh, of course I just read the spec a little more carefully, and SOURCE_DATE_EPOCH is allowed here, so we can do that and make everything deterministic by default, with options to enable non-determinism.
https://spdx.github.io/spdx-spec/v3.0/model/Core/Properties/created/

*facepalm*

Quoted 42 lines

I'm a little stuck between the SPDX side and the Yocto side on the determinism front. It's easy to say "SPDX must be deterministic", and "it's awful if non-deterministic" if your looking at it from just the point of view of you want to spit out some data, but equally, it's easy to say "determinism doesn't really matter" and "the non-deterministic data is important information" when you are consuming the data. This particular patch series errs on the side of making the data the most useful to the end consumers, in part because I really want Yocto to generate the most comprehensive and useful output it can; we are a pretty early adopter of SPDX 3, so being able to provide the most useful data we can early can means consumers writing downstream tools can use us as a reference which drastically improves their compatibility with our output. Yocto has a phenomenal supply chain story to tell, and I really want to tell it to the fullest extent that we can in our SPDX data. I can't reconcile that with "everything must be deterministic" though, so..... ?

Quoted 4 lines

Which leads to a third question, when a build uses sstate-cache, each host could be different then the build host that actually combines the builds into an image (SBOM). Is this a concern?

That is very much on purpose so you can track where your sstate came from, who built it, etc, but also why we don't set any of these by default (see above). This is pretty important from a supply chain tracking perspective, and part of the comprehensive story we can tell about the supply chain (IMHO anyway).

Currently, we are only tracking the actual do_create_spdx() task as the "build" (which isn't clear in the generated SPDX, but I'll make change to fix that), so you have to be a little bit careful about some of the conclusions you draw from that. Once we get the base SPDX 3 support in place, I want to look at having other sstate tasks generate SPDX fragements when they run which would allow us to trace those tasks more precisely. I don't really want to solve that now though as it is going to be quite a bit more complex to solve in a satisfactory manner.

Quoted 1 line

-- snip --

#49 lore raw

PATCH v4 4/10 classes-global/staging: Exclude do_create_spdx from automatic sysroot extension

In reply to Joshua Watt

do_create_spdx is a outlier in that it doesn't need the RSS to be extended just because it depends on do_populate_sysroot. In fact, it only depends on do_populate_sysroot so it can see the actual recipes sysroot, and attempting to extend the sysroot can cause problems for some recipes (e.g. if a recipe does do_populate_sysroot[noexec] = "1")

As such, explicitly exclude do_create_spdx from extending the sysroot just because it depends on do_populate_sysroot.

---
 meta/classes-global/staging.bbclass | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)
meta/classes-global/staging.bbclass +8 -1
index 3678a1b4415..c2213ffa2b4 100644
@@ -652,10 +652,17 @@ python do_prepare_recipe_sysroot () {
 addtask do_prepare_recipe_sysroot before do_configure after do_fetch
 
 python staging_taskhandler() {
+    EXCLUDED_TASKS = (
+        "do_prepare_recipe_sysroot",
+        "do_create_spdx",
+    )
     bbtasks = e.tasklist
     for task in bbtasks:
+        if task in EXCLUDED_TASKS:
+            continue
+
         deps = d.getVarFlag(task, "depends")
-        if task != 'do_prepare_recipe_sysroot' and (task == "do_configure" or (deps and "populate_sysroot" in deps)):
+        if task == "do_configure" or (deps and "populate_sysroot" in deps):
             d.prependVarFlag(task, "prefuncs", "extend_recipe_sysroot ")
 }
 staging_taskhandler[eventmask] = "bb.event.RecipeTaskPreProcess"
Signature · 1 line
2.43.2
#50 lore raw

PATCH v4 5/10 binutils-cross-testsuite: Rename to binutils-testsuite

In reply to Joshua Watt

This recipe needs to be renamed because the "-cross-" substring in the name triggers the cross architecture detection in sstate, but this recipe is not actually a cross recipe.

---
 meta/classes-global/mirrors.bbclass                            | 2 +-
 meta/conf/distro/include/maintainers.inc                       | 2 +-
 meta/lib/oeqa/selftest/cases/binutils.py                       | 2 +-
 ...tils-cross-testsuite_2.42.bb => binutils-testsuite_2.42.bb} | 3 +++
 4 files changed, 6 insertions(+), 3 deletions(-)
 rename meta/recipes-devtools/binutils/{binutils-cross-testsuite_2.42.bb => binutils-testsuite_2.42.bb} (95%)
meta/classes-global/mirrors.bbclass +1 -1
index 862648eec5f..d68d30b0f2a 100644
@@ -88,7 +88,7 @@ git://.*/.*                   git://HOST/git/PATH;protocol=https \
 BB_GIT_SHALLOW:pn-binutils = "1"
 BB_GIT_SHALLOW:pn-binutils-cross-${TARGET_ARCH} = "1"
 BB_GIT_SHALLOW:pn-binutils-cross-canadian-${TRANSLATED_TARGET_ARCH} = "1"
-BB_GIT_SHALLOW:pn-binutils-cross-testsuite = "1"
+BB_GIT_SHALLOW:pn-binutils-testsuite = "1"
 BB_GIT_SHALLOW:pn-binutils-crosssdk-${SDK_SYS} = "1"
 BB_GIT_SHALLOW:pn-binutils-native = "1"
 BB_GIT_SHALLOW:pn-nativesdk-binutils = "1"
meta/conf/distro/include/maintainers.inc +1 -1
index 14e0637c838..368d3de1583 100644
@@ -64,7 +64,7 @@ RECIPE_MAINTAINER:pn-bind = "Unassigned <[email protected]>"
 RECIPE_MAINTAINER:pn-binutils = "Khem Raj <[email protected]>"
 RECIPE_MAINTAINER:pn-binutils-cross-${TARGET_ARCH} = "Khem Raj <[email protected]>"
 RECIPE_MAINTAINER:pn-binutils-cross-canadian-${TRANSLATED_TARGET_ARCH} = "Khem Raj <[email protected]>"
-RECIPE_MAINTAINER:pn-binutils-cross-testsuite = "Khem Raj <[email protected]>"
+RECIPE_MAINTAINER:pn-binutils-testsuite = "Khem Raj <[email protected]>"
 RECIPE_MAINTAINER:pn-binutils-crosssdk-${SDK_SYS} = "Khem Raj <[email protected]>"
 RECIPE_MAINTAINER:pn-bison = "Chen Qi <[email protected]>"
 RECIPE_MAINTAINER:pn-blktool = "Yi Zhao <[email protected]>"
meta/lib/oeqa/selftest/cases/binutils.py +1 -1
index 1688eabe4ec..5ff263d342a 100644
@@ -33,7 +33,7 @@ class BinutilsCrossSelfTest(OESelftestTestCase, OEPTestResultTestCase):
         features.append('CHECK_TARGETS = "{0}"'.format(suite))
         self.write_config("\n".join(features))
 
-        recipe = "binutils-cross-testsuite"
+        recipe = "binutils-testsuite"
         bb_vars = get_bb_vars(["B", "TARGET_SYS", "T"], recipe)
         builddir, target_sys, tdir = bb_vars["B"], bb_vars["TARGET_SYS"], bb_vars["T"]
 
meta/recipes-devtools/binutils/binutils-testsuite_2.42.bb renamed from meta/recipes-devtools/binutils/binutils-cross-testsuite_2.42.bb +3 -0
similarity index 95%
rename from meta/recipes-devtools/binutils/binutils-cross-testsuite_2.42.bb
rename to meta/recipes-devtools/binutils/binutils-testsuite_2.42.bb
index 630815c7a3e..f2facd52c3a 100644
@@ -1,3 +1,6 @@
+# NOTE: This recipe cannot have -cross- in the file name because it triggers
+# the cross build detection in sstate which causes it to use the wrong
+# architecture
 require binutils.inc
 require binutils-${PV}.inc
 
Signature · 1 line
2.43.2
#51 lore raw

PATCH v4 6/10 classes-recipe/image_types: Add SPDX_IMAGE_PURPOSE to images

In reply to Joshua Watt

Adds the variable overrides to set the SPDX image purpose for various image types

---
 meta/classes-recipe/image_types.bbclass     | 2 ++
 meta/classes-recipe/image_types_wic.bbclass | 1 +
 2 files changed, 3 insertions(+)
meta/classes-recipe/image_types.bbclass +2 -0
index 2f948ecbf88..506b9934cb7 100644
@@ -146,6 +146,7 @@ IMAGE_CMD:vfat = "oe_mkvfatfs ${EXTRA_IMAGECMD}"
 IMAGE_CMD_TAR ?= "tar"
 # ignore return code 1 "file changed as we read it" as other tasks(e.g. do_image_wic) may be hardlinking rootfs
 IMAGE_CMD:tar = "${IMAGE_CMD_TAR} --sort=name --format=posix --numeric-owner -cf ${IMGDEPLOYDIR}/${IMAGE_NAME}.tar -C ${IMAGE_ROOTFS} . || [ $? -eq 1 ]"
+SPDX_IMAGE_PURPOSE:tar = "archive"
 
 do_image_cpio[cleandirs] += "${WORKDIR}/cpio_append"
 IMAGE_CMD:cpio () {
@@ -167,6 +168,7 @@ IMAGE_CMD:cpio () {
 		fi
 	fi
 }
+SPDX_IMAGE_PURPOSE:cpio = "archive"
 
 UBI_VOLNAME ?= "${MACHINE}-rootfs"
 UBI_VOLTYPE ?= "dynamic"
meta/classes-recipe/image_types_wic.bbclass +1 -0
index cf3be909b30..86f40633ebc 100644
@@ -91,6 +91,7 @@ IMAGE_CMD:wic () {
 	mv "$build_wic/$(basename "${wks%.wks}")"*.${IMAGER} "$out.wic"
 }
 IMAGE_CMD:wic[vardepsexclude] = "WKS_FULL_PATH WKS_FILES TOPDIR"
+SPDX_IMAGE_PURPOSE:wic = "diskImage"
 do_image_wic[cleandirs] = "${WORKDIR}/build-wic"
 
 PSEUDO_IGNORE_PATHS .= ",${WORKDIR}/build-wic"
Signature · 1 line
2.43.2
#52 lore raw

PATCH v4 7/10 selftest: spdx: Add SPDX 3.0 test cases

In reply to Joshua Watt

Adds test cases for SPDX 3.0. Reworks the SPDX 2.2 test setup so it can also be run even if the default is SPDX 3.0

---
 meta/lib/oeqa/selftest/cases/spdx.py | 119 +++++++++++++++++++++++++--
 1 file changed, 110 insertions(+), 9 deletions(-)
meta/lib/oeqa/selftest/cases/spdx.py +110 -9
index 7685a81e7fb..0ae49aec333 100644
@@ -6,21 +6,26 @@
 
 import json
 import os
+import textwrap
+from pathlib import Path
 from oeqa.selftest.case import OESelftestTestCase
-from oeqa.utils.commands import bitbake, get_bb_var, runCmd
+from oeqa.utils.commands import bitbake, get_bb_var, get_bb_vars, runCmd
 
-class SPDXCheck(OESelftestTestCase):
 
+class SPDX22Check(OESelftestTestCase):
     @classmethod
     def setUpClass(cls):
-        super(SPDXCheck, cls).setUpClass()
+        super().setUpClass()
         bitbake("python3-spdx-tools-native")
         bitbake("-c addto_recipe_sysroot python3-spdx-tools-native")
 
     def check_recipe_spdx(self, high_level_dir, spdx_file, target_name):
-        config = """
-INHERIT += "create-spdx"
-"""
+        config = textwrap.dedent(
+            """\
+            INHERIT:remove = "create-spdx"
+            INHERIT += "create-spdx-2.2"
+            """
+        )
         self.write_config(config)
 
         deploy_dir = get_bb_var("DEPLOY_DIR")
@@ -29,7 +34,9 @@ INHERIT += "create-spdx"
         # qemux86-64 creates the directory qemux86_64
         machine_dir = machine_var.replace("-", "_")
 
-        full_file_path = os.path.join(deploy_dir, "spdx", spdx_version, machine_dir, high_level_dir, spdx_file)
+        full_file_path = os.path.join(
+            deploy_dir, "spdx", spdx_version, machine_dir, high_level_dir, spdx_file
+        )
 
         try:
             os.remove(full_file_path)
@@ -44,8 +51,13 @@ INHERIT += "create-spdx"
                 self.assertNotEqual(report, None)
                 self.assertNotEqual(report["SPDXID"], None)
 
-            python = os.path.join(get_bb_var('STAGING_BINDIR', 'python3-spdx-tools-native'), 'nativepython3')
-            validator = os.path.join(get_bb_var('STAGING_BINDIR', 'python3-spdx-tools-native'), 'pyspdxtools')
+            python = os.path.join(
+                get_bb_var("STAGING_BINDIR", "python3-spdx-tools-native"),
+                "nativepython3",
+            )
+            validator = os.path.join(
+                get_bb_var("STAGING_BINDIR", "python3-spdx-tools-native"), "pyspdxtools"
+            )
             result = runCmd("{} {} -i {}".format(python, validator, filename))
 
         self.assertExists(full_file_path)
@@ -53,3 +65,92 @@ INHERIT += "create-spdx"
 
     def test_spdx_base_files(self):
         self.check_recipe_spdx("packages", "base-files.spdx.json", "base-files")
+
+
+class SPDX3CheckBase(object):
+    """
+    Base class for checking SPDX 3 based tests
+    """
+
+    def check_spdx_file(self, filename):
+        import oe.spdx30
+
+        self.assertExists(filename)
+
+        # Read the file
+        objset = oe.spdx30.SHACLObjectSet()
+        with open(filename, "r") as f:
+            d = oe.spdx30.JSONLDDeserializer()
+            d.read(f, objset)
+
+        return objset
+
+    def check_recipe_spdx(self, target_name, spdx_path, *, task=None):
+        config = textwrap.dedent(
+            f"""\
+            INHERIT:remove = "create-spdx"
+            INHERIT += "{self.SPDX_CLASS}"
+            """
+        )
+        self.write_config(config)
+
+        if task:
+            bitbake(f"-c {task} {target_name}")
+        else:
+            bitbake(target_name)
+
+        filename = spdx_path.format(
+            **get_bb_vars(
+                [
+                    "DEPLOY_DIR_IMAGE",
+                    "DEPLOY_DIR_SPDX",
+                    "MACHINE",
+                    "MACHINE_ARCH",
+                    "SDKMACHINE",
+                    "SDK_DEPLOY",
+                    "SPDX_VERSION",
+                    "TOOLCHAIN_OUTPUTNAME",
+                ],
+                target_name,
+            )
+        )
+
+        return self.check_spdx_file(filename)
+
+    def check_objset_missing_ids(self, objset):
+        if objset.missing_ids:
+            self.assertTrue(
+                False,
+                "The following SPDXIDs are unresolved:\n  "
+                + "\n  ".join(objset.missing_ids),
+            )
+
+
+class SPDX30Check(SPDX3CheckBase, OESelftestTestCase):
+    SPDX_CLASS = "create-spdx-3.0"
+    SCHEMA_URL = "https://spdx.org/schema/3.0.0/spdx-json-schema.json"
+
+    def test_base_files(self):
+        self.check_recipe_spdx(
+            "base-files",
+            "{DEPLOY_DIR_SPDX}/{MACHINE_ARCH}/packages/base-files.spdx.json",
+        )
+
+    def test_core_image_minimal(self):
+        objset = self.check_recipe_spdx(
+            "core-image-minimal",
+            "{DEPLOY_DIR_IMAGE}/core-image-minimal-{MACHINE}.rootfs.spdx.json",
+        )
+
+        # Document should be fully linked
+        self.check_objset_missing_ids(objset)
+
+    def test_core_image_minimal_sdk(self):
+        objset = self.check_recipe_spdx(
+            "core-image-minimal",
+            "{SDK_DEPLOY}/{TOOLCHAIN_OUTPUTNAME}.spdx.json",
+            task="populate_sdk",
+        )
+
+        # Document should be fully linked
+        self.check_objset_missing_ids(objset)
Signature · 1 line
2.43.2
#53 lore raw

PATCH v4 8/10 classes-recipe: nospdx: Add class

In reply to Joshua Watt

Adds a class that allows recipes to opt out of generating SPDX

---
 meta/classes-recipe/nospdx.bbclass           | 13 +++++++++++++
 meta/recipes-core/meta/build-sysroots.bb     |  5 +----
 meta/recipes-core/meta/meta-world-pkgdata.bb |  3 +--
 3 files changed, 15 insertions(+), 6 deletions(-)
 create mode 100644 meta/classes-recipe/nospdx.bbclass
meta/classes-recipe/nospdx.bbclass new file +13 -0
new file mode 100644
index 00000000000..b20e28218be
@@ -0,0 +1,13 @@
+#
+# Copyright OpenEmbedded Contributors
+#
+# SPDX-License-Identifier: MIT
+#
+
+deltask do_collect_spdx_deps
+deltask do_create_spdx
+deltask do_create_spdx_runtime
+deltask do_create_package_spdx
+deltask do_create_rootfs_spdx
+deltask do_create_image_spdx
+deltask do_create_image_sbom
meta/recipes-core/meta/build-sysroots.bb +1 -4
index db05c111ab2..b0b8fb3c79a 100644
@@ -7,7 +7,7 @@ STANDALONE_SYSROOT_NATIVE = "${STAGING_DIR}/${BUILD_ARCH}"
 PACKAGE_ARCH = "${MACHINE_ARCH}"
 EXCLUDE_FROM_WORLD = "1"
 
-inherit nopackages
+inherit nopackages nospdx
 deltask fetch
 deltask unpack
 deltask patch
@@ -17,9 +17,6 @@ deltask configure
 deltask compile
 deltask install
 deltask populate_sysroot
-deltask create_spdx
-deltask collect_spdx_deps
-deltask create_runtime_spdx
 deltask recipe_qa
 
 do_build_warn () {
meta/recipes-core/meta/meta-world-pkgdata.bb +1 -2
index 0438bf61387..244175ddd44 100644
@@ -27,14 +27,13 @@ python do_collect_packagedata() {
     oe.copy_buildsystem.generate_locked_sigs(sigfile, d)
 }
 
+inherit nospdx
 deltask do_fetch
 deltask do_unpack
 deltask do_patch
 deltask do_configure
 deltask do_compile
 deltask do_install
-deltask do_create_spdx
-deltask do_create_spdx_runtime
 
 do_prepare_recipe_sysroot[deptask] = ""
 
Signature · 1 line
2.43.2
#54 lore raw

PATCH v4 9/10 classes/spdx-common: Move SPDX_SUPPLIER

In reply to Joshua Watt

Move the SPDX_SUPPLIER variable to create-spdx-2.2 since it's format only has meaning in SPDX 2.2 (SPDX 3 uses SPDX_PACKAGE_SUPPLIER with a different format)

---
 meta/classes/create-spdx-2.2.bbclass | 8 ++++++++
 meta/classes/spdx-common.bbclass     | 7 -------
 2 files changed, 8 insertions(+), 7 deletions(-)
meta/classes/create-spdx-2.2.bbclass +8 -0
index 3ebf92b5e12..99061320e53 100644
@@ -8,6 +8,14 @@ inherit spdx-common
 
 SPDX_VERSION = "2.2"
 
+SPDX_ORG ??= "OpenEmbedded ()"
+SPDX_SUPPLIER ??= "Organization: ${SPDX_ORG}"
+SPDX_SUPPLIER[doc] = "The SPDX PackageSupplier field for SPDX packages created from \
+    this recipe. For SPDX documents create using this class during the build, this \
+    is the contact information for the person or organization who is doing the \
+    build."
+
+
 def get_namespace(d, name):
     import uuid
     namespace_uuid = uuid.uuid5(uuid.NAMESPACE_DNS, d.getVar("SPDX_UUID_NAMESPACE"))
meta/classes/spdx-common.bbclass +0 -7
index 9dee325ba6b..6dfc1fd9e4c 100644
@@ -37,13 +37,6 @@ SPDX_LICENSES ??= "${COREBASE}/meta/files/spdx-licenses.json"
 
 SPDX_CUSTOM_ANNOTATION_VARS ??= ""
 
-SPDX_ORG ??= "OpenEmbedded ()"
-SPDX_SUPPLIER ??= "Organization: ${SPDX_ORG}"
-SPDX_SUPPLIER[doc] = "The SPDX PackageSupplier field for SPDX packages created from \
-    this recipe. For SPDX documents create using this class during the build, this \
-    is the contact information for the person or organization who is doing the \
-    build."
-
 def extract_licenses(filename):
     import re
 
Signature · 1 line
2.43.2
#55 lore raw

PATCH v4 10/10 Switch default spdx version to 3.0

In reply to Joshua Watt

Changes the default SPDX version to 3.0

---
 meta/classes/create-spdx.bbclass | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
meta/classes/create-spdx.bbclass +1 -1
index 19c6c0ff0b9..b604973ae0a 100644
@@ -5,4 +5,4 @@
 #
 # Include this class when you don't care what version of SPDX you get; it will
 # be updated to the latest stable version that is supported
-inherit create-spdx-2.2
+inherit create-spdx-3.0
Signature · 1 line
2.43.2
#56 lore raw
In reply to Joshua Watt

Hello,

I believe this is the cause of the following failures:

https://autobuilder.yoctoproject.org/typhoon/#/builders/117/builds/4986/steps/12/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/117/builds/4989/steps/13/logs/stdio

I'm not sure why they don't have any diffoscope output.

Quoted 63 lines · Joshua Watt

This patch series add support for SPDX 3.0 and sets it as the default. Currently it is not possible to have SPDX 2.2 and SPDX 3.0 enabled at the same time

v2: Added tests and addressed feedback
v3: Fixed several oe-selftest and build failures
v4: Fixed silly typo mistake in staging.bbclass

Joshua Watt (10):
classes-recipe/image: Add image file manifest classes-recipe/baremetal-image: Add image file manifest classes/create-spdx-3.0: Add classes classes-global/staging: Exclude do_create_spdx from automatic sysroot
extension
binutils-cross-testsuite: Rename to binutils-testsuite classes-recipe/image_types: Add SPDX_IMAGE_PURPOSE to images
selftest: spdx: Add SPDX 3.0 test cases
classes-recipe: nospdx: Add class
classes/spdx-common: Move SPDX_SUPPLIER
Switch default spdx version to 3.0

 meta/classes-global/mirrors.bbclass           |    2 +-
 meta/classes-global/staging.bbclass           |    9 +-
 meta/classes-recipe/baremetal-image.bbclass   |   30 +-
 meta/classes-recipe/image.bbclass             |   58 +
 meta/classes-recipe/image_types.bbclass       |    2 +
 meta/classes-recipe/image_types_wic.bbclass   |    1 +
 meta/classes-recipe/nospdx.bbclass            |   13 +
 meta/classes-recipe/packagegroup.bbclass      |    2 +
 meta/classes/create-spdx-2.2.bbclass          |    8 +
 meta/classes/create-spdx-3.0.bbclass          | 1179 ++++
 meta/classes/create-spdx-image-3.0.bbclass    |  224 +
 meta/classes/create-spdx.bbclass              |    2 +-
 meta/classes/spdx-common.bbclass              |   13 +-
 meta/conf/distro/include/maintainers.inc      |    2 +-
 meta/lib/oe/sbom30.py                         | 1122 +++
 meta/lib/oe/spdx30.py                         | 5996 +++++++++++++++++
 meta/lib/oeqa/selftest/cases/binutils.py      |    2 +-
 meta/lib/oeqa/selftest/cases/spdx.py          |  119 +-
 meta/recipes-core/meta/build-sysroots.bb      |    5 +-
 meta/recipes-core/meta/meta-world-pkgdata.bb  |    3 +-
 ...ite_2.42.bb => binutils-testsuite_2.42.bb} |    3 +
 21 files changed, 8765 insertions(+), 30 deletions(-)
 create mode 100644 meta/classes-recipe/nospdx.bbclass
 create mode 100644 meta/classes/create-spdx-3.0.bbclass
 create mode 100644 meta/classes/create-spdx-image-3.0.bbclass
 create mode 100644 meta/lib/oe/sbom30.py
 create mode 100644 meta/lib/oe/spdx30.py
 rename meta/recipes-devtools/binutils/{binutils-cross-testsuite_2.42.bb => binutils-testsuite_2.42.bb} (95%)
Signature · 11 lines
2.43.2



-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#201102): https://lists.openembedded.org/g/openembedded-core/message/201102
Mute This Topic: https://lists.openembedded.org/mt/106856868/3617179
Group Owner: [email protected]
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [[email protected]]
-=-=-=-=-=-=-=-=-=-=-=-
Signature · 3 lines
Alexandre Belloni, co-owner and COO, Bootlin
Embedded Linux and Kernel engineering
https://bootlin.com
#57 lore raw
Quoted 8 lines · Alexandre Belloni via lists.openembedded.org

Hello,

I believe this is the cause of the following failures:

https://autobuilder.yoctoproject.org/typhoon/#/builders/117/builds/4986/steps/12/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/117/builds/4989/steps/13/logs/stdio

I'm not sure why they don't have any diffoscope output.

"The following rpm packages are missing and not in exclusion list:"

there is no diffoscope as they don't exist at all.

I suspect a cleandirs corrupted the build by deleting things before they were added to the sstate output.

Cheers,

Richard

#58 lore raw

v5 cover Add SPDX 3.0 support

In reply to Joshua Watt

This patch series add support for SPDX 3.0 and sets it as the default. Currently it is not possible to have SPDX 2.2 and SPDX 3.0 enabled at the same time

v2: Added tests and addressed feedback
v3: Fixed several oe-selftest and build failures
v4: Fixed silly typo mistake in staging.bbclass
v5: Reworked to make SPDX 3 output reproducible by default. Variables
that introduce non-reproducible output are documented as such.

Joshua Watt (8):
classes-recipe/image: Add image file manifest classes-recipe/baremetal-image: Add image file manifest classes/create-spdx-3.0: Add classes classes-global/staging: Exclude do_create_spdx from automatic sysroot
extension
classes-recipe/image_types: Add SPDX_IMAGE_PURPOSE to images
selftest: spdx: Add SPDX 3.0 test cases
classes-recipe: nospdx: Add class
Switch default spdx version to 3.0

 meta/classes-global/staging.bbclass          |    9 +-
 meta/classes-recipe/baremetal-image.bbclass  |   32 +-
 meta/classes-recipe/image.bbclass            |   58 +
 meta/classes-recipe/image_types.bbclass      |    2 +
 meta/classes-recipe/image_types_wic.bbclass  |    1 +
 meta/classes-recipe/nospdx.bbclass           |   13 +
 meta/classes-recipe/packagegroup.bbclass     |    2 +
 meta/classes/create-spdx-3.0.bbclass         | 1231 ++++
 meta/classes/create-spdx-image-3.0.bbclass   |  224 +
 meta/classes/create-spdx.bbclass             |    2 +-
 meta/classes/spdx-common.bbclass             |    6 +-
 meta/lib/oe/sbom30.py                        | 1138 ++++
 meta/lib/oe/spdx30.py                        | 5996 ++++++++++++++++++
 meta/lib/oeqa/selftest/cases/spdx.py         |  118 +-
 meta/recipes-core/meta/build-sysroots.bb     |    5 +-
 meta/recipes-core/meta/meta-world-pkgdata.bb |    3 +-
 16 files changed, 8819 insertions(+), 21 deletions(-)
 create mode 100644 meta/classes-recipe/nospdx.bbclass
 create mode 100644 meta/classes/create-spdx-3.0.bbclass
 create mode 100644 meta/classes/create-spdx-image-3.0.bbclass
 create mode 100644 meta/lib/oe/sbom30.py
 create mode 100644 meta/lib/oe/spdx30.py
Signature · 1 line
2.45.2
#59 lore raw

PATCH v5 1/8 classes-recipe/image: Add image file manifest

Downstream tasks may want to know what image files were written by the do_image family of tasks (e.g. SPDX) so have each task write out a manifest file that describes the files it produced, then aggregate them in do_image_complete

---
 meta/classes-recipe/image.bbclass | 58 +++++++++++++++++++++++++++++++
 1 file changed, 58 insertions(+)
meta/classes-recipe/image.bbclass +58 -0
index 28be6c63623..32bafcdbbc3 100644
@@ -88,6 +88,11 @@ PACKAGE_INSTALL_ATTEMPTONLY ?= "${FEATURE_INSTALL_OPTIONAL}"
 
 IMGDEPLOYDIR = "${WORKDIR}/deploy-${PN}-image-complete"
 
+IMGMANIFESTDIR = "${WORKDIR}/image-task-manifest"
+
+IMAGE_OUTPUT_MANIFEST_DIR = "${WORKDIR}/deploy-image-output-manifest"
+IMAGE_OUTPUT_MANIFEST = "${IMAGE_OUTPUT_MANIFEST_DIR}/manifest.json"
+
 # Images are generally built explicitly, do not need to be part of world.
 EXCLUDE_FROM_WORLD = "1"
 
@@ -277,14 +282,28 @@ fakeroot python do_image () {
     execute_pre_post_process(d, pre_process_cmds)
 }
 do_image[dirs] = "${TOPDIR}"
+do_image[cleandirs] += "${IMGMANIFESTDIR}"
 addtask do_image after do_rootfs
 
 fakeroot python do_image_complete () {
     from oe.utils import execute_pre_post_process
+    from pathlib import Path
+    import json
 
     post_process_cmds = d.getVar("IMAGE_POSTPROCESS_COMMAND")
 
     execute_pre_post_process(d, post_process_cmds)
+
+    image_manifest_dir = Path(d.getVar('IMGMANIFESTDIR'))
+
+    data = []
+
+    for manifest_path in image_manifest_dir.glob("*.json"):
+        with manifest_path.open("r") as f:
+            data.extend(json.load(f))
+
+    with open(d.getVar("IMAGE_OUTPUT_MANIFEST"), "w") as f:
+        json.dump(data, f)
 }
 do_image_complete[dirs] = "${TOPDIR}"
 SSTATETASKS += "do_image_complete"
@@ -292,6 +311,8 @@ SSTATE_SKIP_CREATION:task-image-complete = '1'
 do_image_complete[sstate-inputdirs] = "${IMGDEPLOYDIR}"
 do_image_complete[sstate-outputdirs] = "${DEPLOY_DIR_IMAGE}"
 do_image_complete[stamp-extra-info] = "${MACHINE_ARCH}"
+do_image_complete[sstate-plaindirs] += "${IMAGE_OUTPUT_MANIFEST_DIR}"
+do_image_complete[dirs] += "${IMAGE_OUTPUT_MANIFEST_DIR}"
 addtask do_image_complete after do_image before do_build
 python do_image_complete_setscene () {
     sstate_setscene(d)
@@ -507,12 +528,14 @@ python () {
         d.setVar(task, '\n'.join(cmds))
         d.setVarFlag(task, 'func', '1')
         d.setVarFlag(task, 'fakeroot', '1')
+        d.setVarFlag(task, 'imagetype', t)
 
         d.appendVarFlag(task, 'prefuncs', ' ' + debug + ' set_image_size')
         d.prependVarFlag(task, 'postfuncs', 'create_symlinks ')
         d.appendVarFlag(task, 'subimages', ' ' + ' '.join(subimages))
         d.appendVarFlag(task, 'vardeps', ' ' + ' '.join(vardeps))
         d.appendVarFlag(task, 'vardepsexclude', ' DATETIME DATE ' + ' '.join(vardepsexclude))
+        d.appendVarFlag(task, 'postfuncs', ' write_image_output_manifest')
 
         bb.debug(2, "Adding task %s before %s, after %s" % (task, 'do_image_complete', after))
         bb.build.addtask(task, 'do_image_complete', after, d)
@@ -610,6 +633,41 @@ python create_symlinks() {
             bb.note("Skipping symlink, source does not exist: %s -> %s" % (dst, src))
 }
 
+python write_image_output_manifest() {
+    import json
+    from pathlib import Path
+
+    taskname = d.getVar("BB_CURRENTTASK")
+    image_deploy_dir = Path(d.getVar('IMGDEPLOYDIR'))
+    image_manifest_dir = Path(d.getVar('IMGMANIFESTDIR'))
+    manifest_path = image_manifest_dir / ("do_" + d.getVar("BB_CURRENTTASK") + ".json")
+
+    image_name = d.getVar("IMAGE_NAME")
+    image_basename = d.getVar("IMAGE_BASENAME")
+    machine = d.getVar("MACHINE")
+
+    subimages = (d.getVarFlag("do_" + taskname, 'subimages', False) or "").split()
+    imagetype = d.getVarFlag("do_" + taskname, 'imagetype', False)
+
+    data = {
+        "taskname": taskname,
+        "imagetype": imagetype,
+        "images": []
+    }
+
+    for type in subimages:
+        image_filename = image_name + "." + type
+        image_path = image_deploy_dir / image_filename
+        if not image_path.exists():
+            continue
+        data["images"].append({
+            "filename": image_filename,
+        })
+
+    with manifest_path.open("w") as f:
+        json.dump([data], f)
+}
+
 MULTILIBRE_ALLOW_REP += "${base_bindir} ${base_sbindir} ${bindir} ${sbindir} ${libexecdir} ${sysconfdir} ${nonarch_base_libdir}/udev /lib/modules/[^/]*/modules.*"
 MULTILIB_CHECK_FILE = "${WORKDIR}/multilib_check.py"
 MULTILIB_TEMP_ROOTFS = "${WORKDIR}/multilib"
Signature · 1 line
2.45.2