Backport SPDX support
cover Backport SPDX support
Backports support to creates SPDX during the build from the master branch, with a few changes to make it compatible with dunfell. Primarily, the SPDX data is compressed with gzip instead of zstd to prevent needing the zstd host tools on the older distros supported by dunfell
Joshua Watt (5):
classes/create-spdx: Backport classes/package: Add extended packaged data
licenses: Add GPL+ licenses to map
create-spdx: Use gzip for compression classes/package: Use gzip for extended package data
meta/classes/create-spdx-2.2.bbclass | 1067 +++++ meta/classes/create-spdx.bbclass | 8 + meta/classes/package.bbclass | 39 +- meta/conf/licenses.conf | 7 + meta/files/spdx-licenses.json | 5937 ++++++++++++++++++++++++++ meta/lib/oe/packagedata.py | 11 + meta/lib/oe/sbom.py | 84 + meta/lib/oe/spdx.py | 357 ++ 8 files changed, 7507 insertions(+), 3 deletions(-) create mode 100644 meta/classes/create-spdx-2.2.bbclass create mode 100644 meta/classes/create-spdx.bbclass create mode 100644 meta/files/spdx-licenses.json create mode 100644 meta/lib/oe/sbom.py create mode 100644 meta/lib/oe/spdx.py
Signature · 1 line
2.33.0
PATCH 1/5 classes/create-spdx: Backport
Backports the create-spdx classes from the latest versions on master. This backport is a simple copy with no modifications, as its too complex to cherry-pick all the corresponding changes. This will give an appropriate base commit for subsequent changes and if necessary additional backport cherry-picks from master in the future.
--- meta/classes/create-spdx-2.2.bbclass | 1069 +++++ meta/classes/create-spdx.bbclass | 8 + meta/files/spdx-licenses.json | 5937 ++++++++++++++++++++++++++ meta/lib/oe/sbom.py | 84 + meta/lib/oe/spdx.py | 357 ++ 5 files changed, 7455 insertions(+) create mode 100644 meta/classes/create-spdx-2.2.bbclass create mode 100644 meta/classes/create-spdx.bbclass create mode 100644 meta/files/spdx-licenses.json create mode 100644 meta/lib/oe/sbom.py create mode 100644 meta/lib/oe/spdx.py
meta/classes/create-spdx-2.2.bbclass new file +1069 -0
@@ -0,0 +1,1069 @@ +# +# Copyright OpenEmbedded Contributors +# +# SPDX-License-Identifier: GPL-2.0-only +# + +DEPLOY_DIR_SPDX ??= "${DEPLOY_DIR}/spdx/${MACHINE}" + +# The product name that the CVE database uses. Defaults to BPN, but may need to +# be overriden per recipe (for example tiff.bb sets CVE_PRODUCT=libtiff). +CVE_PRODUCT ??= "${BPN}" +CVE_VERSION ??= "${PV}" + +SPDXDIR ??= "${WORKDIR}/spdx" +SPDXDEPLOY = "${SPDXDIR}/deploy" +SPDXWORK = "${SPDXDIR}/work" +SPDXIMAGEWORK = "${SPDXDIR}/image-work" +SPDXSDKWORK = "${SPDXDIR}/sdk-work" + +SPDX_TOOL_NAME ??= "oe-spdx-creator" +SPDX_TOOL_VERSION ??= "1.0" + +SPDXRUNTIMEDEPLOY = "${SPDXDIR}/runtime-deploy" + +SPDX_INCLUDE_SOURCES ??= "0" +SPDX_ARCHIVE_SOURCES ??= "0" +SPDX_ARCHIVE_PACKAGED ??= "0" + +SPDX_UUID_NAMESPACE ??= "sbom.openembedded.org" +SPDX_NAMESPACE_PREFIX ??= "http://spdx.org/spdxdoc" +SPDX_PRETTY ??= "0" + +SPDX_LICENSES ??= "${COREBASE}/meta/files/spdx-licenses.json" + +SPDX_CUSTOM_ANNOTATION_VARS ??= "" + +SPDX_ORG ??= "OpenEmbedded ()" +SPDX_SUPPLIER ??= "Organization: ${SPDX_ORG}" +SPDX_SUPPLIER[doc] = "The SPDX PackageSupplier field for SPDX packages created from \ + this recipe. For SPDX documents create using this class during the build, this \ + is the contact information for the person or organization who is doing the \ + build." + +def extract_licenses(filename): + import re + + lic_regex = re.compile(rb'^\W*SPDX-License-Identifier:\s*([ \w\d.()+-]+?)(?:\s+\W*)?$', re.MULTILINE) + + try: + with open(filename, 'rb') as f: + size = min(15000, os.stat(filename).st_size) + txt = f.read(size) + licenses = re.findall(lic_regex, txt) + if licenses: + ascii_licenses = [lic.decode('ascii') for lic in licenses] + return ascii_licenses + except Exception as e: + bb.warn(f"Exception reading {filename}: {e}") + return None + +def get_doc_namespace(d, doc): + import uuid + namespace_uuid = uuid.uuid5(uuid.NAMESPACE_DNS, d.getVar("SPDX_UUID_NAMESPACE")) + return "%s/%s-%s" % (d.getVar("SPDX_NAMESPACE_PREFIX"), doc.name, str(uuid.uuid5(namespace_uuid, doc.name))) + +def create_annotation(d, comment): + from datetime import datetime, timezone + + creation_time = datetime.now(tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + annotation = oe.spdx.SPDXAnnotation() + annotation.annotationDate = creation_time + annotation.annotationType = "OTHER" + annotation.annotator = "Tool: %s - %s" % (d.getVar("SPDX_TOOL_NAME"), d.getVar("SPDX_TOOL_VERSION")) + annotation.comment = comment + return annotation + +def recipe_spdx_is_native(d, recipe): + return any(a.annotationType == "OTHER" and + a.annotator == "Tool: %s - %s" % (d.getVar("SPDX_TOOL_NAME"), d.getVar("SPDX_TOOL_VERSION")) and + a.comment == "isNative" for a in recipe.annotations) + +def is_work_shared_spdx(d): + return bb.data.inherits_class('kernel', d) or ('work-shared' in d.getVar('WORKDIR')) + +def get_json_indent(d): + if d.getVar("SPDX_PRETTY") == "1": + return 2 + return None + +python() { + import json + if d.getVar("SPDX_LICENSE_DATA"): + return + + with open(d.getVar("SPDX_LICENSES"), "r") as f: + data = json.load(f) + # Transform the license array to a dictionary + data["licenses"] = {l["licenseId"]: l for l in data["licenses"]} + d.setVar("SPDX_LICENSE_DATA", data) +} + +def convert_license_to_spdx(lic, document, d, existing={}): + from pathlib import Path + import oe.spdx + + license_data = d.getVar("SPDX_LICENSE_DATA") + extracted = {} + + def add_extracted_license(ident, name): + nonlocal document + + if name in extracted: + return + + extracted_info = oe.spdx.SPDXExtractedLicensingInfo() + extracted_info.name = name + extracted_info.licenseId = ident + extracted_info.extractedText = None + + if name == "PD": + # Special-case this. + extracted_info.extractedText = "Software released to the public domain" + else: + # Seach for the license in COMMON_LICENSE_DIR and LICENSE_PATH + for directory in [d.getVar('COMMON_LICENSE_DIR')] + (d.getVar('LICENSE_PATH') or '').split(): + try: + with (Path(directory) / name).open(errors="replace") as f: + extracted_info.extractedText = f.read() + break + except FileNotFoundError: + pass + if extracted_info.extractedText is None: + # If it's not SPDX or PD, then NO_GENERIC_LICENSE must be set + filename = d.getVarFlag('NO_GENERIC_LICENSE', name) + if filename: + filename = d.expand("${S}/" + filename) + with open(filename, errors="replace") as f: + extracted_info.extractedText = f.read() + else: + bb.error("Cannot find any text for license %s" % name) + + extracted[name] = extracted_info + document.hasExtractedLicensingInfos.append(extracted_info) + + def convert(l): + if l == "(" or l == ")": + return l + + if l == "&": + return "AND" + + if l == "|": + return "OR" + + if l == "CLOSED": + return "NONE" + + spdx_license = d.getVarFlag("SPDXLICENSEMAP", l) or l + if spdx_license in license_data["licenses"]: + return spdx_license + + try: + spdx_license = existing[l] + except KeyError: + spdx_license = "LicenseRef-" + l + add_extracted_license(spdx_license, l) + + return spdx_license + + lic_split = lic.replace("(", " ( ").replace(")", " ) ").split() + + return ' '.join(convert(l) for l in lic_split) + +def process_sources(d): + pn = d.getVar('PN') + assume_provided = (d.getVar("ASSUME_PROVIDED") or "").split() + if pn in assume_provided: + for p in d.getVar("PROVIDES").split(): + if p != pn: + pn = p + break + + # glibc-locale: do_fetch, do_unpack and do_patch tasks have been deleted, + # so avoid archiving source here. + if pn.startswith('glibc-locale'): + return False + if d.getVar('PN') == "libtool-cross": + return False + if d.getVar('PN') == "libgcc-initial": + return False + if d.getVar('PN') == "shadow-sysroot": + return False + + # We just archive gcc-source for all the gcc related recipes + if d.getVar('BPN') in ['gcc', 'libgcc']: + bb.debug(1, 'spdx: There is bug in scan of %s is, do nothing' % pn) + return False + + return True + + +def add_package_files(d, doc, spdx_pkg, topdir, get_spdxid, get_types, *, archive=None, ignore_dirs=[], ignore_top_level_dirs=[]): + from pathlib import Path + import oe.spdx + import hashlib + + source_date_epoch = d.getVar("SOURCE_DATE_EPOCH") + if source_date_epoch: + source_date_epoch = int(source_date_epoch) + + sha1s = [] + spdx_files = [] + + file_counter = 1 + for subdir, dirs, files in os.walk(topdir): + dirs[:] = [d for d in dirs if d not in ignore_dirs] + if subdir == str(topdir): + dirs[:] = [d for d in dirs if d not in ignore_top_level_dirs] + + for file in files: + filepath = Path(subdir) / file + filename = str(filepath.relative_to(topdir)) + + if not filepath.is_symlink() and filepath.is_file(): + spdx_file = oe.spdx.SPDXFile() + spdx_file.SPDXID = get_spdxid(file_counter) + for t in get_types(filepath): + spdx_file.fileTypes.append(t) + spdx_file.fileName = filename + + if archive is not None: + with filepath.open("rb") as f: + info = archive.gettarinfo(fileobj=f) + info.name = filename + info.uid = 0 + info.gid = 0 + info.uname = "root" + info.gname = "root" + + if source_date_epoch is not None and info.mtime > source_date_epoch: + info.mtime = source_date_epoch + + archive.addfile(info, f) + + sha1 = bb.utils.sha1_file(filepath) + sha1s.append(sha1) + spdx_file.checksums.append(oe.spdx.SPDXChecksum( + algorithm="SHA1", + checksumValue=sha1, + )) + spdx_file.checksums.append(oe.spdx.SPDXChecksum( + algorithm="SHA256", + checksumValue=bb.utils.sha256_file(filepath), + )) + + if "SOURCE" in spdx_file.fileTypes: + extracted_lics = extract_licenses(filepath) + if extracted_lics: + spdx_file.licenseInfoInFiles = extracted_lics + + doc.files.append(spdx_file) + doc.add_relationship(spdx_pkg, "CONTAINS", spdx_file) + spdx_pkg.hasFiles.append(spdx_file.SPDXID) + + spdx_files.append(spdx_file) + + file_counter += 1 + + sha1s.sort() + verifier = hashlib.sha1() + for v in sha1s: + verifier.update(v.encode("utf-8")) + spdx_pkg.packageVerificationCode.packageVerificationCodeValue = verifier.hexdigest() + + return spdx_files + + +def add_package_sources_from_debug(d, package_doc, spdx_package, package, package_files, sources): + from pathlib import Path + import hashlib + import oe.packagedata + import oe.spdx + + debug_search_paths = [ + Path(d.getVar('PKGD')), + Path(d.getVar('STAGING_DIR_TARGET')), + Path(d.getVar('STAGING_DIR_NATIVE')), + Path(d.getVar('STAGING_KERNEL_DIR')), + ] + + pkg_data = oe.packagedata.read_subpkgdata_extended(package, d) + + if pkg_data is None: + return + + for file_path, file_data in pkg_data["files_info"].items(): + if not "debugsrc" in file_data: + continue + + for pkg_file in package_files: + if file_path.lstrip("/") == pkg_file.fileName.lstrip("/"): + break + else: + bb.fatal("No package file found for %s" % str(file_path)) + continue + + for debugsrc in file_data["debugsrc"]: + ref_id = "NOASSERTION" + for search in debug_search_paths: + if debugsrc.startswith("/usr/src/kernel"): + debugsrc_path = search / debugsrc.replace('/usr/src/kernel/', '') + else: + debugsrc_path = search / debugsrc.lstrip("/") + if not debugsrc_path.exists(): + continue + + file_sha256 = bb.utils.sha256_file(debugsrc_path) + + if file_sha256 in sources: + source_file = sources[file_sha256] + + doc_ref = package_doc.find_external_document_ref(source_file.doc.documentNamespace) + if doc_ref is None: + doc_ref = oe.spdx.SPDXExternalDocumentRef() + doc_ref.externalDocumentId = "DocumentRef-dependency-" + source_file.doc.name + doc_ref.spdxDocument = source_file.doc.documentNamespace + doc_ref.checksum.algorithm = "SHA1" + doc_ref.checksum.checksumValue = source_file.doc_sha1 + package_doc.externalDocumentRefs.append(doc_ref) + + ref_id = "%s:%s" % (doc_ref.externalDocumentId, source_file.file.SPDXID) + else: + bb.debug(1, "Debug source %s with SHA256 %s not found in any dependency" % (str(debugsrc_path), file_sha256)) + break + else: + bb.debug(1, "Debug source %s not found" % debugsrc) + + package_doc.add_relationship(pkg_file, "GENERATED_FROM", ref_id, comment=debugsrc) + +def collect_dep_recipes(d, doc, spdx_recipe): + from pathlib import Path + import oe.sbom + import oe.spdx + + deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX")) + + dep_recipes = [] + taskdepdata = d.getVar("BB_TASKDEPDATA", False) + deps = sorted(set( + dep[0] for dep in taskdepdata.values() if + dep[1] == "do_create_spdx" and dep[0] != d.getVar("PN") + )) + for dep_pn in deps: + dep_recipe_path = deploy_dir_spdx / "recipes" / ("recipe-%s.spdx.json" % dep_pn) + + spdx_dep_doc, spdx_dep_sha1 = oe.sbom.read_doc(dep_recipe_path) + + for pkg in spdx_dep_doc.packages: + if pkg.name == dep_pn: + spdx_dep_recipe = pkg + break + else: + continue + + dep_recipes.append(oe.sbom.DepRecipe(spdx_dep_doc, spdx_dep_sha1, spdx_dep_recipe)) + + dep_recipe_ref = oe.spdx.SPDXExternalDocumentRef() + dep_recipe_ref.externalDocumentId = "DocumentRef-dependency-" + spdx_dep_doc.name + dep_recipe_ref.spdxDocument = spdx_dep_doc.documentNamespace + dep_recipe_ref.checksum.algorithm = "SHA1" + dep_recipe_ref.checksum.checksumValue = spdx_dep_sha1 + + doc.externalDocumentRefs.append(dep_recipe_ref) + + doc.add_relationship( + "%s:%s" % (dep_recipe_ref.externalDocumentId, spdx_dep_recipe.SPDXID), + "BUILD_DEPENDENCY_OF", + spdx_recipe + ) + + return dep_recipes + +collect_dep_recipes[vardepsexclude] += "BB_TASKDEPDATA" +collect_dep_recipes[vardeps] += "DEPENDS" + +def collect_dep_sources(d, dep_recipes): + import oe.sbom + + sources = {} + for dep in dep_recipes: + # Don't collect sources from native recipes as they + # match non-native sources also. + if recipe_spdx_is_native(d, dep.recipe): + continue + recipe_files = set(dep.recipe.hasFiles) + + for spdx_file in dep.doc.files: + if spdx_file.SPDXID not in recipe_files: + continue + + if "SOURCE" in spdx_file.fileTypes: + for checksum in spdx_file.checksums: + if checksum.algorithm == "SHA256": + sources[checksum.checksumValue] = oe.sbom.DepSource(dep.doc, dep.doc_sha1, dep.recipe, spdx_file) + break + + return sources + +def add_download_packages(d, doc, recipe): + import os.path + from bb.fetch2 import decodeurl, CHECKSUM_LIST + import bb.process + import oe.spdx + import oe.sbom + + for download_idx, src_uri in enumerate(d.getVar('SRC_URI').split()): + f = bb.fetch2.FetchData(src_uri, d) + + for name in f.names: + package = oe.spdx.SPDXPackage() + package.name = "%s-source-%d" % (d.getVar("PN"), download_idx + 1) + package.SPDXID = oe.sbom.get_download_spdxid(d, download_idx + 1) + + if f.type == "file": + continue + + uri = f.type + proto = getattr(f, "proto", None) + if proto is not None: + uri = uri + "+" + proto + uri = uri + "://" + f.host + f.path + + if f.method.supports_srcrev(): + uri = uri + "@" + f.revisions[name] + + if f.method.supports_checksum(f): + for checksum_id in CHECKSUM_LIST: + if checksum_id.upper() not in oe.spdx.SPDXPackage.ALLOWED_CHECKSUMS: + continue + + expected_checksum = getattr(f, "%s_expected" % checksum_id) + if expected_checksum is None: + continue + + c = oe.spdx.SPDXChecksum() + c.algorithm = checksum_id.upper() + c.checksumValue = expected_checksum + package.checksums.append(c) + + package.downloadLocation = uri + doc.packages.append(package) + doc.add_relationship(doc, "DESCRIBES", package) + # In the future, we might be able to do more fancy dependencies, + # but this should be sufficient for now + doc.add_relationship(package, "BUILD_DEPENDENCY_OF", recipe) + +python do_create_spdx() { + from datetime import datetime, timezone + import oe.sbom + import oe.spdx + import uuid + from pathlib import Path + from contextlib import contextmanager + import oe.cve_check + + @contextmanager + def optional_tarfile(name, guard, mode="w"): + import tarfile + import bb.compress.zstd + + num_threads = int(d.getVar("BB_NUMBER_THREADS")) + + if guard: + name.parent.mkdir(parents=True, exist_ok=True) + with bb.compress.zstd.open(name, mode=mode + "b", num_threads=num_threads) as f: + with tarfile.open(fileobj=f, mode=mode + "|") as tf: + yield tf + else: + yield None + + + deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX")) + spdx_workdir = Path(d.getVar("SPDXWORK")) + include_sources = d.getVar("SPDX_INCLUDE_SOURCES") == "1" + archive_sources = d.getVar("SPDX_ARCHIVE_SOURCES") == "1" + archive_packaged = d.getVar("SPDX_ARCHIVE_PACKAGED") == "1" + + creation_time = datetime.now(tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + doc = oe.spdx.SPDXDocument() + + doc.name = "recipe-" + d.getVar("PN") + doc.documentNamespace = get_doc_namespace(d, doc) + doc.creationInfo.created = creation_time + doc.creationInfo.comment = "This document was created by analyzing recipe files during the build." + doc.creationInfo.licenseListVersion = d.getVar("SPDX_LICENSE_DATA")["licenseListVersion"] + doc.creationInfo.creators.append("Tool: OpenEmbedded Core create-spdx.bbclass") + doc.creationInfo.creators.append("Organization: %s" % d.getVar("SPDX_ORG")) + doc.creationInfo.creators.append("Person: N/A ()") + + recipe = oe.spdx.SPDXPackage() + recipe.name = d.getVar("PN") + recipe.versionInfo = d.getVar("PV") + recipe.SPDXID = oe.sbom.get_recipe_spdxid(d) + recipe.supplier = d.getVar("SPDX_SUPPLIER") + if bb.data.inherits_class("native", d) or bb.data.inherits_class("cross", d): + recipe.annotations.append(create_annotation(d, "isNative")) + + homepage = d.getVar("HOMEPAGE") + if homepage: + recipe.homepage = homepage + + license = d.getVar("LICENSE") + if license: + recipe.licenseDeclared = convert_license_to_spdx(license, doc, d) + + summary = d.getVar("SUMMARY") + if summary: + recipe.summary = summary + + description = d.getVar("DESCRIPTION") + if description: + recipe.description = description + + if d.getVar("SPDX_CUSTOM_ANNOTATION_VARS"): + for var in d.getVar('SPDX_CUSTOM_ANNOTATION_VARS').split(): + recipe.annotations.append(create_annotation(d, var + "=" + d.getVar(var))) + + # Some CVEs may be patched during the build process without incrementing the version number, + # so querying for CVEs based on the CPE id can lead to false positives. To account for this, + # save the CVEs fixed by patches to source information field in the SPDX. + patched_cves = oe.cve_check.get_patched_cves(d) + patched_cves = list(patched_cves) + patched_cves = ' '.join(patched_cves) + if patched_cves: + recipe.sourceInfo = "CVEs fixed: " + patched_cves + + cpe_ids = oe.cve_check.get_cpe_ids(d.getVar("CVE_PRODUCT"), d.getVar("CVE_VERSION")) + if cpe_ids: + for cpe_id in cpe_ids: + cpe = oe.spdx.SPDXExternalReference() + cpe.referenceCategory = "SECURITY" + cpe.referenceType = "http://spdx.org/rdf/references/cpe23Type" + cpe.referenceLocator = cpe_id + recipe.externalRefs.append(cpe) + + doc.packages.append(recipe) + doc.add_relationship(doc, "DESCRIBES", recipe) + + add_download_packages(d, doc, recipe) + + if process_sources(d) and include_sources: + recipe_archive = deploy_dir_spdx / "recipes" / (doc.name + ".tar.zst") + with optional_tarfile(recipe_archive, archive_sources) as archive: + spdx_get_src(d) + + add_package_files( + d, + doc, + recipe, + spdx_workdir, + lambda file_counter: "SPDXRef-SourceFile-%s-%d" % (d.getVar("PN"), file_counter), + lambda filepath: ["SOURCE"], + ignore_dirs=[".git"], + ignore_top_level_dirs=["temp"], + archive=archive, + ) + + if archive is not None: + recipe.packageFileName = str(recipe_archive.name) + + dep_recipes = collect_dep_recipes(d, doc, recipe) + + doc_sha1 = oe.sbom.write_doc(d, doc, "recipes", indent=get_json_indent(d)) + dep_recipes.append(oe.sbom.DepRecipe(doc, doc_sha1, recipe)) + + recipe_ref = oe.spdx.SPDXExternalDocumentRef() + recipe_ref.externalDocumentId = "DocumentRef-recipe-" + recipe.name + recipe_ref.spdxDocument = doc.documentNamespace + recipe_ref.checksum.algorithm = "SHA1" + recipe_ref.checksum.checksumValue = doc_sha1 + + sources = collect_dep_sources(d, dep_recipes) + found_licenses = {license.name:recipe_ref.externalDocumentId + ":" + license.licenseId for license in doc.hasExtractedLicensingInfos} + + if not recipe_spdx_is_native(d, recipe): + bb.build.exec_func("read_subpackage_metadata", d) + + pkgdest = Path(d.getVar("PKGDEST")) + for package in d.getVar("PACKAGES").split(): + if not oe.packagedata.packaged(package, d): + continue + + package_doc = oe.spdx.SPDXDocument() + pkg_name = d.getVar("PKG:%s" % package) or package + package_doc.name = pkg_name + package_doc.documentNamespace = get_doc_namespace(d, package_doc) + package_doc.creationInfo.created = creation_time + package_doc.creationInfo.comment = "This document was created by analyzing packages created during the build." + package_doc.creationInfo.licenseListVersion = d.getVar("SPDX_LICENSE_DATA")["licenseListVersion"] + package_doc.creationInfo.creators.append("Tool: OpenEmbedded Core create-spdx.bbclass") + package_doc.creationInfo.creators.append("Organization: %s" % d.getVar("SPDX_ORG")) + package_doc.creationInfo.creators.append("Person: N/A ()") + package_doc.externalDocumentRefs.append(recipe_ref) + + package_license = d.getVar("LICENSE:%s" % package) or d.getVar("LICENSE") + + spdx_package = oe.spdx.SPDXPackage() + + spdx_package.SPDXID = oe.sbom.get_package_spdxid(pkg_name) + spdx_package.name = pkg_name + spdx_package.versionInfo = d.getVar("PV") + spdx_package.licenseDeclared = convert_license_to_spdx(package_license, package_doc, d, found_licenses) + spdx_package.supplier = d.getVar("SPDX_SUPPLIER") + + package_doc.packages.append(spdx_package) + + package_doc.add_relationship(spdx_package, "GENERATED_FROM", "%s:%s" % (recipe_ref.externalDocumentId, recipe.SPDXID)) + package_doc.add_relationship(package_doc, "DESCRIBES", spdx_package) + + package_archive = deploy_dir_spdx / "packages" / (package_doc.name + ".tar.zst") + with optional_tarfile(package_archive, archive_packaged) as archive: + package_files = add_package_files( + d, + package_doc, + spdx_package, + pkgdest / package, + lambda file_counter: oe.sbom.get_packaged_file_spdxid(pkg_name, file_counter), + lambda filepath: ["BINARY"], + ignore_top_level_dirs=['CONTROL', 'DEBIAN'], + archive=archive, + ) + + if archive is not None: + spdx_package.packageFileName = str(package_archive.name) + + add_package_sources_from_debug(d, package_doc, spdx_package, package, package_files, sources) + + oe.sbom.write_doc(d, package_doc, "packages", indent=get_json_indent(d)) +} +# NOTE: depending on do_unpack is a hack that is necessary to get it's dependencies for archive the source +addtask do_create_spdx after do_package do_packagedata do_unpack before do_populate_sdk do_build do_rm_work + +SSTATETASKS += "do_create_spdx" +do_create_spdx[sstate-inputdirs] = "${SPDXDEPLOY}" +do_create_spdx[sstate-outputdirs] = "${DEPLOY_DIR_SPDX}" + +python do_create_spdx_setscene () { + sstate_setscene(d) +} +addtask do_create_spdx_setscene + +do_create_spdx[dirs] = "${SPDXWORK}" +do_create_spdx[cleandirs] = "${SPDXDEPLOY} ${SPDXWORK}" +do_create_spdx[depends] += "${PATCHDEPENDENCY}" +do_create_spdx[deptask] = "do_create_spdx" + +def collect_package_providers(d): + from pathlib import Path + import oe.sbom + import oe.spdx + import json + + deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX")) + + providers = {} + + taskdepdata = d.getVar("BB_TASKDEPDATA", False) + deps = sorted(set( + dep[0] for dep in taskdepdata.values() if dep[0] != d.getVar("PN") + )) + deps.append(d.getVar("PN")) + + for dep_pn in deps: + recipe_data = oe.packagedata.read_pkgdata(dep_pn, d) + + for pkg in recipe_data.get("PACKAGES", "").split(): + + pkg_data = oe.packagedata.read_subpkgdata_dict(pkg, d) + rprovides = set(n for n, _ in bb.utils.explode_dep_versions2(pkg_data.get("RPROVIDES", "")).items()) + rprovides.add(pkg) + + for r in rprovides: + providers[r] = pkg + + return providers + +collect_package_providers[vardepsexclude] += "BB_TASKDEPDATA" + +python do_create_runtime_spdx() { + from datetime import datetime, timezone + import oe.sbom + import oe.spdx + import oe.packagedata + from pathlib import Path + + deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX")) + spdx_deploy = Path(d.getVar("SPDXRUNTIMEDEPLOY")) + is_native = bb.data.inherits_class("native", d) or bb.data.inherits_class("cross", d) + + creation_time = datetime.now(tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + providers = collect_package_providers(d) + + if not is_native: + bb.build.exec_func("read_subpackage_metadata", d) + + dep_package_cache = {} + + pkgdest = Path(d.getVar("PKGDEST")) + for package in d.getVar("PACKAGES").split(): + localdata = bb.data.createCopy(d) + pkg_name = d.getVar("PKG:%s" % package) or package + localdata.setVar("PKG", pkg_name) + localdata.setVar('OVERRIDES', d.getVar("OVERRIDES", False) + ":" + package) + + if not oe.packagedata.packaged(package, localdata): + continue + + pkg_spdx_path = deploy_dir_spdx / "packages" / (pkg_name + ".spdx.json") + + package_doc, package_doc_sha1 = oe.sbom.read_doc(pkg_spdx_path) + + for p in package_doc.packages: + if p.name == pkg_name: + spdx_package = p + break + else: + bb.fatal("Package '%s' not found in %s" % (pkg_name, pkg_spdx_path)) + + runtime_doc = oe.spdx.SPDXDocument() + runtime_doc.name = "runtime-" + pkg_name + runtime_doc.documentNamespace = get_doc_namespace(localdata, runtime_doc) + runtime_doc.creationInfo.created = creation_time + runtime_doc.creationInfo.comment = "This document was created by analyzing package runtime dependencies." + runtime_doc.creationInfo.licenseListVersion = d.getVar("SPDX_LICENSE_DATA")["licenseListVersion"] + runtime_doc.creationInfo.creators.append("Tool: OpenEmbedded Core create-spdx.bbclass") + runtime_doc.creationInfo.creators.append("Organization: %s" % d.getVar("SPDX_ORG")) + runtime_doc.creationInfo.creators.append("Person: N/A ()") + + package_ref = oe.spdx.SPDXExternalDocumentRef() + package_ref.externalDocumentId = "DocumentRef-package-" + package + package_ref.spdxDocument = package_doc.documentNamespace + package_ref.checksum.algorithm = "SHA1" + package_ref.checksum.checksumValue = package_doc_sha1 + + runtime_doc.externalDocumentRefs.append(package_ref) + + runtime_doc.add_relationship( + runtime_doc.SPDXID, + "AMENDS", + "%s:%s" % (package_ref.externalDocumentId, package_doc.SPDXID) + ) + + deps = bb.utils.explode_dep_versions2(localdata.getVar("RDEPENDS") or "") + seen_deps = set() + for dep, _ in deps.items(): + if dep in seen_deps: + continue + + if dep not in providers: + continue + + dep = providers[dep] + + if not oe.packagedata.packaged(dep, localdata): + continue + + dep_pkg_data = oe.packagedata.read_subpkgdata_dict(dep, d) + dep_pkg = dep_pkg_data["PKG"] + + if dep in dep_package_cache: + (dep_spdx_package, dep_package_ref) = dep_package_cache[dep] + else: + dep_path = deploy_dir_spdx / "packages" / ("%s.spdx.json" % dep_pkg) + + spdx_dep_doc, spdx_dep_sha1 = oe.sbom.read_doc(dep_path) + + for pkg in spdx_dep_doc.packages: + if pkg.name == dep_pkg: + dep_spdx_package = pkg + break + else: + bb.fatal("Package '%s' not found in %s" % (dep_pkg, dep_path)) + + dep_package_ref = oe.spdx.SPDXExternalDocumentRef() + dep_package_ref.externalDocumentId = "DocumentRef-runtime-dependency-" + spdx_dep_doc.name + dep_package_ref.spdxDocument = spdx_dep_doc.documentNamespace + dep_package_ref.checksum.algorithm = "SHA1" + dep_package_ref.checksum.checksumValue = spdx_dep_sha1 + + dep_package_cache[dep] = (dep_spdx_package, dep_package_ref) + + runtime_doc.externalDocumentRefs.append(dep_package_ref) + + runtime_doc.add_relationship( + "%s:%s" % (dep_package_ref.externalDocumentId, dep_spdx_package.SPDXID), + "RUNTIME_DEPENDENCY_OF", + "%s:%s" % (package_ref.externalDocumentId, spdx_package.SPDXID) + ) + seen_deps.add(dep) + + oe.sbom.write_doc(d, runtime_doc, "runtime", spdx_deploy, indent=get_json_indent(d)) +} + +addtask do_create_runtime_spdx after do_create_spdx before do_build do_rm_work +SSTATETASKS += "do_create_runtime_spdx" +do_create_runtime_spdx[sstate-inputdirs] = "${SPDXRUNTIMEDEPLOY}" +do_create_runtime_spdx[sstate-outputdirs] = "${DEPLOY_DIR_SPDX}" + +python do_create_runtime_spdx_setscene () { + sstate_setscene(d) +} +addtask do_create_runtime_spdx_setscene + +do_create_runtime_spdx[dirs] = "${SPDXRUNTIMEDEPLOY}" +do_create_runtime_spdx[cleandirs] = "${SPDXRUNTIMEDEPLOY}" +do_create_runtime_spdx[rdeptask] = "do_create_spdx" + +def spdx_get_src(d): + """ + save patched source of the recipe in SPDX_WORKDIR. + """ + import shutil + spdx_workdir = d.getVar('SPDXWORK') + spdx_sysroot_native = d.getVar('STAGING_DIR_NATIVE') + pn = d.getVar('PN') + + workdir = d.getVar("WORKDIR") + + try: + # The kernel class functions require it to be on work-shared, so we dont change WORKDIR + if not is_work_shared_spdx(d): + # Change the WORKDIR to make do_unpack do_patch run in another dir. + d.setVar('WORKDIR', spdx_workdir) + # Restore the original path to recipe's native sysroot (it's relative to WORKDIR). + d.setVar('STAGING_DIR_NATIVE', spdx_sysroot_native) + + # The changed 'WORKDIR' also caused 'B' changed, create dir 'B' for the + # possibly requiring of the following tasks (such as some recipes's + # do_patch required 'B' existed). + bb.utils.mkdirhier(d.getVar('B')) + + bb.build.exec_func('do_unpack', d) + # Copy source of kernel to spdx_workdir + if is_work_shared_spdx(d): + share_src = d.getVar('WORKDIR') + d.setVar('WORKDIR', spdx_workdir) + d.setVar('STAGING_DIR_NATIVE', spdx_sysroot_native) + src_dir = spdx_workdir + "/" + d.getVar('PN')+ "-" + d.getVar('PV') + "-" + d.getVar('PR') + bb.utils.mkdirhier(src_dir) + if bb.data.inherits_class('kernel',d): + share_src = d.getVar('STAGING_KERNEL_DIR') + cmd_copy_share = "cp -rf " + share_src + "/* " + src_dir + "/" + cmd_copy_shared_res = os.popen(cmd_copy_share).read() + bb.note("cmd_copy_shared_result = " + cmd_copy_shared_res) + + git_path = src_dir + "/.git" + if os.path.exists(git_path): + shutils.rmtree(git_path) + + # Make sure gcc and kernel sources are patched only once + if not (d.getVar('SRC_URI') == "" or is_work_shared_spdx(d)): + bb.build.exec_func('do_patch', d) + + # Some userland has no source. + if not os.path.exists( spdx_workdir ): + bb.utils.mkdirhier(spdx_workdir) + finally: + d.setVar("WORKDIR", workdir) + +do_rootfs[recrdeptask] += "do_create_spdx do_create_runtime_spdx" +do_rootfs[cleandirs] += "${SPDXIMAGEWORK}" + +ROOTFS_POSTUNINSTALL_COMMAND =+ "image_combine_spdx ; " + +do_populate_sdk[recrdeptask] += "do_create_spdx do_create_runtime_spdx" +do_populate_sdk[cleandirs] += "${SPDXSDKWORK}" +POPULATE_SDK_POST_HOST_COMMAND:append:task-populate-sdk = " sdk_host_combine_spdx; " +POPULATE_SDK_POST_TARGET_COMMAND:append:task-populate-sdk = " sdk_target_combine_spdx; " + +python image_combine_spdx() { + import os + import oe.sbom + from pathlib import Path + from oe.rootfs import image_list_installed_packages + + image_name = d.getVar("IMAGE_NAME") + image_link_name = d.getVar("IMAGE_LINK_NAME") + imgdeploydir = Path(d.getVar("IMGDEPLOYDIR")) + img_spdxid = oe.sbom.get_image_spdxid(image_name) + packages = image_list_installed_packages(d) + + combine_spdx(d, image_name, imgdeploydir, img_spdxid, packages, Path(d.getVar("SPDXIMAGEWORK"))) + + def make_image_link(target_path, suffix): + if image_link_name: + link = imgdeploydir / (image_link_name + suffix) + if link != target_path: + link.symlink_to(os.path.relpath(target_path, link.parent)) + + spdx_tar_path = imgdeploydir / (image_name + ".spdx.tar.zst") + make_image_link(spdx_tar_path, ".spdx.tar.zst") +} + +python sdk_host_combine_spdx() { + sdk_combine_spdx(d, "host") +} + +python sdk_target_combine_spdx() { + sdk_combine_spdx(d, "target") +} + +def sdk_combine_spdx(d, sdk_type): + import oe.sbom + from pathlib import Path + from oe.sdk import sdk_list_installed_packages + + sdk_name = d.getVar("SDK_NAME") + "-" + sdk_type + sdk_deploydir = Path(d.getVar("SDKDEPLOYDIR")) + sdk_spdxid = oe.sbom.get_sdk_spdxid(sdk_name) + sdk_packages = sdk_list_installed_packages(d, sdk_type == "target") + combine_spdx(d, sdk_name, sdk_deploydir, sdk_spdxid, sdk_packages, Path(d.getVar('SPDXSDKWORK'))) + +def combine_spdx(d, rootfs_name, rootfs_deploydir, rootfs_spdxid, packages, spdx_workdir): + import os + import oe.spdx + import oe.sbom + import io + import json + from datetime import timezone, datetime + from pathlib import Path + import tarfile + import bb.compress.zstd + + creation_time = datetime.now(tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX")) + source_date_epoch = d.getVar("SOURCE_DATE_EPOCH") + + doc = oe.spdx.SPDXDocument() + doc.name = rootfs_name + doc.documentNamespace = get_doc_namespace(d, doc) + doc.creationInfo.created = creation_time + doc.creationInfo.comment = "This document was created by analyzing the source of the Yocto recipe during the build." + doc.creationInfo.licenseListVersion = d.getVar("SPDX_LICENSE_DATA")["licenseListVersion"] + doc.creationInfo.creators.append("Tool: OpenEmbedded Core create-spdx.bbclass") + doc.creationInfo.creators.append("Organization: %s" % d.getVar("SPDX_ORG")) + doc.creationInfo.creators.append("Person: N/A ()") + + image = oe.spdx.SPDXPackage() + image.name = d.getVar("PN") + image.versionInfo = d.getVar("PV") + image.SPDXID = rootfs_spdxid + image.supplier = d.getVar("SPDX_SUPPLIER") + + doc.packages.append(image) + + for name in sorted(packages.keys()): + pkg_spdx_path = deploy_dir_spdx / "packages" / (name + ".spdx.json") + pkg_doc, pkg_doc_sha1 = oe.sbom.read_doc(pkg_spdx_path) + + for p in pkg_doc.packages: + if p.name == name: + pkg_ref = oe.spdx.SPDXExternalDocumentRef() + pkg_ref.externalDocumentId = "DocumentRef-%s" % pkg_doc.name + pkg_ref.spdxDocument = pkg_doc.documentNamespace + pkg_ref.checksum.algorithm = "SHA1" + pkg_ref.checksum.checksumValue = pkg_doc_sha1 + + doc.externalDocumentRefs.append(pkg_ref) + doc.add_relationship(image, "CONTAINS", "%s:%s" % (pkg_ref.externalDocumentId, p.SPDXID)) + break + else: + bb.fatal("Unable to find package with name '%s' in SPDX file %s" % (name, pkg_spdx_path)) + + runtime_spdx_path = deploy_dir_spdx / "runtime" / ("runtime-" + name + ".spdx.json") + runtime_doc, runtime_doc_sha1 = oe.sbom.read_doc(runtime_spdx_path) + + runtime_ref = oe.spdx.SPDXExternalDocumentRef() + runtime_ref.externalDocumentId = "DocumentRef-%s" % runtime_doc.name + runtime_ref.spdxDocument = runtime_doc.documentNamespace + runtime_ref.checksum.algorithm = "SHA1" + runtime_ref.checksum.checksumValue = runtime_doc_sha1 + + # "OTHER" isn't ideal here, but I can't find a relationship that makes sense + doc.externalDocumentRefs.append(runtime_ref) + doc.add_relationship( + image, + "OTHER", + "%s:%s" % (runtime_ref.externalDocumentId, runtime_doc.SPDXID), + comment="Runtime dependencies for %s" % name + ) + + image_spdx_path = spdx_workdir / (rootfs_name + ".spdx.json") + + with image_spdx_path.open("wb") as f: + doc.to_json(f, sort_keys=True, indent=get_json_indent(d)) + + num_threads = int(d.getVar("BB_NUMBER_THREADS")) + + visited_docs = set() + + index = {"documents": []} + + spdx_tar_path = rootfs_deploydir / (rootfs_name + ".spdx.tar.zst") + with bb.compress.zstd.open(spdx_tar_path, "w", num_threads=num_threads) as f: + with tarfile.open(fileobj=f, mode="w|") as tar: + def collect_spdx_document(path): + nonlocal tar + nonlocal deploy_dir_spdx + nonlocal source_date_epoch + nonlocal index + + if path in visited_docs: + return + + visited_docs.add(path) + + with path.open("rb") as f: + doc, sha1 = oe.sbom.read_doc(f) + f.seek(0) + + if doc.documentNamespace in visited_docs: + return + + bb.note("Adding SPDX document %s" % path) + visited_docs.add(doc.documentNamespace) + info = tar.gettarinfo(fileobj=f) + + info.name = doc.name + ".spdx.json" + info.uid = 0 + info.gid = 0 + info.uname = "root" + info.gname = "root" + + if source_date_epoch is not None and info.mtime > int(source_date_epoch): + info.mtime = int(source_date_epoch) + + tar.addfile(info, f) + + index["documents"].append({ + "filename": info.name, + "documentNamespace": doc.documentNamespace, + "sha1": sha1, + }) + + for ref in doc.externalDocumentRefs: + ref_path = deploy_dir_spdx / "by-namespace" / ref.spdxDocument.replace("/", "_") + collect_spdx_document(ref_path) + + collect_spdx_document(image_spdx_path) + + index["documents"].sort(key=lambda x: x["filename"]) + + index_str = io.BytesIO(json.dumps( + index, + sort_keys=True, + indent=get_json_indent(d), + ).encode("utf-8")) + + info = tarfile.TarInfo() + info.name = "index.json" + info.size = len(index_str.getvalue()) + info.uid = 0 + info.gid = 0 + info.uname = "root" + info.gname = "root" + + tar.addfile(info, fileobj=index_str)
meta/classes/create-spdx.bbclass new file +8 -0
@@ -0,0 +1,8 @@ +# +# Copyright OpenEmbedded Contributors +# +# SPDX-License-Identifier: GPL-2.0-only +# +# Include this class when you don't care what version of SPDX you get; it will +# be updated to the latest stable version that is supported +inherit create-spdx-2.2
meta/files/spdx-licenses.json new file +5937 -0
1,923 lines not on this page · raw message
meta/lib/oe/sbom.py new file +84 -0
84 lines not on this page · raw message
meta/lib/oe/spdx.py new file +357 -0
357 lines not on this page · raw message
The rest of this diff is not shown. View the raw message.
Signature · 1 line
2.33.0
Part of the diff is not shown here. View the raw message.
Re: [OE-core][dunfell][PATCH 1/5] classes/create-spdx: Backport
Den ons 22 mars 2023 kl 21:46 skrev Joshua Watt <[email protected]>:
Quoted 920 lines
Backports the create-spdx classes from the latest versions on master. This backport is a simple copy with no modifications, as its too complex to cherry-pick all the corresponding changes. This will give an appropriate base commit for subsequent changes and if necessary additional backport cherry-picks from master in the future.
Signed-off-by Joshua Watt <[email protected]>--- meta/classes/create-spdx-2.2.bbclass | 1069 +++++ meta/classes/create-spdx.bbclass | 8 + meta/files/spdx-licenses.json | 5937 ++++++++++++++++++++++++++ meta/lib/oe/sbom.py | 84 + meta/lib/oe/spdx.py | 357 ++ 5 files changed, 7455 insertions(+) create mode 100644 meta/classes/create-spdx-2.2.bbclass create mode 100644 meta/classes/create-spdx.bbclass create mode 100644 meta/files/spdx-licenses.json create mode 100644 meta/lib/oe/sbom.py create mode 100644 meta/lib/oe/spdx.pymeta/classes/create-spdx-2.2.bbclass new file +893
-0@@ -0,0 +1,1069 @@ +# +# Copyright OpenEmbedded Contributors +# +# SPDX-License-Identifier: GPL-2.0-only +# + +DEPLOY_DIR_SPDX ??= "${DEPLOY_DIR}/spdx/${MACHINE}" + +# The product name that the CVE database uses. Defaults to BPN, but may need to +# be overriden per recipe (for example tiff.bb sets CVE_PRODUCT=libtiff). +CVE_PRODUCT ??= "${BPN}" +CVE_VERSION ??= "${PV}" + +SPDXDIR ??= "${WORKDIR}/spdx" +SPDXDEPLOY = "${SPDXDIR}/deploy" +SPDXWORK = "${SPDXDIR}/work" +SPDXIMAGEWORK = "${SPDXDIR}/image-work" +SPDXSDKWORK = "${SPDXDIR}/sdk-work" + +SPDX_TOOL_NAME ??= "oe-spdx-creator" +SPDX_TOOL_VERSION ??= "1.0" + +SPDXRUNTIMEDEPLOY = "${SPDXDIR}/runtime-deploy" + +SPDX_INCLUDE_SOURCES ??= "0" +SPDX_ARCHIVE_SOURCES ??= "0" +SPDX_ARCHIVE_PACKAGED ??= "0" + +SPDX_UUID_NAMESPACE ??= "sbom.openembedded.org" +SPDX_NAMESPACE_PREFIX ??= "http://spdx.org/spdxdoc" +SPDX_PRETTY ??= "0" + +SPDX_LICENSES ??= "${COREBASE}/meta/files/spdx-licenses.json" + +SPDX_CUSTOM_ANNOTATION_VARS ??= "" + +SPDX_ORG ??= "OpenEmbedded ()" +SPDX_SUPPLIER ??= "Organization: ${SPDX_ORG}" +SPDX_SUPPLIER[doc] = "The SPDX PackageSupplier field for SPDX packages created from \ + this recipe. For SPDX documents create using this class during the build, this \ + is the contact information for the person or organization who is doing the \ + build." + +def extract_licenses(filename): + import re + + lic_regex = re.compile(rb'^\W*SPDX-License-Identifier:\s*([ \w\d.()+-]+?)(?:\s+\W*)?$', re.MULTILINE) + + try: + with open(filename, 'rb') as f: + size = min(15000, os.stat(filename).st_size) + txt = f.read(size) + licenses = re.findall(lic_regex, txt) + if licenses: + ascii_licenses = [lic.decode('ascii') for lic in licenses] + return ascii_licenses + except Exception as e: + bb.warn(f"Exception reading {filename}: {e}") + return None + +def get_doc_namespace(d, doc): + import uuid + namespace_uuid = uuid.uuid5(uuid.NAMESPACE_DNS, d.getVar("SPDX_UUID_NAMESPACE")) + return "%s/%s-%s" % (d.getVar("SPDX_NAMESPACE_PREFIX"), doc.name, str(uuid.uuid5(namespace_uuid, doc.name))) + +def create_annotation(d, comment): + from datetime import datetime, timezone + + creation_time = datetime.now(tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + annotation = oe.spdx.SPDXAnnotation() + annotation.annotationDate = creation_time + annotation.annotationType = "OTHER" + annotation.annotator = "Tool: %s - %s" % (d.getVar("SPDX_TOOL_NAME"), d.getVar("SPDX_TOOL_VERSION")) + annotation.comment = comment + return annotation + +def recipe_spdx_is_native(d, recipe): + return any(a.annotationType == "OTHER" and + a.annotator == "Tool: %s - %s" % (d.getVar("SPDX_TOOL_NAME"), d.getVar("SPDX_TOOL_VERSION")) and + a.comment == "isNative" for a in recipe.annotations) + +def is_work_shared_spdx(d): + return bb.data.inherits_class('kernel', d) or ('work-shared' in d.getVar('WORKDIR')) + +def get_json_indent(d): + if d.getVar("SPDX_PRETTY") == "1": + return 2 + return None + +python() { + import json + if d.getVar("SPDX_LICENSE_DATA"): + return + + with open(d.getVar("SPDX_LICENSES"), "r") as f: + data = json.load(f) + # Transform the license array to a dictionary + data["licenses"] = {l["licenseId"]: l for l in data["licenses"]} + d.setVar("SPDX_LICENSE_DATA", data) +} + +def convert_license_to_spdx(lic, document, d, existing={}): + from pathlib import Path + import oe.spdx + + license_data = d.getVar("SPDX_LICENSE_DATA") + extracted = {} + + def add_extracted_license(ident, name): + nonlocal document + + if name in extracted: + return + + extracted_info = oe.spdx.SPDXExtractedLicensingInfo() + extracted_info.name = name + extracted_info.licenseId = ident + extracted_info.extractedText = None + + if name == "PD": + # Special-case this. + extracted_info.extractedText = "Software released to the public domain" + else: + # Seach for the license in COMMON_LICENSE_DIR and LICENSE_PATH + for directory in [d.getVar('COMMON_LICENSE_DIR')] + (d.getVar('LICENSE_PATH') or '').split(): + try: + with (Path(directory) / name).open(errors="replace") as f: + extracted_info.extractedText = f.read() + break + except FileNotFoundError: + pass + if extracted_info.extractedText is None: + # If it's not SPDX or PD, then NO_GENERIC_LICENSE must be set + filename = d.getVarFlag('NO_GENERIC_LICENSE', name) + if filename: + filename = d.expand("${S}/" + filename) + with open(filename, errors="replace") as f: + extracted_info.extractedText = f.read() + else: + bb.error("Cannot find any text for license %s" % name) + + extracted[name] = extracted_info + document.hasExtractedLicensingInfos.append(extracted_info) + + def convert(l): + if l == "(" or l == ")": + return l + + if l == "&": + return "AND" + + if l == "|": + return "OR" + + if l == "CLOSED": + return "NONE" + + spdx_license = d.getVarFlag("SPDXLICENSEMAP", l) or l + if spdx_license in license_data["licenses"]: + return spdx_license + + try: + spdx_license = existing[l] + except KeyError: + spdx_license = "LicenseRef-" + l + add_extracted_license(spdx_license, l) + + return spdx_license + + lic_split = lic.replace("(", " ( ").replace(")", " ) ").split() + + return ' '.join(convert(l) for l in lic_split) + +def process_sources(d): + pn = d.getVar('PN') + assume_provided = (d.getVar("ASSUME_PROVIDED") or "").split() + if pn in assume_provided: + for p in d.getVar("PROVIDES").split(): + if p != pn: + pn = p + break + + # glibc-locale: do_fetch, do_unpack and do_patch tasks have been deleted, + # so avoid archiving source here. + if pn.startswith('glibc-locale'): + return False + if d.getVar('PN') == "libtool-cross": + return False + if d.getVar('PN') == "libgcc-initial": + return False + if d.getVar('PN') == "shadow-sysroot": + return False + + # We just archive gcc-source for all the gcc related recipes + if d.getVar('BPN') in ['gcc', 'libgcc']: + bb.debug(1, 'spdx: There is bug in scan of %s is, do nothing' % pn) + return False + + return True + + +def add_package_files(d, doc, spdx_pkg, topdir, get_spdxid, get_types, *, archive=None, ignore_dirs=[], ignore_top_level_dirs=[]): + from pathlib import Path + import oe.spdx + import hashlib + + source_date_epoch = d.getVar("SOURCE_DATE_EPOCH") + if source_date_epoch: + source_date_epoch = int(source_date_epoch) + + sha1s = [] + spdx_files = [] + + file_counter = 1 + for subdir, dirs, files in os.walk(topdir): + dirs[:] = [d for d in dirs if d not in ignore_dirs] + if subdir == str(topdir): + dirs[:] = [d for d in dirs if d not in ignore_top_level_dirs] + + for file in files: + filepath = Path(subdir) / file + filename = str(filepath.relative_to(topdir)) + + if not filepath.is_symlink() and filepath.is_file(): + spdx_file = oe.spdx.SPDXFile() + spdx_file.SPDXID = get_spdxid(file_counter) + for t in get_types(filepath): + spdx_file.fileTypes.append(t) + spdx_file.fileName = filename + + if archive is not None: + with filepath.open("rb") as f: + info = archive.gettarinfo(fileobj=f) + info.name = filename + info.uid = 0 + info.gid = 0 + info.uname = "root" + info.gname = "root" + + if source_date_epoch is not None and info.mtime > source_date_epoch: + info.mtime = source_date_epoch + + archive.addfile(info, f) + + sha1 = bb.utils.sha1_file(filepath) + sha1s.append(sha1) + spdx_file.checksums.append(oe.spdx.SPDXChecksum( + algorithm="SHA1", + checksumValue=sha1, + )) + spdx_file.checksums.append(oe.spdx.SPDXChecksum( + algorithm="SHA256", + checksumValue=bb.utils.sha256_file(filepath), + )) + + if "SOURCE" in spdx_file.fileTypes: + extracted_lics = extract_licenses(filepath) + if extracted_lics: + spdx_file.licenseInfoInFiles = extracted_lics + + doc.files.append(spdx_file) + doc.add_relationship(spdx_pkg, "CONTAINS", spdx_file) + spdx_pkg.hasFiles.append(spdx_file.SPDXID) + + spdx_files.append(spdx_file) + + file_counter += 1 + + sha1s.sort() + verifier = hashlib.sha1() + for v in sha1s: + verifier.update(v.encode("utf-8")) + spdx_pkg.packageVerificationCode.packageVerificationCodeValue = verifier.hexdigest() + + return spdx_files + + +def add_package_sources_from_debug(d, package_doc, spdx_package, package, package_files, sources): + from pathlib import Path + import hashlib + import oe.packagedata + import oe.spdx + + debug_search_paths = [ + Path(d.getVar('PKGD')), + Path(d.getVar('STAGING_DIR_TARGET')), + Path(d.getVar('STAGING_DIR_NATIVE')), + Path(d.getVar('STAGING_KERNEL_DIR')), + ] + + pkg_data = oe.packagedata.read_subpkgdata_extended(package, d) + + if pkg_data is None: + return + + for file_path, file_data in pkg_data["files_info"].items(): + if not "debugsrc" in file_data: + continue + + for pkg_file in package_files: + if file_path.lstrip("/") == pkg_file.fileName.lstrip("/"): + break + else: + bb.fatal("No package file found for %s" % str(file_path)) + continue + + for debugsrc in file_data["debugsrc"]: + ref_id = "NOASSERTION" + for search in debug_search_paths: + if debugsrc.startswith("/usr/src/kernel"): + debugsrc_path = search / debugsrc.replace('/usr/src/kernel/', '') + else: + debugsrc_path = search / debugsrc.lstrip("/") + if not debugsrc_path.exists(): + continue + + file_sha256 = bb.utils.sha256_file(debugsrc_path) + + if file_sha256 in sources: + source_file = sources[file_sha256] + + doc_ref = package_doc.find_external_document_ref(source_file.doc.documentNamespace) + if doc_ref is None: + doc_ref = oe.spdx.SPDXExternalDocumentRef() + doc_ref.externalDocumentId = "DocumentRef-dependency-" + source_file.doc.name + doc_ref.spdxDocument = source_file.doc.documentNamespace + doc_ref.checksum.algorithm = "SHA1" + doc_ref.checksum.checksumValue = source_file.doc_sha1 + package_doc.externalDocumentRefs.append(doc_ref) + + ref_id = "%s:%s" % (doc_ref.externalDocumentId, source_file.file.SPDXID) + else: + bb.debug(1, "Debug source %s with SHA256 %s not found in any dependency" % (str(debugsrc_path), file_sha256)) + break + else: + bb.debug(1, "Debug source %s not found" % debugsrc) + + package_doc.add_relationship(pkg_file, "GENERATED_FROM", ref_id, comment=debugsrc) + +def collect_dep_recipes(d, doc, spdx_recipe): + from pathlib import Path + import oe.sbom + import oe.spdx + + deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX")) + + dep_recipes = [] + taskdepdata = d.getVar("BB_TASKDEPDATA", False) + deps = sorted(set( + dep[0] for dep in taskdepdata.values() if + dep[1] == "do_create_spdx" and dep[0] != d.getVar("PN") + )) + for dep_pn in deps: + dep_recipe_path = deploy_dir_spdx / "recipes" / ("recipe-%s.spdx.json" % dep_pn) + + spdx_dep_doc, spdx_dep_sha1 = oe.sbom.read_doc(dep_recipe_path) + + for pkg in spdx_dep_doc.packages: + if pkg.name == dep_pn: + spdx_dep_recipe = pkg + break + else: + continue + + dep_recipes.append(oe.sbom.DepRecipe(spdx_dep_doc, spdx_dep_sha1, spdx_dep_recipe)) + + dep_recipe_ref = oe.spdx.SPDXExternalDocumentRef() + dep_recipe_ref.externalDocumentId = "DocumentRef-dependency-" + spdx_dep_doc.name + dep_recipe_ref.spdxDocument = spdx_dep_doc.documentNamespace + dep_recipe_ref.checksum.algorithm = "SHA1" + dep_recipe_ref.checksum.checksumValue = spdx_dep_sha1 + + doc.externalDocumentRefs.append(dep_recipe_ref) + + doc.add_relationship( + "%s:%s" % (dep_recipe_ref.externalDocumentId, spdx_dep_recipe.SPDXID), + "BUILD_DEPENDENCY_OF", + spdx_recipe + ) + + return dep_recipes + +collect_dep_recipes[vardepsexclude] += "BB_TASKDEPDATA" +collect_dep_recipes[vardeps] += "DEPENDS" + +def collect_dep_sources(d, dep_recipes): + import oe.sbom + + sources = {} + for dep in dep_recipes: + # Don't collect sources from native recipes as they + # match non-native sources also. + if recipe_spdx_is_native(d, dep.recipe): + continue + recipe_files = set(dep.recipe.hasFiles) + + for spdx_file in dep.doc.files: + if spdx_file.SPDXID not in recipe_files: + continue + + if "SOURCE" in spdx_file.fileTypes: + for checksum in spdx_file.checksums: + if checksum.algorithm == "SHA256": + sources[checksum.checksumValue] = oe.sbom.DepSource(dep.doc, dep.doc_sha1, dep.recipe, spdx_file) + break + + return sources + +def add_download_packages(d, doc, recipe): + import os.path + from bb.fetch2 import decodeurl, CHECKSUM_LIST + import bb.process + import oe.spdx + import oe.sbom + + for download_idx, src_uri in enumerate(d.getVar('SRC_URI').split()): + f = bb.fetch2.FetchData(src_uri, d) + + for name in f.names: + package = oe.spdx.SPDXPackage() + package.name = "%s-source-%d" % (d.getVar("PN"), download_idx + 1) + package.SPDXID = oe.sbom.get_download_spdxid(d, download_idx + 1) + + if f.type == "file": + continue + + uri = f.type + proto = getattr(f, "proto", None) + if proto is not None: + uri = uri + "+" + proto + uri = uri + "://" + f.host + f.path + + if f.method.supports_srcrev(): + uri = uri + "@" + f.revisions[name] + + if f.method.supports_checksum(f): + for checksum_id in CHECKSUM_LIST: + if checksum_id.upper() not in oe.spdx.SPDXPackage.ALLOWED_CHECKSUMS: + continue + + expected_checksum = getattr(f, "%s_expected" % checksum_id) + if expected_checksum is None: + continue + + c = oe.spdx.SPDXChecksum() + c.algorithm = checksum_id.upper() + c.checksumValue = expected_checksum + package.checksums.append(c) + + package.downloadLocation = uri + doc.packages.append(package) + doc.add_relationship(doc, "DESCRIBES", package) + # In the future, we might be able to do more fancy dependencies, + # but this should be sufficient for now + doc.add_relationship(package, "BUILD_DEPENDENCY_OF", recipe) + +python do_create_spdx() { + from datetime import datetime, timezone + import oe.sbom + import oe.spdx + import uuid + from pathlib import Path + from contextlib import contextmanager + import oe.cve_check + + @contextmanager + def optional_tarfile(name, guard, mode="w"): + import tarfile + import bb.compress.zstd + + num_threads = int(d.getVar("BB_NUMBER_THREADS")) + + if guard: + name.parent.mkdir(parents=True, exist_ok=True) + with bb.compress.zstd.open(name, mode=mode + "b", num_threads=num_threads) as f: + with tarfile.open(fileobj=f, mode=mode + "|") as tf: + yield tf + else: + yield None + + + deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX")) + spdx_workdir = Path(d.getVar("SPDXWORK")) + include_sources = d.getVar("SPDX_INCLUDE_SOURCES") == "1" + archive_sources = d.getVar("SPDX_ARCHIVE_SOURCES") == "1" + archive_packaged = d.getVar("SPDX_ARCHIVE_PACKAGED") == "1" + + creation_time = datetime.now(tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + doc = oe.spdx.SPDXDocument() + + doc.name = "recipe-" + d.getVar("PN") + doc.documentNamespace = get_doc_namespace(d, doc) + doc.creationInfo.created = creation_time + doc.creationInfo.comment = "This document was created by analyzing recipe files during the build." + doc.creationInfo.licenseListVersion = d.getVar("SPDX_LICENSE_DATA")["licenseListVersion"] + doc.creationInfo.creators.append("Tool: OpenEmbedded Core create-spdx.bbclass") + doc.creationInfo.creators.append("Organization: %s" % d.getVar("SPDX_ORG")) + doc.creationInfo.creators.append("Person: N/A ()") + + recipe = oe.spdx.SPDXPackage() + recipe.name = d.getVar("PN") + recipe.versionInfo = d.getVar("PV") + recipe.SPDXID = oe.sbom.get_recipe_spdxid(d) + recipe.supplier = d.getVar("SPDX_SUPPLIER") + if bb.data.inherits_class("native", d) or bb.data.inherits_class("cross", d): + recipe.annotations.append(create_annotation(d, "isNative")) + + homepage = d.getVar("HOMEPAGE") + if homepage: + recipe.homepage = homepage + + license = d.getVar("LICENSE") + if license: + recipe.licenseDeclared = convert_license_to_spdx(license, doc, d) + + summary = d.getVar("SUMMARY") + if summary: + recipe.summary = summary + + description = d.getVar("DESCRIPTION") + if description: + recipe.description = description + + if d.getVar("SPDX_CUSTOM_ANNOTATION_VARS"): + for var in d.getVar('SPDX_CUSTOM_ANNOTATION_VARS').split(): + recipe.annotations.append(create_annotation(d, var + "=" + d.getVar(var))) + + # Some CVEs may be patched during the build process without incrementing the version number, + # so querying for CVEs based on the CPE id can lead to false positives. To account for this, + # save the CVEs fixed by patches to source information field in the SPDX. + patched_cves = oe.cve_check.get_patched_cves(d) + patched_cves = list(patched_cves) + patched_cves = ' '.join(patched_cves) + if patched_cves: + recipe.sourceInfo = "CVEs fixed: " + patched_cves + + cpe_ids = oe.cve_check.get_cpe_ids(d.getVar("CVE_PRODUCT"), d.getVar("CVE_VERSION")) + if cpe_ids: + for cpe_id in cpe_ids: + cpe = oe.spdx.SPDXExternalReference() + cpe.referenceCategory = "SECURITY" + cpe.referenceType = "http://spdx.org/rdf/references/cpe23Type" + cpe.referenceLocator = cpe_id + recipe.externalRefs.append(cpe) + + doc.packages.append(recipe) + doc.add_relationship(doc, "DESCRIBES", recipe) + + add_download_packages(d, doc, recipe) + + if process_sources(d) and include_sources: + recipe_archive = deploy_dir_spdx / "recipes" / (doc.name + ".tar.zst") + with optional_tarfile(recipe_archive, archive_sources) as archive: + spdx_get_src(d) + + add_package_files( + d, + doc, + recipe, + spdx_workdir, + lambda file_counter: "SPDXRef-SourceFile-%s-%d" % (d.getVar("PN"), file_counter), + lambda filepath: ["SOURCE"], + ignore_dirs=[".git"], + ignore_top_level_dirs=["temp"], + archive=archive, + ) + + if archive is not None: + recipe.packageFileName = str(recipe_archive.name) + + dep_recipes = collect_dep_recipes(d, doc, recipe) + + doc_sha1 = oe.sbom.write_doc(d, doc, "recipes", indent=get_json_indent(d)) + dep_recipes.append(oe.sbom.DepRecipe(doc, doc_sha1, recipe)) + + recipe_ref = oe.spdx.SPDXExternalDocumentRef() + recipe_ref.externalDocumentId = "DocumentRef-recipe-" + recipe.name + recipe_ref.spdxDocument = doc.documentNamespace + recipe_ref.checksum.algorithm = "SHA1" + recipe_ref.checksum.checksumValue = doc_sha1 + + sources = collect_dep_sources(d, dep_recipes) + found_licenses = {license.name:recipe_ref.externalDocumentId + ":" + license.licenseId for license in doc.hasExtractedLicensingInfos} + + if not recipe_spdx_is_native(d, recipe): + bb.build.exec_func("read_subpackage_metadata", d) + + pkgdest = Path(d.getVar("PKGDEST")) + for package in d.getVar("PACKAGES").split(): + if not oe.packagedata.packaged(package, d): + continue + + package_doc = oe.spdx.SPDXDocument() + pkg_name = d.getVar("PKG:%s" % package) or package + package_doc.name = pkg_name + package_doc.documentNamespace = get_doc_namespace(d, package_doc) + package_doc.creationInfo.created = creation_time + package_doc.creationInfo.comment = "This document was created by analyzing packages created during the build." + package_doc.creationInfo.licenseListVersion = d.getVar("SPDX_LICENSE_DATA")["licenseListVersion"] + package_doc.creationInfo.creators.append("Tool: OpenEmbedded Core create-spdx.bbclass") + package_doc.creationInfo.creators.append("Organization: %s" % d.getVar("SPDX_ORG")) + package_doc.creationInfo.creators.append("Person: N/A ()") + package_doc.externalDocumentRefs.append(recipe_ref) + + package_license = d.getVar("LICENSE:%s" % package) or d.getVar("LICENSE") + + spdx_package = oe.spdx.SPDXPackage() + + spdx_package.SPDXID = oe.sbom.get_package_spdxid(pkg_name) + spdx_package.name = pkg_name + spdx_package.versionInfo = d.getVar("PV") + spdx_package.licenseDeclared = convert_license_to_spdx(package_license, package_doc, d, found_licenses) + spdx_package.supplier = d.getVar("SPDX_SUPPLIER") + + package_doc.packages.append(spdx_package) + + package_doc.add_relationship(spdx_package, "GENERATED_FROM", "%s:%s" % (recipe_ref.externalDocumentId, recipe.SPDXID)) + package_doc.add_relationship(package_doc, "DESCRIBES", spdx_package) + + package_archive = deploy_dir_spdx / "packages" / (package_doc.name + ".tar.zst") + with optional_tarfile(package_archive, archive_packaged) as archive: + package_files = add_package_files( + d, + package_doc, + spdx_package, + pkgdest / package, + lambda file_counter: oe.sbom.get_packaged_file_spdxid(pkg_name, file_counter), + lambda filepath: ["BINARY"], + ignore_top_level_dirs=['CONTROL', 'DEBIAN'], + archive=archive, + ) + + if archive is not None: + spdx_package.packageFileName = str(package_archive.name) + + add_package_sources_from_debug(d, package_doc, spdx_package, package, package_files, sources) + + oe.sbom.write_doc(d, package_doc, "packages", indent=get_json_indent(d)) +} +# NOTE: depending on do_unpack is a hack that is necessary to get it's dependencies for archive the source +addtask do_create_spdx after do_package do_packagedata do_unpack before do_populate_sdk do_build do_rm_work + +SSTATETASKS += "do_create_spdx" +do_create_spdx[sstate-inputdirs] = "${SPDXDEPLOY}" +do_create_spdx[sstate-outputdirs] = "${DEPLOY_DIR_SPDX}" + +python do_create_spdx_setscene () { + sstate_setscene(d) +} +addtask do_create_spdx_setscene + +do_create_spdx[dirs] = "${SPDXWORK}" +do_create_spdx[cleandirs] = "${SPDXDEPLOY} ${SPDXWORK}" +do_create_spdx[depends] += "${PATCHDEPENDENCY}" +do_create_spdx[deptask] = "do_create_spdx" + +def collect_package_providers(d): + from pathlib import Path + import oe.sbom + import oe.spdx + import json + + deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX")) + + providers = {} + + taskdepdata = d.getVar("BB_TASKDEPDATA", False) + deps = sorted(set( + dep[0] for dep in taskdepdata.values() if dep[0] != d.getVar("PN") + )) + deps.append(d.getVar("PN")) + + for dep_pn in deps: + recipe_data = oe.packagedata.read_pkgdata(dep_pn, d) + + for pkg in recipe_data.get("PACKAGES", "").split(): + + pkg_data = oe.packagedata.read_subpkgdata_dict(pkg, d) + rprovides = set(n for n, _ in bb.utils.explode_dep_versions2(pkg_data.get("RPROVIDES", "")).items()) + rprovides.add(pkg) + + for r in rprovides: + providers[r] = pkg + + return providers + +collect_package_providers[vardepsexclude] += "BB_TASKDEPDATA" + +python do_create_runtime_spdx() { + from datetime import datetime, timezone + import oe.sbom + import oe.spdx + import oe.packagedata + from pathlib import Path + + deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX")) + spdx_deploy = Path(d.getVar("SPDXRUNTIMEDEPLOY")) + is_native = bb.data.inherits_class("native", d) or bb.data.inherits_class("cross", d) + + creation_time = datetime.now(tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + providers = collect_package_providers(d) + + if not is_native: + bb.build.exec_func("read_subpackage_metadata", d) + + dep_package_cache = {} + + pkgdest = Path(d.getVar("PKGDEST")) + for package in d.getVar("PACKAGES").split(): + localdata = bb.data.createCopy(d) + pkg_name = d.getVar("PKG:%s" % package) or package + localdata.setVar("PKG", pkg_name) + localdata.setVar('OVERRIDES', d.getVar("OVERRIDES", False) + ":" + package) + + if not oe.packagedata.packaged(package, localdata): + continue + + pkg_spdx_path = deploy_dir_spdx / "packages" / (pkg_name + ".spdx.json") + + package_doc, package_doc_sha1 = oe.sbom.read_doc(pkg_spdx_path) + + for p in package_doc.packages: + if p.name == pkg_name: + spdx_package = p + break + else: + bb.fatal("Package '%s' not found in %s" % (pkg_name, pkg_spdx_path)) + + runtime_doc = oe.spdx.SPDXDocument() + runtime_doc.name = "runtime-" + pkg_name + runtime_doc.documentNamespace = get_doc_namespace(localdata, runtime_doc) + runtime_doc.creationInfo.created = creation_time + runtime_doc.creationInfo.comment = "This document was created by analyzing package runtime dependencies." + runtime_doc.creationInfo.licenseListVersion = d.getVar("SPDX_LICENSE_DATA")["licenseListVersion"] + runtime_doc.creationInfo.creators.append("Tool: OpenEmbedded Core create-spdx.bbclass") + runtime_doc.creationInfo.creators.append("Organization: %s" % d.getVar("SPDX_ORG")) + runtime_doc.creationInfo.creators.append("Person: N/A ()") + + package_ref = oe.spdx.SPDXExternalDocumentRef() + package_ref.externalDocumentId = "DocumentRef-package-" + package + package_ref.spdxDocument = package_doc.documentNamespace + package_ref.checksum.algorithm = "SHA1" + package_ref.checksum.checksumValue = package_doc_sha1 + + runtime_doc.externalDocumentRefs.append(package_ref) + + runtime_doc.add_relationship( + runtime_doc.SPDXID, + "AMENDS", + "%s:%s" % (package_ref.externalDocumentId, package_doc.SPDXID) + ) + + deps = bb.utils.explode_dep_versions2(localdata.getVar("RDEPENDS") or "") + seen_deps = set() + for dep, _ in deps.items(): + if dep in seen_deps: + continue + + if dep not in providers: + continue + + dep = providers[dep] + + if not oe.packagedata.packaged(dep, localdata): + continue + + dep_pkg_data = oe.packagedata.read_subpkgdata_dict(dep, d) + dep_pkg = dep_pkg_data["PKG"] + + if dep in dep_package_cache: + (dep_spdx_package, dep_package_ref) = dep_package_cache[dep] + else: + dep_path = deploy_dir_spdx / "packages" / ("%s.spdx.json" % dep_pkg) + + spdx_dep_doc, spdx_dep_sha1 = oe.sbom.read_doc(dep_path) + + for pkg in spdx_dep_doc.packages: + if pkg.name == dep_pkg: + dep_spdx_package = pkg + break + else: + bb.fatal("Package '%s' not found in %s" % (dep_pkg, dep_path)) + + dep_package_ref = oe.spdx.SPDXExternalDocumentRef() + dep_package_ref.externalDocumentId = "DocumentRef-runtime-dependency-" + spdx_dep_doc.name + dep_package_ref.spdxDocument = spdx_dep_doc.documentNamespace + dep_package_ref.checksum.algorithm = "SHA1" + dep_package_ref.checksum.checksumValue = spdx_dep_sha1 + + dep_package_cache[dep] = (dep_spdx_package, dep_package_ref) + + runtime_doc.externalDocumentRefs.append(dep_package_ref) + + runtime_doc.add_relationship( + "%s:%s" % (dep_package_ref.externalDocumentId, dep_spdx_package.SPDXID), + "RUNTIME_DEPENDENCY_OF", + "%s:%s" % (package_ref.externalDocumentId, spdx_package.SPDXID) + ) + seen_deps.add(dep) + + oe.sbom.write_doc(d, runtime_doc, "runtime", spdx_deploy, indent=get_json_indent(d)) +} + +addtask do_create_runtime_spdx after do_create_spdx before do_build do_rm_work +SSTATETASKS += "do_create_runtime_spdx" +do_create_runtime_spdx[sstate-inputdirs] = "${SPDXRUNTIMEDEPLOY}" +do_create_runtime_spdx[sstate-outputdirs] = "${DEPLOY_DIR_SPDX}" + +python do_create_runtime_spdx_setscene () { + sstate_setscene(d) +} +addtask do_create_runtime_spdx_setscene + +do_create_runtime_spdx[dirs] = "${SPDXRUNTIMEDEPLOY}" +do_create_runtime_spdx[cleandirs] = "${SPDXRUNTIMEDEPLOY}" +do_create_runtime_spdx[rdeptask] = "do_create_spdx" + +def spdx_get_src(d): + """ + save patched source of the recipe in SPDX_WORKDIR. + """ + import shutil + spdx_workdir = d.getVar('SPDXWORK') + spdx_sysroot_native = d.getVar('STAGING_DIR_NATIVE') + pn = d.getVar('PN') + + workdir = d.getVar("WORKDIR") + + try: + # The kernel class functions require it to be on work-shared, so we dont change WORKDIR + if not is_work_shared_spdx(d): + # Change the WORKDIR to make do_unpack do_patch run in another dir. + d.setVar('WORKDIR', spdx_workdir) + # Restore the original path to recipe's native sysroot (it's relative to WORKDIR). + d.setVar('STAGING_DIR_NATIVE', spdx_sysroot_native) + + # The changed 'WORKDIR' also caused 'B' changed, create dir 'B' for the + # possibly requiring of the following tasks (such as some recipes's + # do_patch required 'B' existed). + bb.utils.mkdirhier(d.getVar('B')) + + bb.build.exec_func('do_unpack', d) + # Copy source of kernel to spdx_workdir + if is_work_shared_spdx(d): + share_src = d.getVar('WORKDIR') + d.setVar('WORKDIR', spdx_workdir) + d.setVar('STAGING_DIR_NATIVE', spdx_sysroot_native) + src_dir = spdx_workdir + "/" + d.getVar('PN')+ "-" + d.getVar('PV') + "-" + d.getVar('PR') + bb.utils.mkdirhier(src_dir) + if bb.data.inherits_class('kernel',d): + share_src = d.getVar('STAGING_KERNEL_DIR') + cmd_copy_share = "cp -rf " + share_src + "/* " + src_dir + "/" + cmd_copy_shared_res = os.popen(cmd_copy_share).read() + bb.note("cmd_copy_shared_result = " + cmd_copy_shared_res) + + git_path = src_dir + "/.git" + if os.path.exists(git_path): + shutils.rmtree(git_path) + + # Make sure gcc and kernel sources are patched only once + if not (d.getVar('SRC_URI') == "" or is_work_shared_spdx(d)): + bb.build.exec_func('do_patch', d) + + # Some userland has no source. + if not os.path.exists( spdx_workdir ): + bb.utils.mkdirhier(spdx_workdir) + finally: + d.setVar("WORKDIR", workdir) + +do_rootfs[recrdeptask] += "do_create_spdx do_create_runtime_spdx" +do_rootfs[cleandirs] += "${SPDXIMAGEWORK}" + +ROOTFS_POSTUNINSTALL_COMMAND =+ "image_combine_spdx ; " + +do_populate_sdk[recrdeptask] += "do_create_spdx do_create_runtime_spdx" +do_populate_sdk[cleandirs] += "${SPDXSDKWORK}" +POPULATE_SDK_POST_HOST_COMMAND:append:task-populate-sdk = " sdk_host_combine_spdx; " +POPULATE_SDK_POST_TARGET_COMMAND:append:task-populate-sdk = " sdk_target_combine_spdx; " + +python image_combine_spdx() { + import os + import oe.sbom + from pathlib import Path + from oe.rootfs import image_list_installed_packages + + image_name = d.getVar("IMAGE_NAME") + image_link_name = d.getVar("IMAGE_LINK_NAME") + imgdeploydir = Path(d.getVar("IMGDEPLOYDIR")) + img_spdxid = oe.sbom.get_image_spdxid(image_name) + packages = image_list_installed_packages(d) +
meta/classes/create-spdx-2.2.bbclass line 903
Quoted 10 lines
meta/classes/create-spdx-2.2.bbclass new file +10
-0+ combine_spdx(d, image_name, imgdeploydir, img_spdxid, packages, Path(d.getVar("SPDXIMAGEWORK"))) + + def make_image_link(target_path, suffix): + if image_link_name: + link = imgdeploydir / (image_link_name + suffix) + if link != target_path: + link.symlink_to(os.path.relpath(target_path, link.parent)) + + spdx_tar_path = imgdeploydir / (image_name + ".spdx.tar.zst") + make_image_link(spdx_tar_path, ".spdx.tar.zst")
The image link should be called tar.gz also. Probably best to make a gzip-from-the-start version for Dunfell,
or perhaps squash everything?
Quoted 6,589 lines
meta/classes/create-spdx-2.2.bbclass new file +166
-0+} + +python sdk_host_combine_spdx() { + sdk_combine_spdx(d, "host") +} + +python sdk_target_combine_spdx() { + sdk_combine_spdx(d, "target") +} + +def sdk_combine_spdx(d, sdk_type): + import oe.sbom + from pathlib import Path + from oe.sdk import sdk_list_installed_packages + + sdk_name = d.getVar("SDK_NAME") + "-" + sdk_type + sdk_deploydir = Path(d.getVar("SDKDEPLOYDIR")) + sdk_spdxid = oe.sbom.get_sdk_spdxid(sdk_name) + sdk_packages = sdk_list_installed_packages(d, sdk_type == "target") + combine_spdx(d, sdk_name, sdk_deploydir, sdk_spdxid, sdk_packages, Path(d.getVar('SPDXSDKWORK'))) + +def combine_spdx(d, rootfs_name, rootfs_deploydir, rootfs_spdxid, packages, spdx_workdir): + import os + import oe.spdx + import oe.sbom + import io + import json + from datetime import timezone, datetime + from pathlib import Path + import tarfile + import bb.compress.zstd + + creation_time = datetime.now(tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX")) + source_date_epoch = d.getVar("SOURCE_DATE_EPOCH") + + doc = oe.spdx.SPDXDocument() + doc.name = rootfs_name + doc.documentNamespace = get_doc_namespace(d, doc) + doc.creationInfo.created = creation_time + doc.creationInfo.comment = "This document was created by analyzing the source of the Yocto recipe during the build." + doc.creationInfo.licenseListVersion = d.getVar("SPDX_LICENSE_DATA")["licenseListVersion"] + doc.creationInfo.creators.append("Tool: OpenEmbedded Core create-spdx.bbclass") + doc.creationInfo.creators.append("Organization: %s" % d.getVar("SPDX_ORG")) + doc.creationInfo.creators.append("Person: N/A ()") + + image = oe.spdx.SPDXPackage() + image.name = d.getVar("PN") + image.versionInfo = d.getVar("PV") + image.SPDXID = rootfs_spdxid + image.supplier = d.getVar("SPDX_SUPPLIER") + + doc.packages.append(image) + + for name in sorted(packages.keys()): + pkg_spdx_path = deploy_dir_spdx / "packages" / (name + ".spdx.json") + pkg_doc, pkg_doc_sha1 = oe.sbom.read_doc(pkg_spdx_path) + + for p in pkg_doc.packages: + if p.name == name: + pkg_ref = oe.spdx.SPDXExternalDocumentRef() + pkg_ref.externalDocumentId = "DocumentRef-%s" % pkg_doc.name + pkg_ref.spdxDocument = pkg_doc.documentNamespace + pkg_ref.checksum.algorithm = "SHA1" + pkg_ref.checksum.checksumValue = pkg_doc_sha1 + + doc.externalDocumentRefs.append(pkg_ref) + doc.add_relationship(image, "CONTAINS", "%s:%s" % (pkg_ref.externalDocumentId, p.SPDXID)) + break + else: + bb.fatal("Unable to find package with name '%s' in SPDX file %s" % (name, pkg_spdx_path)) + + runtime_spdx_path = deploy_dir_spdx / "runtime" / ("runtime-" + name + ".spdx.json") + runtime_doc, runtime_doc_sha1 = oe.sbom.read_doc(runtime_spdx_path) + + runtime_ref = oe.spdx.SPDXExternalDocumentRef() + runtime_ref.externalDocumentId = "DocumentRef-%s" % runtime_doc.name + runtime_ref.spdxDocument = runtime_doc.documentNamespace + runtime_ref.checksum.algorithm = "SHA1" + runtime_ref.checksum.checksumValue = runtime_doc_sha1 + + # "OTHER" isn't ideal here, but I can't find a relationship that makes sense + doc.externalDocumentRefs.append(runtime_ref) + doc.add_relationship( + image, + "OTHER", + "%s:%s" % (runtime_ref.externalDocumentId, runtime_doc.SPDXID), + comment="Runtime dependencies for %s" % name + ) + + image_spdx_path = spdx_workdir / (rootfs_name + ".spdx.json") + + with image_spdx_path.open("wb") as f: + doc.to_json(f, sort_keys=True, indent=get_json_indent(d)) + + num_threads = int(d.getVar("BB_NUMBER_THREADS")) + + visited_docs = set() + + index = {"documents": []} + + spdx_tar_path = rootfs_deploydir / (rootfs_name + ".spdx.tar.zst") + with bb.compress.zstd.open(spdx_tar_path, "w", num_threads=num_threads) as f: + with tarfile.open(fileobj=f, mode="w|") as tar: + def collect_spdx_document(path): + nonlocal tar + nonlocal deploy_dir_spdx + nonlocal source_date_epoch + nonlocal index + + if path in visited_docs: + return + + visited_docs.add(path) + + with path.open("rb") as f: + doc, sha1 = oe.sbom.read_doc(f) + f.seek(0) + + if doc.documentNamespace in visited_docs: + return + + bb.note("Adding SPDX document %s" % path) + visited_docs.add(doc.documentNamespace) + info = tar.gettarinfo(fileobj=f) + + info.name = doc.name + ".spdx.json" + info.uid = 0 + info.gid = 0 + info.uname = "root" + info.gname = "root" + + if source_date_epoch is not None and info.mtime > int(source_date_epoch): + info.mtime = int(source_date_epoch) + + tar.addfile(info, f) + + index["documents"].append({ + "filename": info.name, + "documentNamespace": doc.documentNamespace, + "sha1": sha1, + }) + + for ref in doc.externalDocumentRefs: + ref_path = deploy_dir_spdx / "by-namespace" / ref.spdxDocument.replace("/", "_") + collect_spdx_document(ref_path) + + collect_spdx_document(image_spdx_path) + + index["documents"].sort(key=lambda x: x["filename"]) + + index_str = io.BytesIO(json.dumps( + index, + sort_keys=True, + indent=get_json_indent(d), + ).encode("utf-8")) + + info = tarfile.TarInfo() + info.name = "index.json" + info.size = len(index_str.getvalue()) + info.uid = 0 + info.gid = 0 + info.uname = "root" + info.gname = "root" + + tar.addfile(info, fileobj=index_str)meta/classes/create-spdx.bbclass new file +8
-0@@ -0,0 +1,8 @@ +# +# Copyright OpenEmbedded Contributors +# +# SPDX-License-Identifier: GPL-2.0-only +# +# Include this class when you don't care what version of SPDX you get; it will +# be updated to the latest stable version that is supported +inherit create-spdx-2.2meta/files/spdx-licenses.json new file +5937
-0@@ -0,0 +1,5937 @@ +{ + "licenseListVersion": "3.14", + "licenses": [ + { + "reference": "https://spdx.org/licenses/GPL-1.0.html", + "isDeprecatedLicenseId": true, + "detailsUrl": "https://spdx.org/licenses/GPL-1.0.json", + "referenceNumber": 0, + "name": "GNU General Public License v1.0 only", + "licenseId": "GPL-1.0", + "seeAlso": [ + "https://www.gnu.org/licenses/old-licenses/gpl-1.0-standalone.html" + ], + "isOsiApproved": false + }, + { + "reference": "https://spdx.org/licenses/bzip2-1.0.6.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/bzip2-1.0.6.json", + "referenceNumber": 1, + "name": "bzip2 and libbzip2 License v1.0.6", + "licenseId": "bzip2-1.0.6", + "seeAlso": [ + "https://sourceware.org/git/?p\u003dbzip2.git;a\u003dblob;f\u003dLICENSE;hb\u003dbzip2-1.0.6", + "http://bzip.org/1.0.5/bzip2-manual-1.0.5.html" + ], + "isOsiApproved": false + }, + { + "reference": "https://spdx.org/licenses/Intel-ACPI.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/Intel-ACPI.json", + "referenceNumber": 2, + "name": "Intel ACPI Software License Agreement", + "licenseId": "Intel-ACPI", + "seeAlso": [ + "https://fedoraproject.org/wiki/Licensing/Intel_ACPI_Software_License_Agreement" + ], + "isOsiApproved": false + }, + { + "reference": "https://spdx.org/licenses/XSkat.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/XSkat.json", + "referenceNumber": 3, + "name": "XSkat License", + "licenseId": "XSkat", + "seeAlso": [ + "https://fedoraproject.org/wiki/Licensing/XSkat_License" + ], + "isOsiApproved": false + }, + { + "reference": "https://spdx.org/licenses/CC-BY-NC-SA-2.0.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/CC-BY-NC-SA-2.0.json", + "referenceNumber": 4, + "name": "Creative Commons Attribution Non Commercial Share Alike 2.0 Generic", + "licenseId": "CC-BY-NC-SA-2.0", + "seeAlso": [ + "https://creativecommons.org/licenses/by-nc-sa/2.0/legalcode" + ], + "isOsiApproved": false + }, + { + "reference": "https://spdx.org/licenses/Plexus.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/Plexus.json", + "referenceNumber": 5, + "name": "Plexus Classworlds License", + "licenseId": "Plexus", + "seeAlso": [ + "https://fedoraproject.org/wiki/Licensing/Plexus_Classworlds_License" + ], + "isOsiApproved": false + }, + { + "reference": "https://spdx.org/licenses/Giftware.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/Giftware.json", + "referenceNumber": 6, + "name": "Giftware License", + "licenseId": "Giftware", + "seeAlso": [ + "http://liballeg.org/license.html#allegro-4-the-giftware-license" + ], + "isOsiApproved": false + }, + { + "reference": "https://spdx.org/licenses/BitTorrent-1.0.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/BitTorrent-1.0.json", + "referenceNumber": 7, + "name": "BitTorrent Open Source License v1.0", + "licenseId": "BitTorrent-1.0", + "seeAlso": [ + "http://sources.gentoo.org/cgi-bin/viewvc.cgi/gentoo-x86/licenses/BitTorrent?r1\u003d1.1\u0026r2\u003d1.1.1.1\u0026diff_format\u003ds" + ], + "isOsiApproved": false + }, + { + "reference": "https://spdx.org/licenses/APSL-1.1.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/APSL-1.1.json", + "referenceNumber": 8, + "name": "Apple Public Source License 1.1", + "licenseId": "APSL-1.1", + "seeAlso": [ + "http://www.opensource.apple.com/source/IOSerialFamily/IOSerialFamily-7/APPLE_LICENSE" + ], + "isOsiApproved": true + }, + { + "reference": "https://spdx.org/licenses/GPL-2.0-with-GCC-exception.html", + "isDeprecatedLicenseId": true, + "detailsUrl": "https://spdx.org/licenses/GPL-2.0-with-GCC-exception.json", + "referenceNumber": 9, + "name": "GNU General Public License v2.0 w/GCC Runtime Library exception", + "licenseId": "GPL-2.0-with-GCC-exception", + "seeAlso": [ + "https://gcc.gnu.org/git/?p\u003dgcc.git;a\u003dblob;f\u003dgcc/libgcc1.c;h\u003d762f5143fc6eed57b6797c82710f3538aa52b40b;hb\u003dcb143a3ce4fb417c68f5fa2691a1b1b1053dfba9#l10" + ], + "isOsiApproved": false + }, + { + "reference": "https://spdx.org/licenses/UPL-1.0.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/UPL-1.0.json", + "referenceNumber": 10, + "name": "Universal Permissive License v1.0", + "licenseId": "UPL-1.0", + "seeAlso": [ + "https://opensource.org/licenses/UPL" + ], + "isOsiApproved": true, + "isFsfLibre": true + }, + { + "reference": "https://spdx.org/licenses/wxWindows.html", + "isDeprecatedLicenseId": true, + "detailsUrl": "https://spdx.org/licenses/wxWindows.json", + "referenceNumber": 11, + "name": "wxWindows Library License", + "licenseId": "wxWindows", + "seeAlso": [ + "https://opensource.org/licenses/WXwindows" + ], + "isOsiApproved": false + }, + { + "reference": "https://spdx.org/licenses/Caldera.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/Caldera.json", + "referenceNumber": 12, + "name": "Caldera License", + "licenseId": "Caldera", + "seeAlso": [ + "http://www.lemis.com/grog/UNIX/ancient-source-all.pdf" + ], + "isOsiApproved": false + }, + { + "reference": "https://spdx.org/licenses/Zend-2.0.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/Zend-2.0.json", + "referenceNumber": 13, + "name": "Zend License v2.0", + "licenseId": "Zend-2.0", + "seeAlso": [ + "https://web.archive.org/web/20130517195954/http://www.zend.com/license/2_00.txt" + ], + "isOsiApproved": false, + "isFsfLibre": true + }, + { + "reference": "https://spdx.org/licenses/CUA-OPL-1.0.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/CUA-OPL-1.0.json", + "referenceNumber": 14, + "name": "CUA Office Public License v1.0", + "licenseId": "CUA-OPL-1.0", + "seeAlso": [ + "https://opensource.org/licenses/CUA-OPL-1.0" + ], + "isOsiApproved": true + }, + { + "reference": "https://spdx.org/licenses/JPNIC.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/JPNIC.json", + "referenceNumber": 15, + "name": "Japan Network Information Center License", + "licenseId": "JPNIC", + "seeAlso": [ + "https://gitlab.isc.org/isc-projects/bind9/blob/master/COPYRIGHT#L366" + ], + "isOsiApproved": false + }, + { + "reference": "https://spdx.org/licenses/SAX-PD.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/SAX-PD.json", + "referenceNumber": 16, + "name": "Sax Public Domain Notice", + "licenseId": "SAX-PD", + "seeAlso": [ + "http://www.saxproject.org/copying.html" + ], + "isOsiApproved": false + }, + { + "reference": "https://spdx.org/licenses/CC-BY-ND-2.5.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/CC-BY-ND-2.5.json", + "referenceNumber": 17, + "name": "Creative Commons Attribution No Derivatives 2.5 Generic", + "licenseId": "CC-BY-ND-2.5", + "seeAlso": [ + "https://creativecommons.org/licenses/by-nd/2.5/legalcode" + ], + "isOsiApproved": false + }, + { + "reference": "https://spdx.org/licenses/eGenix.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/eGenix.json", + "referenceNumber": 18, + "name": "eGenix.com Public License 1.1.0", + "licenseId": "eGenix", + "seeAlso": [ + "http://www.egenix.com/products/eGenix.com-Public-License-1.1.0.pdf", + "https://fedoraproject.org/wiki/Licensing/eGenix.com_Public_License_1.1.0" + ], + "isOsiApproved": false + }, + { + "reference": "https://spdx.org/licenses/LGPLLR.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/LGPLLR.json", + "referenceNumber": 19, + "name": "Lesser General Public License For Linguistic Resources", + "licenseId": "LGPLLR", + "seeAlso": [ + "http://www-igm.univ-mlv.fr/~unitex/lgpllr.html" + ], + "isOsiApproved": false + }, + { + "reference": "https://spdx.org/licenses/OLDAP-2.2.2.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/OLDAP-2.2.2.json", + "referenceNumber": 20, + "name": "Open LDAP Public License 2.2.2", + "licenseId": "OLDAP-2.2.2", + "seeAlso": [ + "http://www.openldap.org/devel/gitweb.cgi?p\u003dopenldap.git;a\u003dblob;f\u003dLICENSE;hb\u003ddf2cc1e21eb7c160695f5b7cffd6296c151ba188" + ], + "isOsiApproved": false + }, + { + "reference": "https://spdx.org/licenses/CC-BY-ND-3.0-DE.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/CC-BY-ND-3.0-DE.json", + "referenceNumber": 21, + "name": "Creative Commons Attribution No Derivatives 3.0 Germany", + "licenseId": "CC-BY-ND-3.0-DE", + "seeAlso": [ + "https://creativecommons.org/licenses/by-nd/3.0/de/legalcode" + ], + "isOsiApproved": false + }, + { + "reference": "https://spdx.org/licenses/IPA.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/IPA.json", + "referenceNumber": 22, + "name": "IPA Font License", + "licenseId": "IPA", + "seeAlso": [ + "https://opensource.org/licenses/IPA" + ], + "isOsiApproved": true, + "isFsfLibre": true + }, + { + "reference": "https://spdx.org/licenses/NCSA.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/NCSA.json", + "referenceNumber": 23, + "name": "University of Illinois/NCSA Open Source License", + "licenseId": "NCSA", + "seeAlso": [ + "http://otm.illinois.edu/uiuc_openSource", + "https://opensource.org/licenses/NCSA" + ], + "isOsiApproved": true, + "isFsfLibre": true + }, + { + "reference": "https://spdx.org/licenses/W3C.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/W3C.json", + "referenceNumber": 24, + "name": "W3C Software Notice and License (2002-12-31)", + "licenseId": "W3C", + "seeAlso": [ + "http://www.w3.org/Consortium/Legal/2002/copyright-software-20021231.html", + "https://opensource.org/licenses/W3C" + ], + "isOsiApproved": true, + "isFsfLibre": true + }, + { + "reference": "https://spdx.org/licenses/Adobe-2006.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/Adobe-2006.json", + "referenceNumber": 25, + "name": "Adobe Systems Incorporated Source Code License Agreement", + "licenseId": "Adobe-2006", + "seeAlso": [ + "https://fedoraproject.org/wiki/Licensing/AdobeLicense" + ], + "isOsiApproved": false + }, + { + "reference": "https://spdx.org/licenses/Net-SNMP.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/Net-SNMP.json", + "referenceNumber": 26, + "name": "Net-SNMP License", + "licenseId": "Net-SNMP", + "seeAlso": [ + "http://net-snmp.sourceforge.net/about/license.html" + ], + "isOsiApproved": false + }, + { + "reference": "https://spdx.org/licenses/CC-BY-SA-4.0.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/CC-BY-SA-4.0.json", + "referenceNumber": 27, + "name": "Creative Commons Attribution Share Alike 4.0 International", + "licenseId": "CC-BY-SA-4.0", + "seeAlso": [ + "https://creativecommons.org/licenses/by-sa/4.0/legalcode" + ], + "isOsiApproved": false, + "isFsfLibre": true + }, + { + "reference": "https://spdx.org/licenses/YPL-1.0.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/YPL-1.0.json", + "referenceNumber": 28, + "name": "Yahoo! Public License v1.0", + "licenseId": "YPL-1.0", + "seeAlso": [ + "http://www.zimbra.com/license/yahoo_public_license_1.0.html" + ], + "isOsiApproved": false + }, + { + "reference": "https://spdx.org/licenses/Nunit.html", + "isDeprecatedLicenseId": true, + "detailsUrl": "https://spdx.org/licenses/Nunit.json", + "referenceNumber": 29, + "name": "Nunit License", + "licenseId": "Nunit", + "seeAlso": [ + "https://fedoraproject.org/wiki/Licensing/Nunit" + ], + "isOsiApproved": false + }, + { + "reference": "https://spdx.org/licenses/MITNFA.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/MITNFA.json", + "referenceNumber": 30, + "name": "MIT +no-false-attribs license", + "licenseId": "MITNFA", + "seeAlso": [ + "https://fedoraproject.org/wiki/Licensing/MITNFA" + ], + "isOsiApproved": false + }, + { + "reference": "https://spdx.org/licenses/PHP-3.01.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/PHP-3.01.json", + "referenceNumber": 31, + "name": "PHP License v3.01", + "licenseId": "PHP-3.01", + "seeAlso": [ + "http://www.php.net/license/3_01.txt" + ], + "isOsiApproved": true, + "isFsfLibre": true + }, + { + "reference": "https://spdx.org/licenses/BSD-Source-Code.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/BSD-Source-Code.json", + "referenceNumber": 32, + "name": "BSD Source Code Attribution", + "licenseId": "BSD-Source-Code", + "seeAlso": [ + "https://github.com/robbiehanson/CocoaHTTPServer/blob/master/LICENSE.txt" + ], + "isOsiApproved": false + }, + { + "reference": "https://spdx.org/licenses/CC-BY-SA-2.5.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/CC-BY-SA-2.5.json", + "referenceNumber": 33, + "name": "Creative Commons Attribution Share Alike 2.5 Generic", + "licenseId": "CC-BY-SA-2.5", + "seeAlso": [ + "https://creativecommons.org/licenses/by-sa/2.5/legalcode" + ], + "isOsiApproved": false + }, + { + "reference": "https://spdx.org/licenses/Motosoto.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/Motosoto.json", + "referenceNumber": 34, + "name": "Motosoto License", + "licenseId": "Motosoto", + "seeAlso": [ + "https://opensource.org/licenses/Motosoto" + ], + "isOsiApproved": true + }, + { + "reference": "https://spdx.org/licenses/OSL-1.1.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/OSL-1.1.json", + "referenceNumber": 35, + "name": "Open Software License 1.1", + "licenseId": "OSL-1.1", + "seeAlso": [ + "https://fedoraproject.org/wiki/Licensing/OSL1.1" + ], + "isOsiApproved": false, + "isFsfLibre": true + }, + { + "reference": "https://spdx.org/licenses/NGPL.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/NGPL.json", + "referenceNumber": 36, + "name": "Nethack General Public License", + "licenseId": "NGPL", + "seeAlso": [ + "https://opensource.org/licenses/NGPL" + ], + "isOsiApproved": true + }, + { + "reference": "https://spdx.org/licenses/CC-BY-2.5-AU.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/CC-BY-2.5-AU.json", + "referenceNumber": 37, + "name": "Creative Commons Attribution 2.5 Australia", + "licenseId": "CC-BY-2.5-AU", + "seeAlso": [ + "https://creativecommons.org/licenses/by/2.5/au/legalcode" + ], + "isOsiApproved": false + }, + { + "reference": "https://spdx.org/licenses/Unicode-TOU.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/Unicode-TOU.json", + "referenceNumber": 38, + "name": "Unicode Terms of Use", + "licenseId": "Unicode-TOU", + "seeAlso": [ + "http://www.unicode.org/copyright.html" + ], + "isOsiApproved": false + }, + { + "reference": "https://spdx.org/licenses/BSD-3-Clause-No-Nuclear-License.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/BSD-3-Clause-No-Nuclear-License.json", + "referenceNumber": 39, + "name": "BSD 3-Clause No Nuclear License", + "licenseId": "BSD-3-Clause-No-Nuclear-License", + "seeAlso": [ + "http://download.oracle.com/otn-pub/java/licenses/bsd.txt?AuthParam\u003d1467140197_43d516ce1776bd08a58235a7785be1cc" + ], + "isOsiApproved": false + }, + { + "reference": "https://spdx.org/licenses/OPUBL-1.0.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/OPUBL-1.0.json", + "referenceNumber": 40, + "name": "Open Publication License v1.0", + "licenseId": "OPUBL-1.0", + "seeAlso": [ + "http://opencontent.org/openpub/", + "https://www.debian.org/opl", + "https://www.ctan.org/license/opl" + ], + "isOsiApproved": false + }, + { + "reference": "https://spdx.org/licenses/CC-BY-NC-SA-2.0-UK.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/CC-BY-NC-SA-2.0-UK.json", + "referenceNumber": 41, + "name": "Creative Commons Attribution Non Commercial Share Alike 2.0 England and Wales", + "licenseId": "CC-BY-NC-SA-2.0-UK", + "seeAlso": [ + "https://creativecommons.org/licenses/by-nc-sa/2.0/uk/legalcode" + ], + "isOsiApproved": false + }, + { + "reference": "https://spdx.org/licenses/NLOD-2.0.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/NLOD-2.0.json", + "referenceNumber": 42, + "name": "Norwegian Licence for Open Government Data (NLOD) 2.0", + "licenseId": "NLOD-2.0", + "seeAlso": [ + "http://data.norge.no/nlod/en/2.0" + ], + "isOsiApproved": false + }, + { + "reference": "https://spdx.org/licenses/gnuplot.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/gnuplot.json", + "referenceNumber": 43, + "name": "gnuplot License", + "licenseId": "gnuplot", + "seeAlso": [ + "https://fedoraproject.org/wiki/Licensing/Gnuplot" + ], + "isOsiApproved": false, + "isFsfLibre": true + }, + { + "reference": "https://spdx.org/licenses/EPICS.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/EPICS.json", + "referenceNumber": 44, + "name": "EPICS Open License", + "licenseId": "EPICS", + "seeAlso": [ + "https://epics.anl.gov/license/open.php" + ], + "isOsiApproved": false + }, + { + "reference": "https://spdx.org/licenses/Info-ZIP.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/Info-ZIP.json", + "referenceNumber": 45, + "name": "Info-ZIP License", + "licenseId": "Info-ZIP", + "seeAlso": [ + "http://www.info-zip.org/license.html" + ], + "isOsiApproved": false + }, + { + "reference": "https://spdx.org/licenses/OLDAP-2.0.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/OLDAP-2.0.json", + "referenceNumber": 46, + "name": "Open LDAP Public License v2.0 (or possibly 2.0A and 2.0B)", + "licenseId": "OLDAP-2.0", + "seeAlso": [ + "http://www.openldap.org/devel/gitweb.cgi?p\u003dopenldap.git;a\u003dblob;f\u003dLICENSE;hb\u003dcbf50f4e1185a21abd4c0a54d3f4341fe28f36ea" + ], + "isOsiApproved": false + }, + { + "reference": "https://spdx.org/licenses/CERN-OHL-P-2.0.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/CERN-OHL-P-2.0.json", + "referenceNumber": 47, + "name": "CERN Open Hardware Licence Version 2 - Permissive", + "licenseId": "CERN-OHL-P-2.0", + "seeAlso": [ + "https://www.ohwr.org/project/cernohl/wikis/Documents/CERN-OHL-version-2" + ], + "isOsiApproved": true + }, + { + "reference": "https://spdx.org/licenses/BSD-3-Clause-No-Nuclear-Warranty.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/BSD-3-Clause-No-Nuclear-Warranty.json", + "referenceNumber": 48, + "name": "BSD 3-Clause No Nuclear Warranty", + "licenseId": "BSD-3-Clause-No-Nuclear-Warranty", + "seeAlso": [ + "https://jogamp.org/git/?p\u003dgluegen.git;a\u003dblob_plain;f\u003dLICENSE.txt" + ], + "isOsiApproved": false + }, + { + "reference": "https://spdx.org/licenses/AML.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/AML.json", + "referenceNumber": 49, + "name": "Apple MIT License", + "licenseId": "AML", + "seeAlso": [ + "https://fedoraproject.org/wiki/Licensing/Apple_MIT_License" + ], + "isOsiApproved": false + }, + { + "reference": "https://spdx.org/licenses/MulanPSL-1.0.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/MulanPSL-1.0.json", + "referenceNumber": 50, + "name": "Mulan Permissive Software License, Version 1", + "licenseId": "MulanPSL-1.0", + "seeAlso": [ + "https://license.coscl.org.cn/MulanPSL/", + "https://github.com/yuwenlong/longphp/blob/25dfb70cc2a466dc4bb55ba30901cbce08d164b5/LICENSE" + ], + "isOsiApproved": false + }, + { + "reference": "https://spdx.org/licenses/Multics.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/Multics.json", + "referenceNumber": 51, + "name": "Multics License", + "licenseId": "Multics", + "seeAlso": [ + "https://opensource.org/licenses/Multics" + ], + "isOsiApproved": true + }, + { + "reference": "https://spdx.org/licenses/VSL-1.0.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/VSL-1.0.json", + "referenceNumber": 52, + "name": "Vovida Software License v1.0", + "licenseId": "VSL-1.0", + "seeAlso": [ + "https://opensource.org/licenses/VSL-1.0" + ], + "isOsiApproved": true + }, + { + "reference": "https://spdx.org/licenses/RSA-MD.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/RSA-MD.json", + "referenceNumber": 53, + "name": "RSA Message-Digest License", + "licenseId": "RSA-MD", + "seeAlso": [ + "http://www.faqs.org/rfcs/rfc1321.html" + ], + "isOsiApproved": false + }, + { + "reference": "https://spdx.org/licenses/CC-PDDC.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/CC-PDDC.json", + "referenceNumber": 54, + "name": "Creative Commons Public Domain Dedication and Certification", + "licenseId": "CC-PDDC", + "seeAlso": [ + "https://creativecommons.org/licenses/publicdomain/" + ], + "isOsiApproved": false + }, + { + "reference": "https://spdx.org/licenses/CC-BY-SA-2.1-JP.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/CC-BY-SA-2.1-JP.json", + "referenceNumber": 55, + "name": "Creative Commons Attribution Share Alike 2.1 Japan", + "licenseId": "CC-BY-SA-2.1-JP", + "seeAlso": [ + "https://creativecommons.org/licenses/by-sa/2.1/jp/legalcode" + ], + "isOsiApproved": false + }, + { + "reference": "https://spdx.org/licenses/LPPL-1.2.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/LPPL-1.2.json", + "referenceNumber": 56, + "name": "LaTeX Project Public License v1.2", + "licenseId": "LPPL-1.2", + "seeAlso": [ + "http://www.latex-project.org/lppl/lppl-1-2.txt" + ], + "isOsiApproved": false, + "isFsfLibre": true + }, + { + "reference": "https://spdx.org/licenses/Spencer-94.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/Spencer-94.json", + "referenceNumber": 57, + "name": "Spencer License 94", + "licenseId": "Spencer-94", + "seeAlso": [ + "https://fedoraproject.org/wiki/Licensing/Henry_Spencer_Reg-Ex_Library_License" + ], + "isOsiApproved": false + }, + { + "reference": "https://spdx.org/licenses/OLDAP-1.2.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/OLDAP-1.2.json", + "referenceNumber": 58, + "name": "Open LDAP Public License v1.2", + "licenseId": "OLDAP-1.2", + "seeAlso": [ + "http://www.openldap.org/devel/gitweb.cgi?p\u003dopenldap.git;a\u003dblob;f\u003dLICENSE;hb\u003d42b0383c50c299977b5893ee695cf4e486fb0dc7" + ], + "isOsiApproved": false + }, + { + "reference": "https://spdx.org/licenses/O-UDA-1.0.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/O-UDA-1.0.json", + "referenceNumber": 59, + "name": "Open Use of Data Agreement v1.0", + "licenseId": "O-UDA-1.0", + "seeAlso": [ + "https://github.com/microsoft/Open-Use-of-Data-Agreement/blob/v1.0/O-UDA-1.0.md", + "https://cdla.dev/open-use-of-data-agreement-v1-0/" + ], + "isOsiApproved": false + }, + { + "reference": "https://spdx.org/licenses/OLDAP-2.7.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/OLDAP-2.7.json", + "referenceNumber": 60, + "name": "Open LDAP Public License v2.7", + "licenseId": "OLDAP-2.7", + "seeAlso": [ + "http://www.openldap.org/devel/gitweb.cgi?p\u003dopenldap.git;a\u003dblob;f\u003dLICENSE;hb\u003d47c2415c1df81556eeb39be6cad458ef87c534a2" + ], + "isOsiApproved": false, + "isFsfLibre": true + }, + { + "reference": "https://spdx.org/licenses/Glulxe.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/Glulxe.json", + "referenceNumber": 61, + "name": "Glulxe License", + "licenseId": "Glulxe", + "seeAlso": [ + "https://fedoraproject.org/wiki/Licensing/Glulxe" + ], + "isOsiApproved": false + }, + { + "reference": "https://spdx.org/licenses/iMatix.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/iMatix.json", + "referenceNumber": 62, + "name": "iMatix Standard Function Library Agreement", + "licenseId": "iMatix", + "seeAlso": [ + "http://legacy.imatix.com/html/sfl/sfl4.htm#license" + ], + "isOsiApproved": false, + "isFsfLibre": true + }, + { + "reference": "https://spdx.org/licenses/TAPR-OHL-1.0.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/TAPR-OHL-1.0.json", + "referenceNumber": 63, + "name": "TAPR Open Hardware License v1.0", + "licenseId": "TAPR-OHL-1.0", + "seeAlso": [ + "https://www.tapr.org/OHL" + ], + "isOsiApproved": false + }, + { + "reference": "https://spdx.org/licenses/NBPL-1.0.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/NBPL-1.0.json", + "referenceNumber": 64, + "name": "Net Boolean Public License v1", + "licenseId": "NBPL-1.0", + "seeAlso": [ + "http://www.openldap.org/devel/gitweb.cgi?p\u003dopenldap.git;a\u003dblob;f\u003dLICENSE;hb\u003d37b4b3f6cc4bf34e1d3dec61e69914b9819d8894" + ], + "isOsiApproved": false + }, + { + "reference": "https://spdx.org/licenses/LiLiQ-R-1.1.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/LiLiQ-R-1.1.json", + "referenceNumber": 65, + "name": "Licence Libre du Québec – Réciprocité version 1.1", + "licenseId": "LiLiQ-R-1.1", + "seeAlso": [ + "https://www.forge.gouv.qc.ca/participez/licence-logicielle/licence-libre-du-quebec-liliq-en-francais/licence-libre-du-quebec-reciprocite-liliq-r-v1-1/", + "http://opensource.org/licenses/LiLiQ-R-1.1" + ], + "isOsiApproved": true + }, + { + "reference": "https://spdx.org/licenses/Noweb.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/Noweb.json", + "referenceNumber": 66, + "name": "Noweb License", + "licenseId": "Noweb", + "seeAlso": [ + "https://fedoraproject.org/wiki/Licensing/Noweb" + ], + "isOsiApproved": false + }, + { + "reference": "https://spdx.org/licenses/CC0-1.0.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/CC0-1.0.json", + "referenceNumber": 67, + "name": "Creative Commons Zero v1.0 Universal", + "licenseId": "CC0-1.0", + "seeAlso": [ + "https://creativecommons.org/publicdomain/zero/1.0/legalcode" + ], + "isOsiApproved": false, + "isFsfLibre": true + }, + { + "reference": "https://spdx.org/licenses/BSD-Protection.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/BSD-Protection.json", + "referenceNumber": 68, + "name": "BSD Protection License", + "licenseId": "BSD-Protection", + "seeAlso": [ + "https://fedoraproject.org/wiki/Licensing/BSD_Protection_License" + ], + "isOsiApproved": false + }, + { + "reference": "https://spdx.org/licenses/CC-BY-NC-2.5.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/CC-BY-NC-2.5.json", + "referenceNumber": 69, + "name": "Creative Commons Attribution Non Commercial 2.5 Generic", + "licenseId": "CC-BY-NC-2.5", + "seeAlso": [ + "https://creativecommons.org/licenses/by-nc/2.5/legalcode" + ], + "isOsiApproved": false + }, + { + "reference": "https://spdx.org/licenses/Zlib.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/Zlib.json", + "referenceNumber": 70, + "name": "zlib License", + "licenseId": "Zlib", + "seeAlso": [ + "http://www.zlib.net/zlib_license.html", + "https://opensource.org/licenses/Zlib" + ], + "isOsiApproved": true, + "isFsfLibre": true + }, + { + "reference": "https://spdx.org/licenses/GFDL-1.3-invariants-or-later.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/GFDL-1.3-invariants-or-later.json", + "referenceNumber": 71, + "name": "GNU Free Documentation License v1.3 or later - invariants", + "licenseId": "GFDL-1.3-invariants-or-later", + "seeAlso": [ + "https://www.gnu.org/licenses/fdl-1.3.txt" + ], + "isOsiApproved": false + }, + { + "reference": "https://spdx.org/licenses/CC-BY-3.0-AT.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/CC-BY-3.0-AT.json", + "referenceNumber": 72, + "name": "Creative Commons Attribution 3.0 Austria", + "licenseId": "CC-BY-3.0-AT", + "seeAlso": [ + "https://creativecommons.org/licenses/by/3.0/at/legalcode" + ], + "isOsiApproved": false + }, + { + "reference": "https://spdx.org/licenses/LPPL-1.3c.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/LPPL-1.3c.json", + "referenceNumber": 73, + "name": "LaTeX Project Public License v1.3c", + "licenseId": "LPPL-1.3c", + "seeAlso": [ + "http://www.latex-project.org/lppl/lppl-1-3c.txt", + "https://opensource.org/licenses/LPPL-1.3c" + ], + "isOsiApproved": true + }, + { + "reference": "https://spdx.org/licenses/EPL-1.0.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/EPL-1.0.json", + "referenceNumber": 74, + "name": "Eclipse Public License 1.0", + "licenseId": "EPL-1.0", + "seeAlso": [ + "http://www.eclipse.org/legal/epl-v10.html", + "https://opensource.org/licenses/EPL-1.0" + ], + "isOsiApproved": true, + "isFsfLibre": true + }, + { + "reference": "https://spdx.org/licenses/GFDL-1.1-invariants-or-later.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/GFDL-1.1-invariants-or-later.json", + "referenceNumber": 75, + "name": "GNU Free Documentation License v1.1 or later - invariants", + "licenseId": "GFDL-1.1-invariants-or-later", + "seeAlso": [ + "https://www.gnu.org/licenses/old-licenses/fdl-1.1.txt" + ], + "isOsiApproved": false + }, + { + "reference": "https://spdx.org/licenses/ANTLR-PD-fallback.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/ANTLR-PD-fallback.json", + "referenceNumber": 76, + "name": "ANTLR Software Rights Notice with license fallback", + "licenseId": "ANTLR-PD-fallback", + "seeAlso": [ + "http://www.antlr2.org/license.html" + ], + "isOsiApproved": false + }, + { + "reference": "https://spdx.org/licenses/OLDAP-2.4.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/OLDAP-2.4.json", + "referenceNumber": 77, + "name": "Open LDAP Public License v2.4", + "licenseId": "OLDAP-2.4", + "seeAlso": [ + "http://www.openldap.org/devel/gitweb.cgi?p\u003dopenldap.git;a\u003dblob;f\u003dLICENSE;hb\u003dcd1284c4a91a8a380d904eee68d1583f989ed386" + ], + "isOsiApproved": false + }, + { + "reference": "https://spdx.org/licenses/OLDAP-2.3.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/OLDAP-2.3.json", + "referenceNumber": 78, + "name": "Open LDAP Public License v2.3", + "licenseId": "OLDAP-2.3", + "seeAlso": [ + "http://www.openldap.org/devel/gitweb.cgi?p\u003dopenldap.git;a\u003dblob;f\u003dLICENSE;hb\u003dd32cf54a32d581ab475d23c810b0a7fbaf8d63c3" + ], + "isOsiApproved": false, + "isFsfLibre": true + }, + { + "reference": "https://spdx.org/licenses/ZPL-2.1.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/ZPL-2.1.json", + "referenceNumber": 79, + "name": "Zope Public License 2.1", + "licenseId": "ZPL-2.1", + "seeAlso": [ + "http://old.zope.org/Resources/ZPL/" + ], + "isOsiApproved": true, + "isFsfLibre": true + }, + { + "reference": "https://spdx.org/licenses/Apache-2.0.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/Apache-2.0.json", + "referenceNumber": 80, + "name": "Apache License 2.0", + "licenseId": "Apache-2.0", + "seeAlso": [ + "https://www.apache.org/licenses/LICENSE-2.0", + "https://opensource.org/licenses/Apache-2.0" + ], + "isOsiApproved": true, + "isFsfLibre": true + }, + { + "reference": "https://spdx.org/licenses/SGI-B-2.0.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/SGI-B-2.0.json", + "referenceNumber": 81, + "name": "SGI Free Software License B v2.0", + "licenseId": "SGI-B-2.0", + "seeAlso": [ + "http://oss.sgi.com/projects/FreeB/SGIFreeSWLicB.2.0.pdf" + ], + "isOsiApproved": false, + "isFsfLibre": true + }, + { + "reference": "https://spdx.org/licenses/Hippocratic-2.1.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/Hippocratic-2.1.json", + "referenceNumber": 82, + "name": "Hippocratic License 2.1", + "licenseId": "Hippocratic-2.1", + "seeAlso": [ + "https://firstdonoharm.dev/version/2/1/license.html", + "https://github.com/EthicalSource/hippocratic-license/blob/58c0e646d64ff6fbee275bfe2b9492f914e3ab2a/LICENSE.txt" + ], + "isOsiApproved": false + }, + { + "reference": "https://spdx.org/licenses/CC-BY-SA-3.0-DE.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/CC-BY-SA-3.0-DE.json", + "referenceNumber": 83, + "name": "Creative Commons Attribution Share Alike 3.0 Germany", + "licenseId": "CC-BY-SA-3.0-DE", + "seeAlso": [ + "https://creativecommons.org/licenses/by-sa/3.0/de/legalcode" + ], + "isOsiApproved": false + }, + { + "reference": "https://spdx.org/licenses/CC-BY-NC-SA-1.0.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/CC-BY-NC-SA-1.0.json", + "referenceNumber": 84, + "name": "Creative Commons Attribution Non Commercial Share Alike 1.0 Generic", + "licenseId": "CC-BY-NC-SA-1.0", + "seeAlso": [ + "https://creativecommons.org/licenses/by-nc-sa/1.0/legalcode" + ], + "isOsiApproved": false + }, + { + "reference": "https://spdx.org/licenses/LGPL-2.1-or-later.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/LGPL-2.1-or-later.json", + "referenceNumber": 85, + "name": "GNU Lesser General Public License v2.1 or later", + "licenseId": "LGPL-2.1-or-later", + "seeAlso": [ + "https://www.gnu.org/licenses/old-licenses/lgpl-2.1-standalone.html", + "https://opensource.org/licenses/LGPL-2.1" + ], + "isOsiApproved": true, + "isFsfLibre": true + }, + { + "reference": "https://spdx.org/licenses/CC-BY-3.0-US.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/CC-BY-3.0-US.json", + "referenceNumber": 86, + "name": "Creative Commons Attribution 3.0 United States", + "licenseId": "CC-BY-3.0-US", + "seeAlso": [ + "https://creativecommons.org/licenses/by/3.0/us/legalcode" + ], + "isOsiApproved": false + }, + { + "reference": "https://spdx.org/licenses/TCP-wrappers.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/TCP-wrappers.json", + "referenceNumber": 87, + "name": "TCP Wrappers License", + "licenseId": "TCP-wrappers", + "seeAlso": [ + "http://rc.quest.com/topics/openssh/license.php#tcpwrappers" + ], + "isOsiApproved": false + }, + { + "reference": "https://spdx.org/licenses/GFDL-1.2-invariants-or-later.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/GFDL-1.2-invariants-or-later.json", + "referenceNumber": 88, + "name": "GNU Free Documentation License v1.2 or later - invariants", + "licenseId": "GFDL-1.2-invariants-or-later", + "seeAlso": [ + "https://www.gnu.org/licenses/old-licenses/fdl-1.2.txt" + ], + "isOsiApproved": false + }, + { + "reference": "https://spdx.org/licenses/Eurosym.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/Eurosym.json", + "referenceNumber": 89, + "name": "Eurosym License", + "licenseId": "Eurosym", + "seeAlso": [ + "https://fedoraproject.org/wiki/Licensing/Eurosym" + ], + "isOsiApproved": false + }, + { + "reference": "https://spdx.org/licenses/GFDL-1.1.html", + "isDeprecatedLicenseId": true, + "detailsUrl": "https://spdx.org/licenses/GFDL-1.1.json", + "referenceNumber": 90, + "name": "GNU Free Documentation License v1.1", + "licenseId": "GFDL-1.1", + "seeAlso": [ + "https://www.gnu.org/licenses/old-licenses/fdl-1.1.txt" + ], + "isOsiApproved": false, + "isFsfLibre": true + }, + { + "reference": "https://spdx.org/licenses/LPPL-1.0.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/LPPL-1.0.json", + "referenceNumber": 91, + "name": "LaTeX Project Public License v1.0", + "licenseId": "LPPL-1.0", + "seeAlso": [ + "http://www.latex-project.org/lppl/lppl-1-0.txt" + ], + "isOsiApproved": false + }, + { + "reference": "https://spdx.org/licenses/LGPL-2.0+.html", + "isDeprecatedLicenseId": true, + "detailsUrl": "https://spdx.org/licenses/LGPL-2.0+.json", + "referenceNumber": 92, + "name": "GNU Library General Public License v2 or later", + "licenseId": "LGPL-2.0+", + "seeAlso": [ + "https://www.gnu.org/licenses/old-licenses/lgpl-2.0-standalone.html" + ], + "isOsiApproved": true + }, + { + "reference": "https://spdx.org/licenses/SGI-B-1.0.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/SGI-B-1.0.json", + "referenceNumber": 93, + "name": "SGI Free Software License B v1.0", + "licenseId": "SGI-B-1.0", + "seeAlso": [ + "http://oss.sgi.com/projects/FreeB/SGIFreeSWLicB.1.0.html" + ], + "isOsiApproved": false + }, + { + "reference": "https://spdx.org/licenses/APL-1.0.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/APL-1.0.json", + "referenceNumber": 94, + "name": "Adaptive Public License 1.0", + "licenseId": "APL-1.0", + "seeAlso": [ + "https://opensource.org/licenses/APL-1.0" + ], + "isOsiApproved": true + }, + { + "reference": "https://spdx.org/licenses/libtiff.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/libtiff.json", + "referenceNumber": 95, + "name": "libtiff License", + "licenseId": "libtiff", + "seeAlso": [ + "https://fedoraproject.org/wiki/Licensing/libtiff" + ], + "isOsiApproved": false + }, + { + "reference": "https://spdx.org/licenses/AFL-2.1.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/AFL-2.1.json", + "referenceNumber": 96, + "name": "Academic Free License v2.1", + "licenseId": "AFL-2.1", + "seeAlso": [ + "http://opensource.linux-mirror.org/licenses/afl-2.1.txt" + ], + "isOsiApproved": true, + "isFsfLibre": true + }, + { + "reference": "https://spdx.org/licenses/CC-BY-NC-1.0.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/CC-BY-NC-1.0.json", + "referenceNumber": 97, + "name": "Creative Commons Attribution Non Commercial 1.0 Generic", + "licenseId": "CC-BY-NC-1.0", + "seeAlso": [ + "https://creativecommons.org/licenses/by-nc/1.0/legalcode" + ], + "isOsiApproved": false + }, + { + "reference": "https://spdx.org/licenses/GD.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/GD.json", + "referenceNumber": 98, + "name": "GD License", + "licenseId": "GD", + "seeAlso": [ + "https://libgd.github.io/manuals/2.3.0/files/license-txt.html" + ], + "isOsiApproved": false + }, + { + "reference": "https://spdx.org/licenses/AFL-1.1.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/AFL-1.1.json", + "referenceNumber": 99, + "name": "Academic Free License v1.1", + "licenseId": "AFL-1.1", + "seeAlso": [ + "http://opensource.linux-mirror.org/licenses/afl-1.1.txt", + "http://wayback.archive.org/web/20021004124254/http://www.opensource.org/licenses/academic.php" + ], + "isOsiApproved": true, + "isFsfLibre": true + }, + { + "reference": "https://spdx.org/licenses/CC-BY-NC-ND-3.0-IGO.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/CC-BY-NC-ND-3.0-IGO.json", + "referenceNumber": 100, + "name": "Creative Commons Attribution Non Commercial No Derivatives 3.0 IGO", + "licenseId": "CC-BY-NC-ND-3.0-IGO", + "seeAlso": [ + "https://creativecommons.org/licenses/by-nc-nd/3.0/igo/legalcode" + ], + "isOsiApproved": false + }, + { + "reference": "https://spdx.org/licenses/Unicode-DFS-2015.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/Unicode-DFS-2015.json", + "referenceNumber": 101, + "name": "Unicode License Agreement - Data Files and Software (2015)", + "licenseId": "Unicode-DFS-2015", + "seeAlso": [ + "https://web.archive.org/web/20151224134844/http://unicode.org/copyright.html" + ], + "isOsiApproved": false + }, + { + "reference": "https://spdx.org/licenses/GFDL-1.2-only.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/GFDL-1.2-only.json", + "referenceNumber": 102, + "name": "GNU Free Documentation License v1.2 only", + "licenseId": "GFDL-1.2-only", + "seeAlso": [ + "https://www.gnu.org/licenses/old-licenses/fdl-1.2.txt" + ], + "isOsiApproved": false, + "isFsfLibre": true + }, + { + "reference": "https://spdx.org/licenses/MPL-1.1.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/MPL-1.1.json", + "referenceNumber": 103, + "name": "Mozilla Public License 1.1", + "licenseId": "MPL-1.1", + "seeAlso": [ + "http://www.mozilla.org/MPL/MPL-1.1.html", + "https://opensource.org/licenses/MPL-1.1" + ], + "isOsiApproved": true, + "isFsfLibre": true + }, + { + "reference": "https://spdx.org/licenses/GPL-2.0-only.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/GPL-2.0-only.json", + "referenceNumber": 104, + "name": "GNU General Public License v2.0 only", + "licenseId": "GPL-2.0-only", + "seeAlso": [ + "https://www.gnu.org/licenses/old-licenses/gpl-2.0-standalone.html", + "https://opensource.org/licenses/GPL-2.0" + ], + "isOsiApproved": true, + "isFsfLibre": true + }, + { + "reference": "https://spdx.org/licenses/CC-BY-NC-4.0.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/CC-BY-NC-4.0.json", + "referenceNumber": 105, + "name": "Creative Commons Attribution Non Commercial 4.0 International", + "licenseId": "CC-BY-NC-4.0", + "seeAlso": [ + "https://creativecommons.org/licenses/by-nc/4.0/legalcode" + ], + "isOsiApproved": false + }, + { + "reference": "https://spdx.org/licenses/FreeImage.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/FreeImage.json", + "referenceNumber": 106, + "name": "FreeImage Public License v1.0", + "licenseId": "FreeImage", + "seeAlso": [ + "http://freeimage.sourceforge.net/freeimage-license.txt" + ], + "isOsiApproved": false + }, + { + "reference": "https://spdx.org/licenses/SHL-0.51.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/SHL-0.51.json", + "referenceNumber": 107, + "name": "Solderpad Hardware License, Version 0.51", + "licenseId": "SHL-0.51", + "seeAlso": [ + "https://solderpad.org/licenses/SHL-0.51/" + ], + "isOsiApproved": false + }, + { + "reference": "https://spdx.org/licenses/CNRI-Jython.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/CNRI-Jython.json", + "referenceNumber": 108, + "name": "CNRI Jython License", + "licenseId": "CNRI-Jython", + "seeAlso": [ + "http://www.jython.org/license.html" + ], + "isOsiApproved": false + }, + { + "reference": "https://spdx.org/licenses/ZPL-1.1.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/ZPL-1.1.json", + "referenceNumber": 109, + "name": "Zope Public License 1.1", + "licenseId": "ZPL-1.1", + "seeAlso": [ + "http://old.zope.org/Resources/License/ZPL-1.1" + ], + "isOsiApproved": false + }, + { + "reference": "https://spdx.org/licenses/Afmparse.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/Afmparse.json", + "referenceNumber": 110, + "name": "Afmparse License", + "licenseId": "Afmparse", + "seeAlso": [ + "https://fedoraproject.org/wiki/Licensing/Afmparse" + ], + "isOsiApproved": false + }, + { + "reference": "https://spdx.org/licenses/OLDAP-2.1.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/OLDAP-2.1.json", + "referenceNumber": 111, + "name": "Open LDAP Public License v2.1", + "licenseId": "OLDAP-2.1", + "seeAlso": [ + "http://www.openldap.org/devel/gitweb.cgi?p\u003dopenldap.git;a\u003dblob;f\u003dLICENSE;hb\u003db0d176738e96a0d3b9f85cb51e140a86f21be715" + ], + "isOsiApproved": false + }, + { + "reference": "https://spdx.org/licenses/Rdisc.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/Rdisc.json", + "referenceNumber": 112, + "name": "Rdisc License", + "licenseId": "Rdisc", + "seeAlso": [ + "https://fedoraproject.org/wiki/Licensing/Rdisc_License" + ], + "isOsiApproved": false + }, + { + "reference": "https://spdx.org/licenses/Imlib2.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/Imlib2.json", + "referenceNumber": 113, + "name": "Imlib2 License", + "licenseId": "Imlib2", + "seeAlso": [ + "http://trac.enlightenment.org/e/browser/trunk/imlib2/COPYING", + "https://git.enlightenment.org/legacy/imlib2.git/tree/COPYING" + ], + "isOsiApproved": false, + "isFsfLibre": true + }, + { + "reference": "https://spdx.org/licenses/BSD-4-Clause-Shortened.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/BSD-4-Clause-Shortened.json", + "referenceNumber": 114, + "name": "BSD 4 Clause Shortened", + "licenseId": "BSD-4-Clause-Shortened", + "seeAlso": [ + "https://metadata.ftp-master.debian.org/changelogs//main/a/arpwatch/arpwatch_2.1a15-7_copyright" + ], + "isOsiApproved": false + }, + { + "reference": "https://spdx.org/licenses/Sendmail.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/Sendmail.json", + "referenceNumber": 115, + "name": "Sendmail License", + "licenseId": "Sendmail", + "seeAlso": [ + "http://www.sendmail.com/pdfs/open_source/sendmail_license.pdf", + "https://web.archive.org/web/20160322142305/https://www.sendmail.com/pdfs/open_source/sendmail_license.pdf" + ], + "isOsiApproved": false + }, + { + "reference": "https://spdx.org/licenses/CC-BY-2.5.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/CC-BY-2.5.json", + "referenceNumber": 116, + "name": "Creative Commons Attribution 2.5 Generic", + "licenseId": "CC-BY-2.5", + "seeAlso": [ + "https://creativecommons.org/licenses/by/2.5/legalcode" + ], + "isOsiApproved": false + }, + { + "reference": "https://spdx.org/licenses/AAL.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/AAL.json", + "referenceNumber": 117, + "name": "Attribution Assurance License", + "licenseId": "AAL", + "seeAlso": [ + "https://opensource.org/licenses/attribution" + ], + "isOsiApproved": true + }, + { + "reference": "https://spdx.org/licenses/MPL-2.0-no-copyleft-exception.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/MPL-2.0-no-copyleft-exception.json", + "referenceNumber": 118, + "name": "Mozilla Public License 2.0 (no copyleft exception)", + "licenseId": "MPL-2.0-no-copyleft-exception", + "seeAlso": [ + "http://www.mozilla.org/MPL/2.0/", + "https://opensource.org/licenses/MPL-2.0" + ], + "isOsiApproved": true + }, + { + "reference": "https://spdx.org/licenses/CC-BY-NC-ND-2.5.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/CC-BY-NC-ND-2.5.json", + "referenceNumber": 119, + "name": "Creative Commons Attribution Non Commercial No Derivatives 2.5 Generic", + "licenseId": "CC-BY-NC-ND-2.5", + "seeAlso": [ + "https://creativecommons.org/licenses/by-nc-nd/2.5/legalcode" + ], + "isOsiApproved": false + }, + { + "reference": "https://spdx.org/licenses/CC-BY-3.0-NL.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/CC-BY-3.0-NL.json", + "referenceNumber": 120, + "name": "Creative Commons Attribution 3.0 Netherlands", + "licenseId": "CC-BY-3.0-NL", + "seeAlso": [ + "https://creativecommons.org/licenses/by/3.0/nl/legalcode" + ], + "isOsiApproved": false + }, + { + "reference": "https://spdx.org/licenses/LPL-1.02.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/LPL-1.02.json", + "referenceNumber": 121, + "name": "Lucent Public License v1.02", + "licenseId": "LPL-1.02", + "seeAlso": [ + "http://plan9.bell-labs.com/plan9/license.html", + "https://opensource.org/licenses/LPL-1.02" + ], + "isOsiApproved": true, + "isFsfLibre": true + }, + { + "reference": "https://spdx.org/licenses/ECL-1.0.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/ECL-1.0.json", + "referenceNumber": 122, + "name": "Educational Community License v1.0", + "licenseId": "ECL-1.0", + "seeAlso": [ + "https://opensource.org/licenses/ECL-1.0" + ], + "isOsiApproved": true + }, + { + "reference": "https://spdx.org/licenses/OFL-1.0-no-RFN.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/OFL-1.0-no-RFN.json", + "referenceNumber": 123, + "name": "SIL Open Font License 1.0 with no Reserved Font Name", + "licenseId": "OFL-1.0-no-RFN", + "seeAlso": [ + "http://scripts.sil.org/cms/scripts/page.php?item_id\u003dOFL10_web" + ], + "isOsiApproved": false + }, + { + "reference": "https://spdx.org/licenses/CC-BY-NC-SA-3.0-DE.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/CC-BY-NC-SA-3.0-DE.json", + "referenceNumber": 124, + "name": "Creative Commons Attribution Non Commercial Share Alike 3.0 Germany", + "licenseId": "CC-BY-NC-SA-3.0-DE", + "seeAlso": [ + "https://creativecommons.org/licenses/by-nc-sa/3.0/de/legalcode" + ], + "isOsiApproved": false + }, + { + "reference": "https://spdx.org/licenses/CC-BY-SA-3.0.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/CC-BY-SA-3.0.json", + "referenceNumber": 125, + "name": "Creative Commons Attribution Share Alike 3.0 Unported", + "licenseId": "CC-BY-SA-3.0", + "seeAlso": [ + "https://creativecommons.org/licenses/by-sa/3.0/legalcode" + ], + "isOsiApproved": false + }, + { + "reference": "https://spdx.org/licenses/NTP.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/NTP.json", + "referenceNumber": 126, + "name": "NTP License", + "licenseId": "NTP", + "seeAlso": [ + "https://opensource.org/licenses/NTP" + ], + "isOsiApproved": true + }, + { + "reference": "https://spdx.org/licenses/MPL-2.0.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/MPL-2.0.json", + "referenceNumber": 127, + "name": "Mozilla Public License 2.0", + "licenseId": "MPL-2.0", + "seeAlso": [ + "https://www.mozilla.org/MPL/2.0/", + "https://opensource.org/licenses/MPL-2.0" + ], + "isOsiApproved": true, + "isFsfLibre": true + }, + { + "reference": "https://spdx.org/licenses/APSL-1.2.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/APSL-1.2.json", + "referenceNumber": 128, + "name": "Apple Public Source License 1.2", + "licenseId": "APSL-1.2", + "seeAlso": [ + "http://www.samurajdata.se/opensource/mirror/licenses/apsl.php" + ], + "isOsiApproved": true + }, + { + "reference": "https://spdx.org/licenses/GFDL-1.2-no-invariants-only.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/GFDL-1.2-no-invariants-only.json", + "referenceNumber": 129, + "name": "GNU Free Documentation License v1.2 only - no invariants", + "licenseId": "GFDL-1.2-no-invariants-only", + "seeAlso": [ + "https://www.gnu.org/licenses/old-licenses/fdl-1.2.txt" + ], + "isOsiApproved": false + }, + { + "reference": "https://spdx.org/licenses/Artistic-2.0.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/Artistic-2.0.json", + "referenceNumber": 130, + "name": "Artistic License 2.0", + "licenseId": "Artistic-2.0", + "seeAlso": [ + "http://www.perlfoundation.org/artistic_license_2_0", + "https://www.perlfoundation.org/artistic-license-20.html", + "https://opensource.org/licenses/artistic-license-2.0" + ], + "isOsiApproved": true, + "isFsfLibre": true + }, + { + "reference": "https://spdx.org/licenses/GPL-2.0.html", + "isDeprecatedLicenseId": true, + "detailsUrl": "https://spdx.org/licenses/GPL-2.0.json", + "referenceNumber": 131, + "name": "GNU General Public License v2.0 only", + "licenseId": "GPL-2.0", + "seeAlso": [ + "https://www.gnu.org/licenses/old-licenses/gpl-2.0-standalone.html", + "https://opensource.org/licenses/GPL-2.0" + ], + "isOsiApproved": true, + "isFsfLibre": true + }, + { + "reference": "https://spdx.org/licenses/RSCPL.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/RSCPL.json", + "referenceNumber": 132, + "name": "Ricoh Source Code Public License", + "licenseId": "RSCPL", + "seeAlso": [ + "http://wayback.archive.org/web/20060715140826/http://www.risource.org/RPL/RPL-1.0A.shtml", + "https://opensource.org/licenses/RSCPL" + ], + "isOsiApproved": true + }, + { + "reference": "https://spdx.org/licenses/Sleepycat.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/Sleepycat.json", + "referenceNumber": 133, + "name": "Sleepycat License", + "licenseId": "Sleepycat", + "seeAlso": [ + "https://opensource.org/licenses/Sleepycat" + ], + "isOsiApproved": true, + "isFsfLibre": true + }, + { + "reference": "https://spdx.org/licenses/xpp.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/xpp.json", + "referenceNumber": 134, + "name": "XPP License", + "licenseId": "xpp", + "seeAlso": [ + "https://fedoraproject.org/wiki/Licensing/xpp" + ], + "isOsiApproved": false + }, + { + "reference": "https://spdx.org/licenses/CDLA-Sharing-1.0.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/CDLA-Sharing-1.0.json", + "referenceNumber": 135, + "name": "Community Data License Agreement Sharing 1.0", + "licenseId": "CDLA-Sharing-1.0", + "seeAlso": [ + "https://cdla.io/sharing-1-0" + ], + "isOsiApproved": false + }, + { + "reference": "https://spdx.org/licenses/ClArtistic.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/ClArtistic.json", + "referenceNumber": 136, + "name": "Clarified Artistic License", + "licenseId": "ClArtistic", + "seeAlso": [ + "http://gianluca.dellavedova.org/2011/01/03/clarified-artistic-license/", + "http://www.ncftp.com/ncftp/doc/LICENSE.txt" + ], + "isOsiApproved": false, + "isFsfLibre": true + }, + { + "reference": "https://spdx.org/licenses/AGPL-1.0-only.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/AGPL-1.0-only.json", + "referenceNumber": 137, + "name": "Affero General Public License v1.0 only", + "licenseId": "AGPL-1.0-only", + "seeAlso": [ + "http://www.affero.org/oagpl.html" + ], + "isOsiApproved": false + }, + { + "reference": "https://spdx.org/licenses/CC-BY-3.0-DE.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/CC-BY-3.0-DE.json", + "referenceNumber": 138, + "name": "Creative Commons Attribution 3.0 Germany", + "licenseId": "CC-BY-3.0-DE", + "seeAlso": [ + "https://creativecommons.org/licenses/by/3.0/de/legalcode" + ], + "isOsiApproved": false + }, + { + "reference": "https://spdx.org/licenses/AFL-2.0.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/AFL-2.0.json", + "referenceNumber": 139, + "name": "Academic Free License v2.0", + "licenseId": "AFL-2.0", + "seeAlso": [ + "http://wayback.archive.org/web/20060924134533/http://www.opensource.org/licenses/afl-2.0.txt" + ], + "isOsiApproved": true, + "isFsfLibre": true + }, + { + "reference": "https://spdx.org/licenses/Intel.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/Intel.json", + "referenceNumber": 140, + "name": "Intel Open Source License", + "licenseId": "Intel", + "seeAlso": [ + "https://opensource.org/licenses/Intel" + ], + "isOsiApproved": true, + "isFsfLibre": true + }, + { + "reference": "https://spdx.org/licenses/GFDL-1.1-no-invariants-or-later.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/GFDL-1.1-no-invariants-or-later.json", + "referenceNumber": 141, + "name": "GNU Free Documentation License v1.1 or later - no invariants", + "licenseId": "GFDL-1.1-no-invariants-or-later", + "seeAlso": [ + "https://www.gnu.org/licenses/old-licenses/fdl-1.1.txt" + ], + "isOsiApproved": false + }, + { + "reference": "https://spdx.org/licenses/APAFML.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/APAFML.json", + "referenceNumber": 142, + "name": "Adobe Postscript AFM License", + "licenseId": "APAFML", + "seeAlso": [ + "https://fedoraproject.org/wiki/Licensing/AdobePostscriptAFM" + ], + "isOsiApproved": false + }, + { + "reference": "https://spdx.org/licenses/GFDL-1.2.html", + "isDeprecatedLicenseId": true, + "detailsUrl": "https://spdx.org/licenses/GFDL-1.2.json", + "referenceNumber": 143, + "name": "GNU Free Documentation License v1.2", + "licenseId": "GFDL-1.2", + "seeAlso": [ + "https://www.gnu.org/licenses/old-licenses/fdl-1.2.txt" + ], + "isOsiApproved": false, + "isFsfLibre": true + }, + { + "reference": "https://spdx.org/licenses/SISSL.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/SISSL.json", + "referenceNumber": 144, + "name": "Sun Industry Standards Source License v1.1", + "licenseId": "SISSL", + "seeAlso": [ + "http://www.openoffice.org/licenses/sissl_license.html", + "https://opensource.org/licenses/SISSL" + ], + "isOsiApproved": true, + "isFsfLibre": true + }, + { + "reference": "https://spdx.org/licenses/Naumen.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/Naumen.json", + "referenceNumber": 145, + "name": "Naumen Public License", + "licenseId": "Naumen", + "seeAlso": [ + "https://opensource.org/licenses/Naumen" + ], + "isOsiApproved": true + }, + { + "reference": "https://spdx.org/licenses/HTMLTIDY.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/HTMLTIDY.json", + "referenceNumber": 146, + "name": "HTML Tidy License", + "licenseId": "HTMLTIDY", + "seeAlso": [ + "https://github.com/htacg/tidy-html5/blob/next/README/LICENSE.md" + ], + "isOsiApproved": false + }, + { + "reference": "https://spdx.org/licenses/OLDAP-2.8.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/OLDAP-2.8.json", + "referenceNumber": 147, + "name": "Open LDAP Public License v2.8", + "licenseId": "OLDAP-2.8", + "seeAlso": [ + "http://www.openldap.org/software/release/license.html" + ], + "isOsiApproved": true + }, + { + "reference": "https://spdx.org/licenses/blessing.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/blessing.json", + "referenceNumber": 148, + "name": "SQLite Blessing", + "licenseId": "blessing", + "seeAlso": [ + "https://www.sqlite.org/src/artifact/e33a4df7e32d742a?ln\u003d4-9", + "https://sqlite.org/src/artifact/df5091916dbb40e6" + ], + "isOsiApproved": false + }, + { + "reference": "https://spdx.org/licenses/CC-BY-ND-2.0.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/CC-BY-ND-2.0.json", + "referenceNumber": 149, + "name": "Creative Commons Attribution No Derivatives 2.0 Generic", + "licenseId": "CC-BY-ND-2.0", + "seeAlso": [ + "https://creativecommons.org/licenses/by-nd/2.0/legalcode" + ], + "isOsiApproved": false + }, + { + "reference": "https://spdx.org/licenses/OGTSL.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/OGTSL.json", + "referenceNumber": 150, + "name": "Open Group Test Suite License", + "licenseId": "OGTSL", + "seeAlso": [ + "http://www.opengroup.org/testing/downloads/The_Open_Group_TSL.txt", + "https://opensource.org/licenses/OGTSL" + ], + "isOsiApproved": true + }, + { + "reference": "https://spdx.org/licenses/LGPL-2.0-or-later.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/LGPL-2.0-or-later.json", + "referenceNumber": 151, + "name": "GNU Library General Public License v2 or later", + "licenseId": "LGPL-2.0-or-later", + "seeAlso": [ + "https://www.gnu.org/licenses/old-licenses/lgpl-2.0-standalone.html" + ], + "isOsiApproved": true + }, + { + "reference": "https://spdx.org/licenses/Parity-7.0.0.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/Parity-7.0.0.json", + "referenceNumber": 152, + "name": "The Parity Public License 7.0.0", + "licenseId": "Parity-7.0.0", + "seeAlso": [ + "https://paritylicense.com/versions/7.0.0.html" + ], + "isOsiApproved": false + }, + { + "reference": "https://spdx.org/licenses/CC-BY-ND-1.0.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/CC-BY-ND-1.0.json", + "referenceNumber": 153, + "name": "Creative Commons Attribution No Derivatives 1.0 Generic", + "licenseId": "CC-BY-ND-1.0", + "seeAlso": [ + "https://creativecommons.org/licenses/by-nd/1.0/legalcode" + ], + "isOsiApproved": false + }, + { + "reference": "https://spdx.org/licenses/dvipdfm.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/dvipdfm.json",Lines of this file are not shown. View the raw message.
The rest of this diff is not shown. View the raw message.
2.33.0
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#178957): https://lists.openembedded.org/g/openembedded-core/message/178957
Mute This Topic: https://lists.openembedded.org/mt/97787873/4947266
Group Owner: [email protected]
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [[email protected]]
-=-=-=-=-=-=-=-=-=-=-=-
Part of the diff is not shown here. View the raw message.
Re: [OE-core][dunfell][PATCH 1/5] classes/create-spdx: Backport
Quoted 927 lines · Ernst Sjöstrand
Den ons 22 mars 2023 kl 21:46 skrev Joshua Watt <[email protected]>:
Quoted 926 lines
Backports the create-spdx classes from the latest versions on master. This backport is a simple copy with no modifications, as its too complex to cherry-pick all the corresponding changes. This will give an appropriate base commit for subsequent changes and if necessary additional backport cherry-picks from master in the future.
Signed-off-by Joshua Watt <[email protected]>--- meta/classes/create-spdx-2.2.bbclass | 1069 +++++ meta/classes/create-spdx.bbclass | 8 + meta/files/spdx-licenses.json | 5937 ++++++++++++++++++++++++++ meta/lib/oe/sbom.py | 84 + meta/lib/oe/spdx.py | 357 ++ 5 files changed, 7455 insertions(+) create mode 100644 meta/classes/create-spdx-2.2.bbclass create mode 100644 meta/classes/create-spdx.bbclass create mode 100644 meta/files/spdx-licenses.json create mode 100644 meta/lib/oe/sbom.py create mode 100644 meta/lib/oe/spdx.pymeta/classes/create-spdx-2.2.bbclass new file +899
-0@@ -0,0 +1,1069 @@ +# +# Copyright OpenEmbedded Contributors +# +# SPDX-License-Identifier: GPL-2.0-only +# + +DEPLOY_DIR_SPDX ??= "${DEPLOY_DIR}/spdx/${MACHINE}" + +# The product name that the CVE database uses. Defaults to BPN, but may need to +# be overriden per recipe (for example tiff.bb sets CVE_PRODUCT=libtiff). +CVE_PRODUCT ??= "${BPN}" +CVE_VERSION ??= "${PV}" + +SPDXDIR ??= "${WORKDIR}/spdx" +SPDXDEPLOY = "${SPDXDIR}/deploy" +SPDXWORK = "${SPDXDIR}/work" +SPDXIMAGEWORK = "${SPDXDIR}/image-work" +SPDXSDKWORK = "${SPDXDIR}/sdk-work" + +SPDX_TOOL_NAME ??= "oe-spdx-creator" +SPDX_TOOL_VERSION ??= "1.0" + +SPDXRUNTIMEDEPLOY = "${SPDXDIR}/runtime-deploy" + +SPDX_INCLUDE_SOURCES ??= "0" +SPDX_ARCHIVE_SOURCES ??= "0" +SPDX_ARCHIVE_PACKAGED ??= "0" + +SPDX_UUID_NAMESPACE ??= "sbom.openembedded.org" +SPDX_NAMESPACE_PREFIX ??= "http://spdx.org/spdxdoc" +SPDX_PRETTY ??= "0" + +SPDX_LICENSES ??= "${COREBASE}/meta/files/spdx-licenses.json" + +SPDX_CUSTOM_ANNOTATION_VARS ??= "" + +SPDX_ORG ??= "OpenEmbedded ()" +SPDX_SUPPLIER ??= "Organization: ${SPDX_ORG}" +SPDX_SUPPLIER[doc] = "The SPDX PackageSupplier field for SPDX packages created from \ + this recipe. For SPDX documents create using this class during the build, this \ + is the contact information for the person or organization who is doing the \ + build." + +def extract_licenses(filename): + import re + + lic_regex = re.compile(rb'^\W*SPDX-License-Identifier:\s*([ \w\d.()+-]+?)(?:\s+\W*)?$', re.MULTILINE) + + try: + with open(filename, 'rb') as f: + size = min(15000, os.stat(filename).st_size) + txt = f.read(size) + licenses = re.findall(lic_regex, txt) + if licenses: + ascii_licenses = [lic.decode('ascii') for lic in licenses] + return ascii_licenses + except Exception as e: + bb.warn(f"Exception reading {filename}: {e}") + return None + +def get_doc_namespace(d, doc): + import uuid + namespace_uuid = uuid.uuid5(uuid.NAMESPACE_DNS, d.getVar("SPDX_UUID_NAMESPACE")) + return "%s/%s-%s" % (d.getVar("SPDX_NAMESPACE_PREFIX"), doc.name, str(uuid.uuid5(namespace_uuid, doc.name))) + +def create_annotation(d, comment): + from datetime import datetime, timezone + + creation_time = datetime.now(tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + annotation = oe.spdx.SPDXAnnotation() + annotation.annotationDate = creation_time + annotation.annotationType = "OTHER" + annotation.annotator = "Tool: %s - %s" % (d.getVar("SPDX_TOOL_NAME"), d.getVar("SPDX_TOOL_VERSION")) + annotation.comment = comment + return annotation + +def recipe_spdx_is_native(d, recipe): + return any(a.annotationType == "OTHER" and + a.annotator == "Tool: %s - %s" % (d.getVar("SPDX_TOOL_NAME"), d.getVar("SPDX_TOOL_VERSION")) and + a.comment == "isNative" for a in recipe.annotations) + +def is_work_shared_spdx(d): + return bb.data.inherits_class('kernel', d) or ('work-shared' in d.getVar('WORKDIR')) + +def get_json_indent(d): + if d.getVar("SPDX_PRETTY") == "1": + return 2 + return None + +python() { + import json + if d.getVar("SPDX_LICENSE_DATA"): + return + + with open(d.getVar("SPDX_LICENSES"), "r") as f: + data = json.load(f) + # Transform the license array to a dictionary + data["licenses"] = {l["licenseId"]: l for l in data["licenses"]} + d.setVar("SPDX_LICENSE_DATA", data) +} + +def convert_license_to_spdx(lic, document, d, existing={}): + from pathlib import Path + import oe.spdx + + license_data = d.getVar("SPDX_LICENSE_DATA") + extracted = {} + + def add_extracted_license(ident, name): + nonlocal document + + if name in extracted: + return + + extracted_info = oe.spdx.SPDXExtractedLicensingInfo() + extracted_info.name = name + extracted_info.licenseId = ident + extracted_info.extractedText = None + + if name == "PD": + # Special-case this. + extracted_info.extractedText = "Software released to the public domain" + else: + # Seach for the license in COMMON_LICENSE_DIR and LICENSE_PATH + for directory in [d.getVar('COMMON_LICENSE_DIR')] + (d.getVar('LICENSE_PATH') or '').split(): + try: + with (Path(directory) / name).open(errors="replace") as f: + extracted_info.extractedText = f.read() + break + except FileNotFoundError: + pass + if extracted_info.extractedText is None: + # If it's not SPDX or PD, then NO_GENERIC_LICENSE must be set + filename = d.getVarFlag('NO_GENERIC_LICENSE', name) + if filename: + filename = d.expand("${S}/" + filename) + with open(filename, errors="replace") as f: + extracted_info.extractedText = f.read() + else: + bb.error("Cannot find any text for license %s" % name) + + extracted[name] = extracted_info + document.hasExtractedLicensingInfos.append(extracted_info) + + def convert(l): + if l == "(" or l == ")": + return l + + if l == "&": + return "AND" + + if l == "|": + return "OR" + + if l == "CLOSED": + return "NONE" + + spdx_license = d.getVarFlag("SPDXLICENSEMAP", l) or l + if spdx_license in license_data["licenses"]: + return spdx_license + + try: + spdx_license = existing[l] + except KeyError: + spdx_license = "LicenseRef-" + l + add_extracted_license(spdx_license, l) + + return spdx_license + + lic_split = lic.replace("(", " ( ").replace(")", " ) ").split() + + return ' '.join(convert(l) for l in lic_split) + +def process_sources(d): + pn = d.getVar('PN') + assume_provided = (d.getVar("ASSUME_PROVIDED") or "").split() + if pn in assume_provided: + for p in d.getVar("PROVIDES").split(): + if p != pn: + pn = p + break + + # glibc-locale: do_fetch, do_unpack and do_patch tasks have been deleted, + # so avoid archiving source here. + if pn.startswith('glibc-locale'): + return False + if d.getVar('PN') == "libtool-cross": + return False + if d.getVar('PN') == "libgcc-initial": + return False + if d.getVar('PN') == "shadow-sysroot": + return False + + # We just archive gcc-source for all the gcc related recipes + if d.getVar('BPN') in ['gcc', 'libgcc']: + bb.debug(1, 'spdx: There is bug in scan of %s is, do nothing' % pn) + return False + + return True + + +def add_package_files(d, doc, spdx_pkg, topdir, get_spdxid, get_types, *, archive=None, ignore_dirs=[], ignore_top_level_dirs=[]): + from pathlib import Path + import oe.spdx + import hashlib + + source_date_epoch = d.getVar("SOURCE_DATE_EPOCH") + if source_date_epoch: + source_date_epoch = int(source_date_epoch) + + sha1s = [] + spdx_files = [] + + file_counter = 1 + for subdir, dirs, files in os.walk(topdir): + dirs[:] = [d for d in dirs if d not in ignore_dirs] + if subdir == str(topdir): + dirs[:] = [d for d in dirs if d not in ignore_top_level_dirs] + + for file in files: + filepath = Path(subdir) / file + filename = str(filepath.relative_to(topdir)) + + if not filepath.is_symlink() and filepath.is_file(): + spdx_file = oe.spdx.SPDXFile() + spdx_file.SPDXID = get_spdxid(file_counter) + for t in get_types(filepath): + spdx_file.fileTypes.append(t) + spdx_file.fileName = filename + + if archive is not None: + with filepath.open("rb") as f: + info = archive.gettarinfo(fileobj=f) + info.name = filename + info.uid = 0 + info.gid = 0 + info.uname = "root" + info.gname = "root" + + if source_date_epoch is not None and info.mtime > source_date_epoch: + info.mtime = source_date_epoch + + archive.addfile(info, f) + + sha1 = bb.utils.sha1_file(filepath) + sha1s.append(sha1) + spdx_file.checksums.append(oe.spdx.SPDXChecksum( + algorithm="SHA1", + checksumValue=sha1, + )) + spdx_file.checksums.append(oe.spdx.SPDXChecksum( + algorithm="SHA256", + checksumValue=bb.utils.sha256_file(filepath), + )) + + if "SOURCE" in spdx_file.fileTypes: + extracted_lics = extract_licenses(filepath) + if extracted_lics: + spdx_file.licenseInfoInFiles = extracted_lics + + doc.files.append(spdx_file) + doc.add_relationship(spdx_pkg, "CONTAINS", spdx_file) + spdx_pkg.hasFiles.append(spdx_file.SPDXID) + + spdx_files.append(spdx_file) + + file_counter += 1 + + sha1s.sort() + verifier = hashlib.sha1() + for v in sha1s: + verifier.update(v.encode("utf-8")) + spdx_pkg.packageVerificationCode.packageVerificationCodeValue = verifier.hexdigest() + + return spdx_files + + +def add_package_sources_from_debug(d, package_doc, spdx_package, package, package_files, sources): + from pathlib import Path + import hashlib + import oe.packagedata + import oe.spdx + + debug_search_paths = [ + Path(d.getVar('PKGD')), + Path(d.getVar('STAGING_DIR_TARGET')), + Path(d.getVar('STAGING_DIR_NATIVE')), + Path(d.getVar('STAGING_KERNEL_DIR')), + ] + + pkg_data = oe.packagedata.read_subpkgdata_extended(package, d) + + if pkg_data is None: + return + + for file_path, file_data in pkg_data["files_info"].items(): + if not "debugsrc" in file_data: + continue + + for pkg_file in package_files: + if file_path.lstrip("/") == pkg_file.fileName.lstrip("/"): + break + else: + bb.fatal("No package file found for %s" % str(file_path)) + continue + + for debugsrc in file_data["debugsrc"]: + ref_id = "NOASSERTION" + for search in debug_search_paths: + if debugsrc.startswith("/usr/src/kernel"): + debugsrc_path = search / debugsrc.replace('/usr/src/kernel/', '') + else: + debugsrc_path = search / debugsrc.lstrip("/") + if not debugsrc_path.exists(): + continue + + file_sha256 = bb.utils.sha256_file(debugsrc_path) + + if file_sha256 in sources: + source_file = sources[file_sha256] + + doc_ref = package_doc.find_external_document_ref(source_file.doc.documentNamespace) + if doc_ref is None: + doc_ref = oe.spdx.SPDXExternalDocumentRef() + doc_ref.externalDocumentId = "DocumentRef-dependency-" + source_file.doc.name + doc_ref.spdxDocument = source_file.doc.documentNamespace + doc_ref.checksum.algorithm = "SHA1" + doc_ref.checksum.checksumValue = source_file.doc_sha1 + package_doc.externalDocumentRefs.append(doc_ref) + + ref_id = "%s:%s" % (doc_ref.externalDocumentId, source_file.file.SPDXID) + else: + bb.debug(1, "Debug source %s with SHA256 %s not found in any dependency" % (str(debugsrc_path), file_sha256)) + break + else: + bb.debug(1, "Debug source %s not found" % debugsrc) + + package_doc.add_relationship(pkg_file, "GENERATED_FROM", ref_id, comment=debugsrc) + +def collect_dep_recipes(d, doc, spdx_recipe): + from pathlib import Path + import oe.sbom + import oe.spdx + + deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX")) + + dep_recipes = [] + taskdepdata = d.getVar("BB_TASKDEPDATA", False) + deps = sorted(set( + dep[0] for dep in taskdepdata.values() if + dep[1] == "do_create_spdx" and dep[0] != d.getVar("PN") + )) + for dep_pn in deps: + dep_recipe_path = deploy_dir_spdx / "recipes" / ("recipe-%s.spdx.json" % dep_pn) + + spdx_dep_doc, spdx_dep_sha1 = oe.sbom.read_doc(dep_recipe_path) + + for pkg in spdx_dep_doc.packages: + if pkg.name == dep_pn: + spdx_dep_recipe = pkg + break + else: + continue + + dep_recipes.append(oe.sbom.DepRecipe(spdx_dep_doc, spdx_dep_sha1, spdx_dep_recipe)) + + dep_recipe_ref = oe.spdx.SPDXExternalDocumentRef() + dep_recipe_ref.externalDocumentId = "DocumentRef-dependency-" + spdx_dep_doc.name + dep_recipe_ref.spdxDocument = spdx_dep_doc.documentNamespace + dep_recipe_ref.checksum.algorithm = "SHA1" + dep_recipe_ref.checksum.checksumValue = spdx_dep_sha1 + + doc.externalDocumentRefs.append(dep_recipe_ref) + + doc.add_relationship( + "%s:%s" % (dep_recipe_ref.externalDocumentId, spdx_dep_recipe.SPDXID), + "BUILD_DEPENDENCY_OF", + spdx_recipe + ) + + return dep_recipes + +collect_dep_recipes[vardepsexclude] += "BB_TASKDEPDATA" +collect_dep_recipes[vardeps] += "DEPENDS" + +def collect_dep_sources(d, dep_recipes): + import oe.sbom + + sources = {} + for dep in dep_recipes: + # Don't collect sources from native recipes as they + # match non-native sources also. + if recipe_spdx_is_native(d, dep.recipe): + continue + recipe_files = set(dep.recipe.hasFiles) + + for spdx_file in dep.doc.files: + if spdx_file.SPDXID not in recipe_files: + continue + + if "SOURCE" in spdx_file.fileTypes: + for checksum in spdx_file.checksums: + if checksum.algorithm == "SHA256": + sources[checksum.checksumValue] = oe.sbom.DepSource(dep.doc, dep.doc_sha1, dep.recipe, spdx_file) + break + + return sources + +def add_download_packages(d, doc, recipe): + import os.path + from bb.fetch2 import decodeurl, CHECKSUM_LIST + import bb.process + import oe.spdx + import oe.sbom + + for download_idx, src_uri in enumerate(d.getVar('SRC_URI').split()): + f = bb.fetch2.FetchData(src_uri, d) + + for name in f.names: + package = oe.spdx.SPDXPackage() + package.name = "%s-source-%d" % (d.getVar("PN"), download_idx + 1) + package.SPDXID = oe.sbom.get_download_spdxid(d, download_idx + 1) + + if f.type == "file": + continue + + uri = f.type + proto = getattr(f, "proto", None) + if proto is not None: + uri = uri + "+" + proto + uri = uri + "://" + f.host + f.path + + if f.method.supports_srcrev(): + uri = uri + "@" + f.revisions[name] + + if f.method.supports_checksum(f): + for checksum_id in CHECKSUM_LIST: + if checksum_id.upper() not in oe.spdx.SPDXPackage.ALLOWED_CHECKSUMS: + continue + + expected_checksum = getattr(f, "%s_expected" % checksum_id) + if expected_checksum is None: + continue + + c = oe.spdx.SPDXChecksum() + c.algorithm = checksum_id.upper() + c.checksumValue = expected_checksum + package.checksums.append(c) + + package.downloadLocation = uri + doc.packages.append(package) + doc.add_relationship(doc, "DESCRIBES", package) + # In the future, we might be able to do more fancy dependencies, + # but this should be sufficient for now + doc.add_relationship(package, "BUILD_DEPENDENCY_OF", recipe) + +python do_create_spdx() { + from datetime import datetime, timezone + import oe.sbom + import oe.spdx + import uuid + from pathlib import Path + from contextlib import contextmanager + import oe.cve_check + + @contextmanager + def optional_tarfile(name, guard, mode="w"): + import tarfile + import bb.compress.zstd + + num_threads = int(d.getVar("BB_NUMBER_THREADS")) + + if guard: + name.parent.mkdir(parents=True, exist_ok=True) + with bb.compress.zstd.open(name, mode=mode + "b", num_threads=num_threads) as f: + with tarfile.open(fileobj=f, mode=mode + "|") as tf: + yield tf + else: + yield None + + + deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX")) + spdx_workdir = Path(d.getVar("SPDXWORK")) + include_sources = d.getVar("SPDX_INCLUDE_SOURCES") == "1" + archive_sources = d.getVar("SPDX_ARCHIVE_SOURCES") == "1" + archive_packaged = d.getVar("SPDX_ARCHIVE_PACKAGED") == "1" + + creation_time = datetime.now(tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + doc = oe.spdx.SPDXDocument() + + doc.name = "recipe-" + d.getVar("PN") + doc.documentNamespace = get_doc_namespace(d, doc) + doc.creationInfo.created = creation_time + doc.creationInfo.comment = "This document was created by analyzing recipe files during the build." + doc.creationInfo.licenseListVersion = d.getVar("SPDX_LICENSE_DATA")["licenseListVersion"] + doc.creationInfo.creators.append("Tool: OpenEmbedded Core create-spdx.bbclass") + doc.creationInfo.creators.append("Organization: %s" % d.getVar("SPDX_ORG")) + doc.creationInfo.creators.append("Person: N/A ()") + + recipe = oe.spdx.SPDXPackage() + recipe.name = d.getVar("PN") + recipe.versionInfo = d.getVar("PV") + recipe.SPDXID = oe.sbom.get_recipe_spdxid(d) + recipe.supplier = d.getVar("SPDX_SUPPLIER") + if bb.data.inherits_class("native", d) or bb.data.inherits_class("cross", d): + recipe.annotations.append(create_annotation(d, "isNative")) + + homepage = d.getVar("HOMEPAGE") + if homepage: + recipe.homepage = homepage + + license = d.getVar("LICENSE") + if license: + recipe.licenseDeclared = convert_license_to_spdx(license, doc, d) + + summary = d.getVar("SUMMARY") + if summary: + recipe.summary = summary + + description = d.getVar("DESCRIPTION") + if description: + recipe.description = description + + if d.getVar("SPDX_CUSTOM_ANNOTATION_VARS"): + for var in d.getVar('SPDX_CUSTOM_ANNOTATION_VARS').split(): + recipe.annotations.append(create_annotation(d, var + "=" + d.getVar(var))) + + # Some CVEs may be patched during the build process without incrementing the version number, + # so querying for CVEs based on the CPE id can lead to false positives. To account for this, + # save the CVEs fixed by patches to source information field in the SPDX. + patched_cves = oe.cve_check.get_patched_cves(d) + patched_cves = list(patched_cves) + patched_cves = ' '.join(patched_cves) + if patched_cves: + recipe.sourceInfo = "CVEs fixed: " + patched_cves + + cpe_ids = oe.cve_check.get_cpe_ids(d.getVar("CVE_PRODUCT"), d.getVar("CVE_VERSION")) + if cpe_ids: + for cpe_id in cpe_ids: + cpe = oe.spdx.SPDXExternalReference() + cpe.referenceCategory = "SECURITY" + cpe.referenceType = "http://spdx.org/rdf/references/cpe23Type" + cpe.referenceLocator = cpe_id + recipe.externalRefs.append(cpe) + + doc.packages.append(recipe) + doc.add_relationship(doc, "DESCRIBES", recipe) + + add_download_packages(d, doc, recipe) + + if process_sources(d) and include_sources: + recipe_archive = deploy_dir_spdx / "recipes" / (doc.name + ".tar.zst") + with optional_tarfile(recipe_archive, archive_sources) as archive: + spdx_get_src(d) + + add_package_files( + d, + doc, + recipe, + spdx_workdir, + lambda file_counter: "SPDXRef-SourceFile-%s-%d" % (d.getVar("PN"), file_counter), + lambda filepath: ["SOURCE"], + ignore_dirs=[".git"], + ignore_top_level_dirs=["temp"], + archive=archive, + ) + + if archive is not None: + recipe.packageFileName = str(recipe_archive.name) + + dep_recipes = collect_dep_recipes(d, doc, recipe) + + doc_sha1 = oe.sbom.write_doc(d, doc, "recipes", indent=get_json_indent(d)) + dep_recipes.append(oe.sbom.DepRecipe(doc, doc_sha1, recipe)) + + recipe_ref = oe.spdx.SPDXExternalDocumentRef() + recipe_ref.externalDocumentId = "DocumentRef-recipe-" + recipe.name + recipe_ref.spdxDocument = doc.documentNamespace + recipe_ref.checksum.algorithm = "SHA1" + recipe_ref.checksum.checksumValue = doc_sha1 + + sources = collect_dep_sources(d, dep_recipes) + found_licenses = {license.name:recipe_ref.externalDocumentId + ":" + license.licenseId for license in doc.hasExtractedLicensingInfos} + + if not recipe_spdx_is_native(d, recipe): + bb.build.exec_func("read_subpackage_metadata", d) + + pkgdest = Path(d.getVar("PKGDEST")) + for package in d.getVar("PACKAGES").split(): + if not oe.packagedata.packaged(package, d): + continue + + package_doc = oe.spdx.SPDXDocument() + pkg_name = d.getVar("PKG:%s" % package) or package + package_doc.name = pkg_name + package_doc.documentNamespace = get_doc_namespace(d, package_doc) + package_doc.creationInfo.created = creation_time + package_doc.creationInfo.comment = "This document was created by analyzing packages created during the build." + package_doc.creationInfo.licenseListVersion = d.getVar("SPDX_LICENSE_DATA")["licenseListVersion"] + package_doc.creationInfo.creators.append("Tool: OpenEmbedded Core create-spdx.bbclass") + package_doc.creationInfo.creators.append("Organization: %s" % d.getVar("SPDX_ORG")) + package_doc.creationInfo.creators.append("Person: N/A ()") + package_doc.externalDocumentRefs.append(recipe_ref) + + package_license = d.getVar("LICENSE:%s" % package) or d.getVar("LICENSE") + + spdx_package = oe.spdx.SPDXPackage() + + spdx_package.SPDXID = oe.sbom.get_package_spdxid(pkg_name) + spdx_package.name = pkg_name + spdx_package.versionInfo = d.getVar("PV") + spdx_package.licenseDeclared = convert_license_to_spdx(package_license, package_doc, d, found_licenses) + spdx_package.supplier = d.getVar("SPDX_SUPPLIER") + + package_doc.packages.append(spdx_package) + + package_doc.add_relationship(spdx_package, "GENERATED_FROM", "%s:%s" % (recipe_ref.externalDocumentId, recipe.SPDXID)) + package_doc.add_relationship(package_doc, "DESCRIBES", spdx_package) + + package_archive = deploy_dir_spdx / "packages" / (package_doc.name + ".tar.zst") + with optional_tarfile(package_archive, archive_packaged) as archive: + package_files = add_package_files( + d, + package_doc, + spdx_package, + pkgdest / package, + lambda file_counter: oe.sbom.get_packaged_file_spdxid(pkg_name, file_counter), + lambda filepath: ["BINARY"], + ignore_top_level_dirs=['CONTROL', 'DEBIAN'], + archive=archive, + ) + + if archive is not None: + spdx_package.packageFileName = str(package_archive.name) + + add_package_sources_from_debug(d, package_doc, spdx_package, package, package_files, sources) + + oe.sbom.write_doc(d, package_doc, "packages", indent=get_json_indent(d)) +} +# NOTE: depending on do_unpack is a hack that is necessary to get it's dependencies for archive the source +addtask do_create_spdx after do_package do_packagedata do_unpack before do_populate_sdk do_build do_rm_work + +SSTATETASKS += "do_create_spdx" +do_create_spdx[sstate-inputdirs] = "${SPDXDEPLOY}" +do_create_spdx[sstate-outputdirs] = "${DEPLOY_DIR_SPDX}" + +python do_create_spdx_setscene () { + sstate_setscene(d) +} +addtask do_create_spdx_setscene + +do_create_spdx[dirs] = "${SPDXWORK}" +do_create_spdx[cleandirs] = "${SPDXDEPLOY} ${SPDXWORK}" +do_create_spdx[depends] += "${PATCHDEPENDENCY}" +do_create_spdx[deptask] = "do_create_spdx" + +def collect_package_providers(d): + from pathlib import Path + import oe.sbom + import oe.spdx + import json + + deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX")) + + providers = {} + + taskdepdata = d.getVar("BB_TASKDEPDATA", False) + deps = sorted(set( + dep[0] for dep in taskdepdata.values() if dep[0] != d.getVar("PN") + )) + deps.append(d.getVar("PN")) + + for dep_pn in deps: + recipe_data = oe.packagedata.read_pkgdata(dep_pn, d) + + for pkg in recipe_data.get("PACKAGES", "").split(): + + pkg_data = oe.packagedata.read_subpkgdata_dict(pkg, d) + rprovides = set(n for n, _ in bb.utils.explode_dep_versions2(pkg_data.get("RPROVIDES", "")).items()) + rprovides.add(pkg) + + for r in rprovides: + providers[r] = pkg + + return providers + +collect_package_providers[vardepsexclude] += "BB_TASKDEPDATA" + +python do_create_runtime_spdx() { + from datetime import datetime, timezone + import oe.sbom + import oe.spdx + import oe.packagedata + from pathlib import Path + + deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX")) + spdx_deploy = Path(d.getVar("SPDXRUNTIMEDEPLOY")) + is_native = bb.data.inherits_class("native", d) or bb.data.inherits_class("cross", d) + + creation_time = datetime.now(tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + providers = collect_package_providers(d) + + if not is_native: + bb.build.exec_func("read_subpackage_metadata", d) + + dep_package_cache = {} + + pkgdest = Path(d.getVar("PKGDEST")) + for package in d.getVar("PACKAGES").split(): + localdata = bb.data.createCopy(d) + pkg_name = d.getVar("PKG:%s" % package) or package + localdata.setVar("PKG", pkg_name) + localdata.setVar('OVERRIDES', d.getVar("OVERRIDES", False) + ":" + package) + + if not oe.packagedata.packaged(package, localdata): + continue + + pkg_spdx_path = deploy_dir_spdx / "packages" / (pkg_name + ".spdx.json") + + package_doc, package_doc_sha1 = oe.sbom.read_doc(pkg_spdx_path) + + for p in package_doc.packages: + if p.name == pkg_name: + spdx_package = p + break + else: + bb.fatal("Package '%s' not found in %s" % (pkg_name, pkg_spdx_path)) + + runtime_doc = oe.spdx.SPDXDocument() + runtime_doc.name = "runtime-" + pkg_name + runtime_doc.documentNamespace = get_doc_namespace(localdata, runtime_doc) + runtime_doc.creationInfo.created = creation_time + runtime_doc.creationInfo.comment = "This document was created by analyzing package runtime dependencies." + runtime_doc.creationInfo.licenseListVersion = d.getVar("SPDX_LICENSE_DATA")["licenseListVersion"] + runtime_doc.creationInfo.creators.append("Tool: OpenEmbedded Core create-spdx.bbclass") + runtime_doc.creationInfo.creators.append("Organization: %s" % d.getVar("SPDX_ORG")) + runtime_doc.creationInfo.creators.append("Person: N/A ()") + + package_ref = oe.spdx.SPDXExternalDocumentRef() + package_ref.externalDocumentId = "DocumentRef-package-" + package + package_ref.spdxDocument = package_doc.documentNamespace + package_ref.checksum.algorithm = "SHA1" + package_ref.checksum.checksumValue = package_doc_sha1 + + runtime_doc.externalDocumentRefs.append(package_ref) + + runtime_doc.add_relationship( + runtime_doc.SPDXID, + "AMENDS", + "%s:%s" % (package_ref.externalDocumentId, package_doc.SPDXID) + ) + + deps = bb.utils.explode_dep_versions2(localdata.getVar("RDEPENDS") or "") + seen_deps = set() + for dep, _ in deps.items(): + if dep in seen_deps: + continue + + if dep not in providers: + continue + + dep = providers[dep] + + if not oe.packagedata.packaged(dep, localdata): + continue + + dep_pkg_data = oe.packagedata.read_subpkgdata_dict(dep, d) + dep_pkg = dep_pkg_data["PKG"] + + if dep in dep_package_cache: + (dep_spdx_package, dep_package_ref) = dep_package_cache[dep] + else: + dep_path = deploy_dir_spdx / "packages" / ("%s.spdx.json" % dep_pkg) + + spdx_dep_doc, spdx_dep_sha1 = oe.sbom.read_doc(dep_path) + + for pkg in spdx_dep_doc.packages: + if pkg.name == dep_pkg: + dep_spdx_package = pkg + break + else: + bb.fatal("Package '%s' not found in %s" % (dep_pkg, dep_path)) + + dep_package_ref = oe.spdx.SPDXExternalDocumentRef() + dep_package_ref.externalDocumentId = "DocumentRef-runtime-dependency-" + spdx_dep_doc.name + dep_package_ref.spdxDocument = spdx_dep_doc.documentNamespace + dep_package_ref.checksum.algorithm = "SHA1" + dep_package_ref.checksum.checksumValue = spdx_dep_sha1 + + dep_package_cache[dep] = (dep_spdx_package, dep_package_ref) + + runtime_doc.externalDocumentRefs.append(dep_package_ref) + + runtime_doc.add_relationship( + "%s:%s" % (dep_package_ref.externalDocumentId, dep_spdx_package.SPDXID), + "RUNTIME_DEPENDENCY_OF", + "%s:%s" % (package_ref.externalDocumentId, spdx_package.SPDXID) + ) + seen_deps.add(dep) + + oe.sbom.write_doc(d, runtime_doc, "runtime", spdx_deploy, indent=get_json_indent(d)) +} + +addtask do_create_runtime_spdx after do_create_spdx before do_build do_rm_work +SSTATETASKS += "do_create_runtime_spdx" +do_create_runtime_spdx[sstate-inputdirs] = "${SPDXRUNTIMEDEPLOY}" +do_create_runtime_spdx[sstate-outputdirs] = "${DEPLOY_DIR_SPDX}" + +python do_create_runtime_spdx_setscene () { + sstate_setscene(d) +} +addtask do_create_runtime_spdx_setscene + +do_create_runtime_spdx[dirs] = "${SPDXRUNTIMEDEPLOY}" +do_create_runtime_spdx[cleandirs] = "${SPDXRUNTIMEDEPLOY}" +do_create_runtime_spdx[rdeptask] = "do_create_spdx" + +def spdx_get_src(d): + """ + save patched source of the recipe in SPDX_WORKDIR. + """ + import shutil + spdx_workdir = d.getVar('SPDXWORK') + spdx_sysroot_native = d.getVar('STAGING_DIR_NATIVE') + pn = d.getVar('PN') + + workdir = d.getVar("WORKDIR") + + try: + # The kernel class functions require it to be on work-shared, so we dont change WORKDIR + if not is_work_shared_spdx(d): + # Change the WORKDIR to make do_unpack do_patch run in another dir. + d.setVar('WORKDIR', spdx_workdir) + # Restore the original path to recipe's native sysroot (it's relative to WORKDIR). + d.setVar('STAGING_DIR_NATIVE', spdx_sysroot_native) + + # The changed 'WORKDIR' also caused 'B' changed, create dir 'B' for the + # possibly requiring of the following tasks (such as some recipes's + # do_patch required 'B' existed). + bb.utils.mkdirhier(d.getVar('B')) + + bb.build.exec_func('do_unpack', d) + # Copy source of kernel to spdx_workdir + if is_work_shared_spdx(d): + share_src = d.getVar('WORKDIR') + d.setVar('WORKDIR', spdx_workdir) + d.setVar('STAGING_DIR_NATIVE', spdx_sysroot_native) + src_dir = spdx_workdir + "/" + d.getVar('PN')+ "-" + d.getVar('PV') + "-" + d.getVar('PR') + bb.utils.mkdirhier(src_dir) + if bb.data.inherits_class('kernel',d): + share_src = d.getVar('STAGING_KERNEL_DIR') + cmd_copy_share = "cp -rf " + share_src + "/* " + src_dir + "/" + cmd_copy_shared_res = os.popen(cmd_copy_share).read() + bb.note("cmd_copy_shared_result = " + cmd_copy_shared_res) + + git_path = src_dir + "/.git" + if os.path.exists(git_path): + shutils.rmtree(git_path) + + # Make sure gcc and kernel sources are patched only once + if not (d.getVar('SRC_URI') == "" or is_work_shared_spdx(d)): + bb.build.exec_func('do_patch', d) + + # Some userland has no source. + if not os.path.exists( spdx_workdir ): + bb.utils.mkdirhier(spdx_workdir) + finally: + d.setVar("WORKDIR", workdir) + +do_rootfs[recrdeptask] += "do_create_spdx do_create_runtime_spdx" +do_rootfs[cleandirs] += "${SPDXIMAGEWORK}" + +ROOTFS_POSTUNINSTALL_COMMAND =+ "image_combine_spdx ; " + +do_populate_sdk[recrdeptask] += "do_create_spdx do_create_runtime_spdx" +do_populate_sdk[cleandirs] += "${SPDXSDKWORK}" +POPULATE_SDK_POST_HOST_COMMAND:append:task-populate-sdk = " sdk_host_combine_spdx; " +POPULATE_SDK_POST_TARGET_COMMAND:append:task-populate-sdk = " sdk_target_combine_spdx; " + +python image_combine_spdx() { + import os + import oe.sbom + from pathlib import Path + from oe.rootfs import image_list_installed_packages + + image_name = d.getVar("IMAGE_NAME") + image_link_name = d.getVar("IMAGE_LINK_NAME") + imgdeploydir = Path(d.getVar("IMGDEPLOYDIR")) + img_spdxid = oe.sbom.get_image_spdxid(image_name) + packages = image_list_installed_packages(d) + + combine_spdx(d, image_name, imgdeploydir, img_spdxid, packages, Path(d.getVar("SPDXIMAGEWORK"))) + + def make_image_link(target_path, suffix): + if image_link_name: + link = imgdeploydir / (image_link_name + suffix) + if link != target_path:
Quoted 8 lines
meta/classes/create-spdx-2.2.bbclass line 903
Quoted 4 lines
meta/classes/create-spdx-2.2.bbclass new file +4
-0+ link.symlink_to(os.path.relpath(target_path, link.parent)) + + spdx_tar_path = imgdeploydir / (image_name + ".spdx.tar.zst") + make_image_link(spdx_tar_path, ".spdx.tar.zst")The image link should be called tar.gz also. Probably best to make a gzip-from-the-start version for Dunfell,
or perhaps squash everything?
This is a bit of an unusual situation where we're taking a different path to master. I think this does make sense as it shows the delta with master more clearly and that could be useful in the future so I suspect that it does make sense as it is here, albeit a little unusual.
Cheers,
Richard
Re: [OE-core][dunfell][PATCH 1/5] classes/create-spdx: Backport
Right that makes sense, LGTM then!
Den tors 23 mars 2023 kl 21:53 skrev Richard Purdie <[email protected]>:
Quoted 948 lines · Ernst Sjöstrand
Quoted 927 lines
Den ons 22 mars 2023 kl 21:46 skrev Joshua Watt <[email protected]>:
Quoted 926 lines
Backports the create-spdx classes from the latest versions on master. This backport is a simple copy with no modifications, as its too complex to cherry-pick all the corresponding changes. This will give an appropriate base commit for subsequent changes and if necessary additional backport cherry-picks from master in the future.
Signed-off-by Joshua Watt <[email protected]>--- meta/classes/create-spdx-2.2.bbclass | 1069 +++++ meta/classes/create-spdx.bbclass | 8 + meta/files/spdx-licenses.json | 5937 ++++++++++++++++++++++++++ meta/lib/oe/sbom.py | 84 + meta/lib/oe/spdx.py | 357 ++ 5 files changed, 7455 insertions(+) create mode 100644 meta/classes/create-spdx-2.2.bbclass create mode 100644 meta/classes/create-spdx.bbclass create mode 100644 meta/files/spdx-licenses.json create mode 100644 meta/lib/oe/sbom.py create mode 100644 meta/lib/oe/spdx.pymeta/classes/create-spdx-2.2.bbclass new file +899
-0@@ -0,0 +1,1069 @@ +# +# Copyright OpenEmbedded Contributors +# +# SPDX-License-Identifier: GPL-2.0-only +# + +DEPLOY_DIR_SPDX ??= "${DEPLOY_DIR}/spdx/${MACHINE}" + +# The product name that the CVE database uses. Defaults to BPN, but may need to +# be overriden per recipe (for example tiff.bb sets CVE_PRODUCT=libtiff). +CVE_PRODUCT ??= "${BPN}" +CVE_VERSION ??= "${PV}" + +SPDXDIR ??= "${WORKDIR}/spdx" +SPDXDEPLOY = "${SPDXDIR}/deploy" +SPDXWORK = "${SPDXDIR}/work" +SPDXIMAGEWORK = "${SPDXDIR}/image-work" +SPDXSDKWORK = "${SPDXDIR}/sdk-work" + +SPDX_TOOL_NAME ??= "oe-spdx-creator" +SPDX_TOOL_VERSION ??= "1.0" + +SPDXRUNTIMEDEPLOY = "${SPDXDIR}/runtime-deploy" + +SPDX_INCLUDE_SOURCES ??= "0" +SPDX_ARCHIVE_SOURCES ??= "0" +SPDX_ARCHIVE_PACKAGED ??= "0" + +SPDX_UUID_NAMESPACE ??= "sbom.openembedded.org" +SPDX_NAMESPACE_PREFIX ??= "http://spdx.org/spdxdoc" +SPDX_PRETTY ??= "0" + +SPDX_LICENSES ??= "${COREBASE}/meta/files/spdx-licenses.json" + +SPDX_CUSTOM_ANNOTATION_VARS ??= "" + +SPDX_ORG ??= "OpenEmbedded ()" +SPDX_SUPPLIER ??= "Organization: ${SPDX_ORG}" +SPDX_SUPPLIER[doc] = "The SPDX PackageSupplier field for SPDX packages created from \ + this recipe. For SPDX documents create using this class during the build, this \ + is the contact information for the person or organization who is doing the \ + build." + +def extract_licenses(filename): + import re + + lic_regex = re.compile(rb'^\W*SPDX-License-Identifier:\s*([ \w\d.()+-]+?)(?:\s+\W*)?$', re.MULTILINE) + + try: + with open(filename, 'rb') as f: + size = min(15000, os.stat(filename).st_size) + txt = f.read(size) + licenses = re.findall(lic_regex, txt) + if licenses: + ascii_licenses = [lic.decode('ascii') for lic in licenses] + return ascii_licenses + except Exception as e: + bb.warn(f"Exception reading {filename}: {e}") + return None + +def get_doc_namespace(d, doc): + import uuid + namespace_uuid = uuid.uuid5(uuid.NAMESPACE_DNS, d.getVar("SPDX_UUID_NAMESPACE")) + return "%s/%s-%s" % (d.getVar("SPDX_NAMESPACE_PREFIX"), doc.name, str(uuid.uuid5(namespace_uuid, doc.name))) + +def create_annotation(d, comment): + from datetime import datetime, timezone + + creation_time = datetime.now(tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + annotation = oe.spdx.SPDXAnnotation() + annotation.annotationDate = creation_time + annotation.annotationType = "OTHER" + annotation.annotator = "Tool: %s - %s" % (d.getVar("SPDX_TOOL_NAME"), d.getVar("SPDX_TOOL_VERSION")) + annotation.comment = comment + return annotation + +def recipe_spdx_is_native(d, recipe): + return any(a.annotationType == "OTHER" and + a.annotator == "Tool: %s - %s" % (d.getVar("SPDX_TOOL_NAME"), d.getVar("SPDX_TOOL_VERSION")) and + a.comment == "isNative" for a in recipe.annotations) + +def is_work_shared_spdx(d): + return bb.data.inherits_class('kernel', d) or ('work-shared' in d.getVar('WORKDIR')) + +def get_json_indent(d): + if d.getVar("SPDX_PRETTY") == "1": + return 2 + return None + +python() { + import json + if d.getVar("SPDX_LICENSE_DATA"): + return + + with open(d.getVar("SPDX_LICENSES"), "r") as f: + data = json.load(f) + # Transform the license array to a dictionary + data["licenses"] = {l["licenseId"]: l for l in data["licenses"]} + d.setVar("SPDX_LICENSE_DATA", data) +} + +def convert_license_to_spdx(lic, document, d, existing={}): + from pathlib import Path + import oe.spdx + + license_data = d.getVar("SPDX_LICENSE_DATA") + extracted = {} + + def add_extracted_license(ident, name): + nonlocal document + + if name in extracted: + return + + extracted_info = oe.spdx.SPDXExtractedLicensingInfo() + extracted_info.name = name + extracted_info.licenseId = ident + extracted_info.extractedText = None + + if name == "PD": + # Special-case this. + extracted_info.extractedText = "Software released to the public domain" + else: + # Seach for the license in COMMON_LICENSE_DIR and LICENSE_PATH + for directory in [d.getVar('COMMON_LICENSE_DIR')] + (d.getVar('LICENSE_PATH') or '').split(): + try: + with (Path(directory) / name).open(errors="replace") as f: + extracted_info.extractedText = f.read() + break + except FileNotFoundError: + pass + if extracted_info.extractedText is None: + # If it's not SPDX or PD, then NO_GENERIC_LICENSE must be set + filename = d.getVarFlag('NO_GENERIC_LICENSE', name) + if filename: + filename = d.expand("${S}/" + filename) + with open(filename, errors="replace") as f: + extracted_info.extractedText = f.read() + else: + bb.error("Cannot find any text for license %s" % name) + + extracted[name] = extracted_info + document.hasExtractedLicensingInfos.append(extracted_info) + + def convert(l): + if l == "(" or l == ")": + return l + + if l == "&": + return "AND" + + if l == "|": + return "OR" + + if l == "CLOSED": + return "NONE" + + spdx_license = d.getVarFlag("SPDXLICENSEMAP", l) or l + if spdx_license in license_data["licenses"]: + return spdx_license + + try: + spdx_license = existing[l] + except KeyError: + spdx_license = "LicenseRef-" + l + add_extracted_license(spdx_license, l) + + return spdx_license + + lic_split = lic.replace("(", " ( ").replace(")", " ) ").split() + + return ' '.join(convert(l) for l in lic_split) + +def process_sources(d): + pn = d.getVar('PN') + assume_provided = (d.getVar("ASSUME_PROVIDED") or "").split() + if pn in assume_provided: + for p in d.getVar("PROVIDES").split(): + if p != pn: + pn = p + break + + # glibc-locale: do_fetch, do_unpack and do_patch tasks have been deleted, + # so avoid archiving source here. + if pn.startswith('glibc-locale'): + return False + if d.getVar('PN') == "libtool-cross": + return False + if d.getVar('PN') == "libgcc-initial": + return False + if d.getVar('PN') == "shadow-sysroot": + return False + + # We just archive gcc-source for all the gcc related recipes + if d.getVar('BPN') in ['gcc', 'libgcc']: + bb.debug(1, 'spdx: There is bug in scan of %s is, do nothing' % pn) + return False + + return True + + +def add_package_files(d, doc, spdx_pkg, topdir, get_spdxid, get_types, *, archive=None, ignore_dirs=[], ignore_top_level_dirs=[]): + from pathlib import Path + import oe.spdx + import hashlib + + source_date_epoch = d.getVar("SOURCE_DATE_EPOCH") + if source_date_epoch: + source_date_epoch = int(source_date_epoch) + + sha1s = [] + spdx_files = [] + + file_counter = 1 + for subdir, dirs, files in os.walk(topdir): + dirs[:] = [d for d in dirs if d not in ignore_dirs] + if subdir == str(topdir): + dirs[:] = [d for d in dirs if d not in ignore_top_level_dirs] + + for file in files: + filepath = Path(subdir) / file + filename = str(filepath.relative_to(topdir)) + + if not filepath.is_symlink() and filepath.is_file(): + spdx_file = oe.spdx.SPDXFile() + spdx_file.SPDXID = get_spdxid(file_counter) + for t in get_types(filepath): + spdx_file.fileTypes.append(t) + spdx_file.fileName = filename + + if archive is not None: + with filepath.open("rb") as f: + info = archive.gettarinfo(fileobj=f) + info.name = filename + info.uid = 0 + info.gid = 0 + info.uname = "root" + info.gname = "root" + + if source_date_epoch is not None and info.mtime > source_date_epoch: + info.mtime = source_date_epoch + + archive.addfile(info, f) + + sha1 = bb.utils.sha1_file(filepath) + sha1s.append(sha1) + spdx_file.checksums.append(oe.spdx.SPDXChecksum( + algorithm="SHA1", + checksumValue=sha1, + )) + spdx_file.checksums.append(oe.spdx.SPDXChecksum( + algorithm="SHA256", + checksumValue=bb.utils.sha256_file(filepath), + )) + + if "SOURCE" in spdx_file.fileTypes: + extracted_lics = extract_licenses(filepath) + if extracted_lics: + spdx_file.licenseInfoInFiles = extracted_lics + + doc.files.append(spdx_file) + doc.add_relationship(spdx_pkg, "CONTAINS", spdx_file) + spdx_pkg.hasFiles.append(spdx_file.SPDXID) + + spdx_files.append(spdx_file) + + file_counter += 1 + + sha1s.sort() + verifier = hashlib.sha1() + for v in sha1s: + verifier.update(v.encode("utf-8")) + spdx_pkg.packageVerificationCode.packageVerificationCodeValue = verifier.hexdigest() + + return spdx_files + + +def add_package_sources_from_debug(d, package_doc, spdx_package, package, package_files, sources): + from pathlib import Path + import hashlib + import oe.packagedata + import oe.spdx + + debug_search_paths = [ + Path(d.getVar('PKGD')), + Path(d.getVar('STAGING_DIR_TARGET')), + Path(d.getVar('STAGING_DIR_NATIVE')), + Path(d.getVar('STAGING_KERNEL_DIR')), + ] + + pkg_data = oe.packagedata.read_subpkgdata_extended(package, d) + + if pkg_data is None: + return + + for file_path, file_data in pkg_data["files_info"].items(): + if not "debugsrc" in file_data: + continue + + for pkg_file in package_files: + if file_path.lstrip("/") == pkg_file.fileName.lstrip("/"): + break + else: + bb.fatal("No package file found for %s" % str(file_path)) + continue + + for debugsrc in file_data["debugsrc"]: + ref_id = "NOASSERTION" + for search in debug_search_paths: + if debugsrc.startswith("/usr/src/kernel"): + debugsrc_path = search / debugsrc.replace('/usr/src/kernel/', '') + else: + debugsrc_path = search / debugsrc.lstrip("/") + if not debugsrc_path.exists(): + continue + + file_sha256 = bb.utils.sha256_file(debugsrc_path) + + if file_sha256 in sources: + source_file = sources[file_sha256] + + doc_ref = package_doc.find_external_document_ref(source_file.doc.documentNamespace) + if doc_ref is None: + doc_ref = oe.spdx.SPDXExternalDocumentRef() + doc_ref.externalDocumentId = "DocumentRef-dependency-" + source_file.doc.name + doc_ref.spdxDocument = source_file.doc.documentNamespace + doc_ref.checksum.algorithm = "SHA1" + doc_ref.checksum.checksumValue = source_file.doc_sha1 + package_doc.externalDocumentRefs.append(doc_ref) + + ref_id = "%s:%s" % (doc_ref.externalDocumentId, source_file.file.SPDXID) + else: + bb.debug(1, "Debug source %s with SHA256 %s not found in any dependency" % (str(debugsrc_path), file_sha256)) + break + else: + bb.debug(1, "Debug source %s not found" % debugsrc) + + package_doc.add_relationship(pkg_file, "GENERATED_FROM", ref_id, comment=debugsrc) + +def collect_dep_recipes(d, doc, spdx_recipe): + from pathlib import Path + import oe.sbom + import oe.spdx + + deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX")) + + dep_recipes = [] + taskdepdata = d.getVar("BB_TASKDEPDATA", False) + deps = sorted(set( + dep[0] for dep in taskdepdata.values() if + dep[1] == "do_create_spdx" and dep[0] != d.getVar("PN") + )) + for dep_pn in deps: + dep_recipe_path = deploy_dir_spdx / "recipes" / ("recipe-%s.spdx.json" % dep_pn) + + spdx_dep_doc, spdx_dep_sha1 = oe.sbom.read_doc(dep_recipe_path) + + for pkg in spdx_dep_doc.packages: + if pkg.name == dep_pn: + spdx_dep_recipe = pkg + break + else: + continue + + dep_recipes.append(oe.sbom.DepRecipe(spdx_dep_doc, spdx_dep_sha1, spdx_dep_recipe)) + + dep_recipe_ref = oe.spdx.SPDXExternalDocumentRef() + dep_recipe_ref.externalDocumentId = "DocumentRef-dependency-" + spdx_dep_doc.name + dep_recipe_ref.spdxDocument = spdx_dep_doc.documentNamespace + dep_recipe_ref.checksum.algorithm = "SHA1" + dep_recipe_ref.checksum.checksumValue = spdx_dep_sha1 + + doc.externalDocumentRefs.append(dep_recipe_ref) + + doc.add_relationship( + "%s:%s" % (dep_recipe_ref.externalDocumentId, spdx_dep_recipe.SPDXID), + "BUILD_DEPENDENCY_OF", + spdx_recipe + ) + + return dep_recipes + +collect_dep_recipes[vardepsexclude] += "BB_TASKDEPDATA" +collect_dep_recipes[vardeps] += "DEPENDS" + +def collect_dep_sources(d, dep_recipes): + import oe.sbom + + sources = {} + for dep in dep_recipes: + # Don't collect sources from native recipes as they + # match non-native sources also. + if recipe_spdx_is_native(d, dep.recipe): + continue + recipe_files = set(dep.recipe.hasFiles) + + for spdx_file in dep.doc.files: + if spdx_file.SPDXID not in recipe_files: + continue + + if "SOURCE" in spdx_file.fileTypes: + for checksum in spdx_file.checksums: + if checksum.algorithm == "SHA256": + sources[checksum.checksumValue] = oe.sbom.DepSource(dep.doc, dep.doc_sha1, dep.recipe, spdx_file) + break + + return sources + +def add_download_packages(d, doc, recipe): + import os.path + from bb.fetch2 import decodeurl, CHECKSUM_LIST + import bb.process + import oe.spdx + import oe.sbom + + for download_idx, src_uri in enumerate(d.getVar('SRC_URI').split()): + f = bb.fetch2.FetchData(src_uri, d) + + for name in f.names: + package = oe.spdx.SPDXPackage() + package.name = "%s-source-%d" % (d.getVar("PN"), download_idx + 1) + package.SPDXID = oe.sbom.get_download_spdxid(d, download_idx + 1) + + if f.type == "file": + continue + + uri = f.type + proto = getattr(f, "proto", None) + if proto is not None: + uri = uri + "+" + proto + uri = uri + "://" + f.host + f.path + + if f.method.supports_srcrev(): + uri = uri + "@" + f.revisions[name] + + if f.method.supports_checksum(f): + for checksum_id in CHECKSUM_LIST: + if checksum_id.upper() not in oe.spdx.SPDXPackage.ALLOWED_CHECKSUMS: + continue + + expected_checksum = getattr(f, "%s_expected" % checksum_id) + if expected_checksum is None: + continue + + c = oe.spdx.SPDXChecksum() + c.algorithm = checksum_id.upper() + c.checksumValue = expected_checksum + package.checksums.append(c) + + package.downloadLocation = uri + doc.packages.append(package) + doc.add_relationship(doc, "DESCRIBES", package) + # In the future, we might be able to do more fancy dependencies, + # but this should be sufficient for now + doc.add_relationship(package, "BUILD_DEPENDENCY_OF", recipe) + +python do_create_spdx() { + from datetime import datetime, timezone + import oe.sbom + import oe.spdx + import uuid + from pathlib import Path + from contextlib import contextmanager + import oe.cve_check + + @contextmanager + def optional_tarfile(name, guard, mode="w"): + import tarfile + import bb.compress.zstd + + num_threads = int(d.getVar("BB_NUMBER_THREADS")) + + if guard: + name.parent.mkdir(parents=True, exist_ok=True) + with bb.compress.zstd.open(name, mode=mode + "b", num_threads=num_threads) as f: + with tarfile.open(fileobj=f, mode=mode + "|") as tf: + yield tf + else: + yield None + + + deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX")) + spdx_workdir = Path(d.getVar("SPDXWORK")) + include_sources = d.getVar("SPDX_INCLUDE_SOURCES") == "1" + archive_sources = d.getVar("SPDX_ARCHIVE_SOURCES") == "1" + archive_packaged = d.getVar("SPDX_ARCHIVE_PACKAGED") == "1" + + creation_time = datetime.now(tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + doc = oe.spdx.SPDXDocument() + + doc.name = "recipe-" + d.getVar("PN") + doc.documentNamespace = get_doc_namespace(d, doc) + doc.creationInfo.created = creation_time + doc.creationInfo.comment = "This document was created by analyzing recipe files during the build." + doc.creationInfo.licenseListVersion = d.getVar("SPDX_LICENSE_DATA")["licenseListVersion"] + doc.creationInfo.creators.append("Tool: OpenEmbedded Core create-spdx.bbclass") + doc.creationInfo.creators.append("Organization: %s" % d.getVar("SPDX_ORG")) + doc.creationInfo.creators.append("Person: N/A ()") + + recipe = oe.spdx.SPDXPackage() + recipe.name = d.getVar("PN") + recipe.versionInfo = d.getVar("PV") + recipe.SPDXID = oe.sbom.get_recipe_spdxid(d) + recipe.supplier = d.getVar("SPDX_SUPPLIER") + if bb.data.inherits_class("native", d) or bb.data.inherits_class("cross", d): + recipe.annotations.append(create_annotation(d, "isNative")) + + homepage = d.getVar("HOMEPAGE") + if homepage: + recipe.homepage = homepage + + license = d.getVar("LICENSE") + if license: + recipe.licenseDeclared = convert_license_to_spdx(license, doc, d) + + summary = d.getVar("SUMMARY") + if summary: + recipe.summary = summary + + description = d.getVar("DESCRIPTION") + if description: + recipe.description = description + + if d.getVar("SPDX_CUSTOM_ANNOTATION_VARS"): + for var in d.getVar('SPDX_CUSTOM_ANNOTATION_VARS').split(): + recipe.annotations.append(create_annotation(d, var + "=" + d.getVar(var))) + + # Some CVEs may be patched during the build process without incrementing the version number, + # so querying for CVEs based on the CPE id can lead to false positives. To account for this, + # save the CVEs fixed by patches to source information field in the SPDX. + patched_cves = oe.cve_check.get_patched_cves(d) + patched_cves = list(patched_cves) + patched_cves = ' '.join(patched_cves) + if patched_cves: + recipe.sourceInfo = "CVEs fixed: " + patched_cves + + cpe_ids = oe.cve_check.get_cpe_ids(d.getVar("CVE_PRODUCT"), d.getVar("CVE_VERSION")) + if cpe_ids: + for cpe_id in cpe_ids: + cpe = oe.spdx.SPDXExternalReference() + cpe.referenceCategory = "SECURITY" + cpe.referenceType = "http://spdx.org/rdf/references/cpe23Type" + cpe.referenceLocator = cpe_id + recipe.externalRefs.append(cpe) + + doc.packages.append(recipe) + doc.add_relationship(doc, "DESCRIBES", recipe) + + add_download_packages(d, doc, recipe) + + if process_sources(d) and include_sources: + recipe_archive = deploy_dir_spdx / "recipes" / (doc.name + ".tar.zst") + with optional_tarfile(recipe_archive, archive_sources) as archive: + spdx_get_src(d) + + add_package_files( + d, + doc, + recipe, + spdx_workdir, + lambda file_counter: "SPDXRef-SourceFile-%s-%d" % (d.getVar("PN"), file_counter), + lambda filepath: ["SOURCE"], + ignore_dirs=[".git"], + ignore_top_level_dirs=["temp"], + archive=archive, + ) + + if archive is not None: + recipe.packageFileName = str(recipe_archive.name) + + dep_recipes = collect_dep_recipes(d, doc, recipe) + + doc_sha1 = oe.sbom.write_doc(d, doc, "recipes", indent=get_json_indent(d)) + dep_recipes.append(oe.sbom.DepRecipe(doc, doc_sha1, recipe)) + + recipe_ref = oe.spdx.SPDXExternalDocumentRef() + recipe_ref.externalDocumentId = "DocumentRef-recipe-" + recipe.name + recipe_ref.spdxDocument = doc.documentNamespace + recipe_ref.checksum.algorithm = "SHA1" + recipe_ref.checksum.checksumValue = doc_sha1 + + sources = collect_dep_sources(d, dep_recipes) + found_licenses = {license.name:recipe_ref.externalDocumentId + ":" + license.licenseId for license in doc.hasExtractedLicensingInfos} + + if not recipe_spdx_is_native(d, recipe): + bb.build.exec_func("read_subpackage_metadata", d) + + pkgdest = Path(d.getVar("PKGDEST")) + for package in d.getVar("PACKAGES").split(): + if not oe.packagedata.packaged(package, d): + continue + + package_doc = oe.spdx.SPDXDocument() + pkg_name = d.getVar("PKG:%s" % package) or package + package_doc.name = pkg_name + package_doc.documentNamespace = get_doc_namespace(d, package_doc) + package_doc.creationInfo.created = creation_time + package_doc.creationInfo.comment = "This document was created by analyzing packages created during the build." + package_doc.creationInfo.licenseListVersion = d.getVar("SPDX_LICENSE_DATA")["licenseListVersion"] + package_doc.creationInfo.creators.append("Tool: OpenEmbedded Core create-spdx.bbclass") + package_doc.creationInfo.creators.append("Organization: %s" % d.getVar("SPDX_ORG")) + package_doc.creationInfo.creators.append("Person: N/A ()") + package_doc.externalDocumentRefs.append(recipe_ref) + + package_license = d.getVar("LICENSE:%s" % package) or d.getVar("LICENSE") + + spdx_package = oe.spdx.SPDXPackage() + + spdx_package.SPDXID = oe.sbom.get_package_spdxid(pkg_name) + spdx_package.name = pkg_name + spdx_package.versionInfo = d.getVar("PV") + spdx_package.licenseDeclared = convert_license_to_spdx(package_license, package_doc, d, found_licenses) + spdx_package.supplier = d.getVar("SPDX_SUPPLIER") + + package_doc.packages.append(spdx_package) + + package_doc.add_relationship(spdx_package, "GENERATED_FROM", "%s:%s" % (recipe_ref.externalDocumentId, recipe.SPDXID)) + package_doc.add_relationship(package_doc, "DESCRIBES", spdx_package) + + package_archive = deploy_dir_spdx / "packages" / (package_doc.name + ".tar.zst") + with optional_tarfile(package_archive, archive_packaged) as archive: + package_files = add_package_files( + d, + package_doc, + spdx_package, + pkgdest / package, + lambda file_counter: oe.sbom.get_packaged_file_spdxid(pkg_name, file_counter), + lambda filepath: ["BINARY"], + ignore_top_level_dirs=['CONTROL', 'DEBIAN'], + archive=archive, + ) + + if archive is not None: + spdx_package.packageFileName = str(package_archive.name) + + add_package_sources_from_debug(d, package_doc, spdx_package, package, package_files, sources) + + oe.sbom.write_doc(d, package_doc, "packages", indent=get_json_indent(d)) +} +# NOTE: depending on do_unpack is a hack that is necessary to get it's dependencies for archive the source +addtask do_create_spdx after do_package do_packagedata do_unpack before do_populate_sdk do_build do_rm_work + +SSTATETASKS += "do_create_spdx" +do_create_spdx[sstate-inputdirs] = "${SPDXDEPLOY}" +do_create_spdx[sstate-outputdirs] = "${DEPLOY_DIR_SPDX}" + +python do_create_spdx_setscene () { + sstate_setscene(d) +} +addtask do_create_spdx_setscene + +do_create_spdx[dirs] = "${SPDXWORK}" +do_create_spdx[cleandirs] = "${SPDXDEPLOY} ${SPDXWORK}" +do_create_spdx[depends] += "${PATCHDEPENDENCY}" +do_create_spdx[deptask] = "do_create_spdx" + +def collect_package_providers(d): + from pathlib import Path + import oe.sbom + import oe.spdx + import json + + deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX")) + + providers = {} + + taskdepdata = d.getVar("BB_TASKDEPDATA", False) + deps = sorted(set( + dep[0] for dep in taskdepdata.values() if dep[0] != d.getVar("PN") + )) + deps.append(d.getVar("PN")) + + for dep_pn in deps: + recipe_data = oe.packagedata.read_pkgdata(dep_pn, d) + + for pkg in recipe_data.get("PACKAGES", "").split(): + + pkg_data = oe.packagedata.read_subpkgdata_dict(pkg, d) + rprovides = set(n for n, _ in bb.utils.explode_dep_versions2(pkg_data.get("RPROVIDES", "")).items()) + rprovides.add(pkg) + + for r in rprovides: + providers[r] = pkg + + return providers + +collect_package_providers[vardepsexclude] += "BB_TASKDEPDATA" + +python do_create_runtime_spdx() { + from datetime import datetime, timezone + import oe.sbom + import oe.spdx + import oe.packagedata + from pathlib import Path + + deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX")) + spdx_deploy = Path(d.getVar("SPDXRUNTIMEDEPLOY")) + is_native = bb.data.inherits_class("native", d) or bb.data.inherits_class("cross", d) + + creation_time = datetime.now(tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + providers = collect_package_providers(d) + + if not is_native: + bb.build.exec_func("read_subpackage_metadata", d) + + dep_package_cache = {} + + pkgdest = Path(d.getVar("PKGDEST")) + for package in d.getVar("PACKAGES").split(): + localdata = bb.data.createCopy(d) + pkg_name = d.getVar("PKG:%s" % package) or package + localdata.setVar("PKG", pkg_name) + localdata.setVar('OVERRIDES', d.getVar("OVERRIDES", False) + ":" + package) + + if not oe.packagedata.packaged(package, localdata): + continue + + pkg_spdx_path = deploy_dir_spdx / "packages" / (pkg_name + ".spdx.json") + + package_doc, package_doc_sha1 = oe.sbom.read_doc(pkg_spdx_path) + + for p in package_doc.packages: + if p.name == pkg_name: + spdx_package = p + break + else: + bb.fatal("Package '%s' not found in %s" % (pkg_name, pkg_spdx_path)) + + runtime_doc = oe.spdx.SPDXDocument() + runtime_doc.name = "runtime-" + pkg_name + runtime_doc.documentNamespace = get_doc_namespace(localdata, runtime_doc) + runtime_doc.creationInfo.created = creation_time + runtime_doc.creationInfo.comment = "This document was created by analyzing package runtime dependencies." + runtime_doc.creationInfo.licenseListVersion = d.getVar("SPDX_LICENSE_DATA")["licenseListVersion"] + runtime_doc.creationInfo.creators.append("Tool: OpenEmbedded Core create-spdx.bbclass") + runtime_doc.creationInfo.creators.append("Organization: %s" % d.getVar("SPDX_ORG")) + runtime_doc.creationInfo.creators.append("Person: N/A ()") + + package_ref = oe.spdx.SPDXExternalDocumentRef() + package_ref.externalDocumentId = "DocumentRef-package-" + package + package_ref.spdxDocument = package_doc.documentNamespace + package_ref.checksum.algorithm = "SHA1" + package_ref.checksum.checksumValue = package_doc_sha1 + + runtime_doc.externalDocumentRefs.append(package_ref) + + runtime_doc.add_relationship( + runtime_doc.SPDXID, + "AMENDS", + "%s:%s" % (package_ref.externalDocumentId, package_doc.SPDXID) + ) + + deps = bb.utils.explode_dep_versions2(localdata.getVar("RDEPENDS") or "") + seen_deps = set() + for dep, _ in deps.items(): + if dep in seen_deps: + continue + + if dep not in providers: + continue + + dep = providers[dep] + + if not oe.packagedata.packaged(dep, localdata): + continue + + dep_pkg_data = oe.packagedata.read_subpkgdata_dict(dep, d) + dep_pkg = dep_pkg_data["PKG"] + + if dep in dep_package_cache: + (dep_spdx_package, dep_package_ref) = dep_package_cache[dep] + else: + dep_path = deploy_dir_spdx / "packages" / ("%s.spdx.json" % dep_pkg) + + spdx_dep_doc, spdx_dep_sha1 = oe.sbom.read_doc(dep_path) + + for pkg in spdx_dep_doc.packages: + if pkg.name == dep_pkg: + dep_spdx_package = pkg + break + else: + bb.fatal("Package '%s' not found in %s" % (dep_pkg, dep_path)) + + dep_package_ref = oe.spdx.SPDXExternalDocumentRef() + dep_package_ref.externalDocumentId = "DocumentRef-runtime-dependency-" + spdx_dep_doc.name + dep_package_ref.spdxDocument = spdx_dep_doc.documentNamespace + dep_package_ref.checksum.algorithm = "SHA1" + dep_package_ref.checksum.checksumValue = spdx_dep_sha1 + + dep_package_cache[dep] = (dep_spdx_package, dep_package_ref) + + runtime_doc.externalDocumentRefs.append(dep_package_ref) + + runtime_doc.add_relationship( + "%s:%s" % (dep_package_ref.externalDocumentId, dep_spdx_package.SPDXID), + "RUNTIME_DEPENDENCY_OF", + "%s:%s" % (package_ref.externalDocumentId, spdx_package.SPDXID) + ) + seen_deps.add(dep) + + oe.sbom.write_doc(d, runtime_doc, "runtime", spdx_deploy, indent=get_json_indent(d)) +} + +addtask do_create_runtime_spdx after do_create_spdx before do_build do_rm_work +SSTATETASKS += "do_create_runtime_spdx" +do_create_runtime_spdx[sstate-inputdirs] = "${SPDXRUNTIMEDEPLOY}" +do_create_runtime_spdx[sstate-outputdirs] = "${DEPLOY_DIR_SPDX}" + +python do_create_runtime_spdx_setscene () { + sstate_setscene(d) +} +addtask do_create_runtime_spdx_setscene + +do_create_runtime_spdx[dirs] = "${SPDXRUNTIMEDEPLOY}" +do_create_runtime_spdx[cleandirs] = "${SPDXRUNTIMEDEPLOY}" +do_create_runtime_spdx[rdeptask] = "do_create_spdx" + +def spdx_get_src(d): + """ + save patched source of the recipe in SPDX_WORKDIR. + """ + import shutil + spdx_workdir = d.getVar('SPDXWORK') + spdx_sysroot_native = d.getVar('STAGING_DIR_NATIVE') + pn = d.getVar('PN') + + workdir = d.getVar("WORKDIR") + + try: + # The kernel class functions require it to be on work-shared, so we dont change WORKDIR + if not is_work_shared_spdx(d): + # Change the WORKDIR to make do_unpack do_patch run in another dir. + d.setVar('WORKDIR', spdx_workdir) + # Restore the original path to recipe's native sysroot (it's relative to WORKDIR). + d.setVar('STAGING_DIR_NATIVE', spdx_sysroot_native) + + # The changed 'WORKDIR' also caused 'B' changed, create dir 'B' for the + # possibly requiring of the following tasks (such as some recipes's + # do_patch required 'B' existed). + bb.utils.mkdirhier(d.getVar('B')) + + bb.build.exec_func('do_unpack', d) + # Copy source of kernel to spdx_workdir + if is_work_shared_spdx(d): + share_src = d.getVar('WORKDIR') + d.setVar('WORKDIR', spdx_workdir) + d.setVar('STAGING_DIR_NATIVE', spdx_sysroot_native) + src_dir = spdx_workdir + "/" + d.getVar('PN')+ "-" + d.getVar('PV') + "-" + d.getVar('PR') + bb.utils.mkdirhier(src_dir) + if bb.data.inherits_class('kernel',d): + share_src = d.getVar('STAGING_KERNEL_DIR') + cmd_copy_share = "cp -rf " + share_src + "/* " + src_dir + "/" + cmd_copy_shared_res = os.popen(cmd_copy_share).read() + bb.note("cmd_copy_shared_result = " + cmd_copy_shared_res) + + git_path = src_dir + "/.git" + if os.path.exists(git_path): + shutils.rmtree(git_path) + + # Make sure gcc and kernel sources are patched only once + if not (d.getVar('SRC_URI') == "" or is_work_shared_spdx(d)): + bb.build.exec_func('do_patch', d) + + # Some userland has no source. + if not os.path.exists( spdx_workdir ): + bb.utils.mkdirhier(spdx_workdir) + finally: + d.setVar("WORKDIR", workdir) + +do_rootfs[recrdeptask] += "do_create_spdx do_create_runtime_spdx" +do_rootfs[cleandirs] += "${SPDXIMAGEWORK}" + +ROOTFS_POSTUNINSTALL_COMMAND =+ "image_combine_spdx ; " + +do_populate_sdk[recrdeptask] += "do_create_spdx do_create_runtime_spdx" +do_populate_sdk[cleandirs] += "${SPDXSDKWORK}" +POPULATE_SDK_POST_HOST_COMMAND:append:task-populate-sdk = " sdk_host_combine_spdx; " +POPULATE_SDK_POST_TARGET_COMMAND:append:task-populate-sdk = " sdk_target_combine_spdx; " + +python image_combine_spdx() { + import os + import oe.sbom + from pathlib import Path + from oe.rootfs import image_list_installed_packages + + image_name = d.getVar("IMAGE_NAME") + image_link_name = d.getVar("IMAGE_LINK_NAME") + imgdeploydir = Path(d.getVar("IMGDEPLOYDIR")) + img_spdxid = oe.sbom.get_image_spdxid(image_name) + packages = image_list_installed_packages(d) + + combine_spdx(d, image_name, imgdeploydir, img_spdxid, packages, Path(d.getVar("SPDXIMAGEWORK"))) + + def make_image_link(target_path, suffix): + if image_link_name: + link = imgdeploydir / (image_link_name + suffix) + if link != target_path:Quoted 8 lines
meta/classes/create-spdx-2.2.bbclass line 903
Quoted 4 lines
meta/classes/create-spdx-2.2.bbclass new file +4
-0+ link.symlink_to(os.path.relpath(target_path, link.parent)) + + spdx_tar_path = imgdeploydir / (image_name + ".spdx.tar.zst") + make_image_link(spdx_tar_path, ".spdx.tar.zst")The image link should be called tar.gz also. Probably best to make a gzip-from-the-start version for Dunfell,
or perhaps squash everything?This is a bit of an unusual situation where we're taking a different path to master. I think this does make sense as it shows the delta with master more clearly and that could be useful in the future so I suspect that it does make sense as it is here, albeit a little unusual.
Cheers,
Richard
PATCH 2/5 classes/package: Add extended packaged data
In reply to Joshua WattAdds extended package data which is encoded as JSON which allows it to encode more structure than the "flat" package data files. The extended data might be much larger than the standard package data, so it is not read by default and instead requires oe.packagedata.read_subpkgdata_extended() to be called
Currently, the file sizes and ELF debug sources are saved off into the extended package data
(cherry picked from commit db9cf430e54ae68da80fbc3fba80ce88d8df164d)
--- meta/classes/package.bbclass | 40 +++++++++++++++++++++++++++++++++--- meta/lib/oe/packagedata.py | 12 +++++++++++ 2 files changed, 49 insertions(+), 3 deletions(-)
meta/classes/package.bbclass +37 -3
@@ -1140,6 +1140,14 @@ python split_and_strip_files () { # Modified the file so clear the cache cpath.updatecache(file) + def strip_pkgd_prefix(f): + nonlocal dvar + + if f.startswith(dvar): + return f[len(dvar):] + + return f + # # First lets process debug splitting #
@@ -1153,6 +1161,8 @@ python split_and_strip_files () { for file in staticlibs: results.append( (file,source_info(file, d)) ) + d.setVar("PKGDEBUGSOURCES", {strip_pkgd_prefix(f): sorted(s) for f, s in results}) + sources = set() for r in results: sources.update(r[1])
@@ -1460,6 +1470,7 @@ PKGDATA_VARS = "PN PE PV PR PKGE PKGV PKGR LICENSE DESCRIPTION SUMMARY RDEPENDS python emit_pkgdata() { from glob import glob import json + import bb.compress.zstd def process_postinst_on_target(pkg, mlprefix): pkgval = d.getVar('PKG_%s' % pkg)
@@ -1532,6 +1543,8 @@ fi with open(data_file, 'w') as fd: fd.write("PACKAGES: %s\n" % packages) + pkgdebugsource = d.getVar("PKGDEBUGSOURCES") or [] + pn = d.getVar('PN') global_variants = (d.getVar('MULTILIB_GLOBAL_VARIANTS') or "").split() variants = (d.getVar('MULTILIB_VARIANTS') or "").split()
@@ -1551,17 +1564,32 @@ fi pkgval = pkg d.setVar('PKG_%s' % pkg, pkg) + extended_data = { + "files_info": {} + } + pkgdestpkg = os.path.join(pkgdest, pkg) files = {} + files_extra = {} total_size = 0 seen = set() for f in pkgfiles[pkg]: - relpth = os.path.relpath(f, pkgdestpkg) + fpath = os.sep + os.path.relpath(f, pkgdestpkg) + fstat = os.lstat(f) - files[os.sep + relpth] = fstat.st_size + files[fpath] = fstat.st_size + + extended_data["files_info"].setdefault(fpath, {}) + extended_data["files_info"][fpath]['size'] = fstat.st_size + if fstat.st_ino not in seen: seen.add(fstat.st_ino) total_size += fstat.st_size + + if fpath in pkgdebugsource: + extended_data["files_info"][fpath]['debugsrc'] = pkgdebugsource[fpath] + del pkgdebugsource[fpath] + d.setVar('FILES_INFO', json.dumps(files, sort_keys=True)) process_postinst_on_target(pkg, d.getVar("MLPREFIX"))
@@ -1582,6 +1610,11 @@ fi sf.write('%s_%s: %d\n' % ('PKGSIZE', pkg, total_size)) + subdata_extended_file = pkgdatadir + "/extended/%s.json.zstd" % pkg + num_threads = int(d.getVar("BB_NUMBER_THREADS")) + with bb.compress.zstd.open(subdata_extended_file, "wt", encoding="utf-8", num_threads=num_threads) as f: + json.dump(extended_data, f, sort_keys=True, separators=(",", ":")) + # Symlinks needed for rprovides lookup rprov = d.getVar('RPROVIDES_%s' % pkg) or d.getVar('RPROVIDES') if rprov:
@@ -1612,7 +1645,8 @@ fi write_extra_runtime_pkgs(global_variants, packages, pkgdatadir) } -emit_pkgdata[dirs] = "${PKGDESTWORK}/runtime ${PKGDESTWORK}/runtime-reverse ${PKGDESTWORK}/runtime-rprovides" +emit_pkgdata[dirs] = "${PKGDESTWORK}/runtime ${PKGDESTWORK}/runtime-reverse ${PKGDESTWORK}/runtime-rprovides ${PKGDESTWORK}/extended" +emit_pkgdata[vardepsexclude] = "BB_NUMBER_THREADS" ldconfig_postinst_fragment() { if [ x"$D" = "x" ]; then
meta/lib/oe/packagedata.py +12 -0
@@ -57,6 +57,18 @@ def read_subpkgdata_dict(pkg, d): ret[newvar] = subd[var] return ret +def read_subpkgdata_extended(pkg, d): + import json + import bb.compress.zstd + + fn = d.expand("${PKGDATA_DIR}/extended/%s.json.zstd" % pkg) + try: + num_threads = int(d.getVar("BB_NUMBER_THREADS")) + with bb.compress.zstd.open(fn, "rt", encoding="utf-8", num_threads=num_threads) as f: + return json.load(f) + except FileNotFoundError: + return None + def _pkgmap(d): """Return a dictionary mapping package to recipe name."""
Signature · 1 line
2.33.0
PATCH 3/5 licenses: Add GPL+ licenses to map
In reply to Joshua WattAdds some missing licenses mappings related to the "+" versions of the GPL licenses
--- meta/conf/licenses.conf | 7 +++++++ 1 file changed, 7 insertions(+)
meta/conf/licenses.conf +7 -0
@@ -22,21 +22,28 @@ SPDXLICENSEMAP[GPLv1.0] = "GPL-1.0" SPDXLICENSEMAP[GPL-1.0-only] = "GPL-1.0" SPDXLICENSEMAP[GPL-2] = "GPL-2.0" SPDXLICENSEMAP[GPLv2] = "GPL-2.0" +SPDXLICENSEMAP[GPLv2+] = "GPL-2.0+" SPDXLICENSEMAP[GPLv2.0] = "GPL-2.0" +SPDXLICENSEMAP[GPLv2.0+] = "GPL-2.0+" SPDXLICENSEMAP[GPL-2.0-only] = "GPL-2.0" SPDXLICENSEMAP[GPL-3] = "GPL-3.0" SPDXLICENSEMAP[GPLv3] = "GPL-3.0" +SPDXLICENSEMAP[GPLv3+] = "GPL-3.0+" SPDXLICENSEMAP[GPLv3.0] = "GPL-3.0" +SPDXLICENSEMAP[GPLv3.0+] = "GPL-3.0+" SPDXLICENSEMAP[GPL-3.0-only] = "GPL-3.0" #LGPL variations SPDXLICENSEMAP[LGPLv2] = "LGPL-2.0" +SPDXLICENSEMAP[LGPLv2+] = "LGPL-2.0+" SPDXLICENSEMAP[LGPLv2.0] = "LGPL-2.0" SPDXLICENSEMAP[LGPL-2.0-only] = "LGPL-2.0" SPDXLICENSEMAP[LGPL2.1] = "LGPL-2.1" SPDXLICENSEMAP[LGPLv2.1] = "LGPL-2.1" +SPDXLICENSEMAP[LGPLv2.1+] = "LGPL-2.1+" SPDXLICENSEMAP[LGPL-2.1-only] = "LGPL-2.1" SPDXLICENSEMAP[LGPLv3] = "LGPL-3.0" +SPDXLICENSEMAP[LGPLv3+] = "LGPL-3.0+" SPDXLICENSEMAP[LGPL-3.0-only] = "LGPL-3.0" #MPL variations
Signature · 1 line
2.33.0
PATCH 4/5 create-spdx: Use gzip for compression
In reply to Joshua WattThe master version of the SPDX classes uses zstd for efficient compression, but it relies on the zstd tool to be present on the host system. Since dunfell supports older distros, we don't want to add this tool as an additional requirement so switch to using gzip instead.
--- meta/classes/create-spdx-2.2.bbclass | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-)
meta/classes/create-spdx-2.2.bbclass +5 -7
@@ -466,13 +466,11 @@ python do_create_spdx() { @contextmanager def optional_tarfile(name, guard, mode="w"): import tarfile - import bb.compress.zstd - - num_threads = int(d.getVar("BB_NUMBER_THREADS")) + import gzip if guard: name.parent.mkdir(parents=True, exist_ok=True) - with bb.compress.zstd.open(name, mode=mode + "b", num_threads=num_threads) as f: + with gzip.open(name, mode=mode + "b") as f: with tarfile.open(fileobj=f, mode=mode + "|") as tf: yield tf else:
@@ -931,7 +929,7 @@ def combine_spdx(d, rootfs_name, rootfs_deploydir, rootfs_spdxid, packages, spdx from datetime import timezone, datetime from pathlib import Path import tarfile - import bb.compress.zstd + import gzip creation_time = datetime.now(tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX"))
@@ -1002,8 +1000,8 @@ def combine_spdx(d, rootfs_name, rootfs_deploydir, rootfs_spdxid, packages, spdx index = {"documents": []} - spdx_tar_path = rootfs_deploydir / (rootfs_name + ".spdx.tar.zst") - with bb.compress.zstd.open(spdx_tar_path, "w", num_threads=num_threads) as f: + spdx_tar_path = rootfs_deploydir / (rootfs_name + ".spdx.tar.gz") + with gzip.open(spdx_tar_path, "w") as f: with tarfile.open(fileobj=f, mode="w|") as tar: def collect_spdx_document(path): nonlocal tar
Signature · 1 line
2.33.0
PATCH 5/5 classes/package: Use gzip for extended package data
In reply to Joshua WattThe master version of extended package data uses zstd for efficient compression, but it relies on the zstd tool to be present on the host system. Since dunfell supports older distros, we don't want to add this tool as an additional requirement so switch to using gzip instead.
--- meta/classes/package.bbclass | 7 +++---- meta/lib/oe/packagedata.py | 7 +++---- 2 files changed, 6 insertions(+), 8 deletions(-)
meta/classes/package.bbclass +3 -4
@@ -1470,7 +1470,7 @@ PKGDATA_VARS = "PN PE PV PR PKGE PKGV PKGR LICENSE DESCRIPTION SUMMARY RDEPENDS python emit_pkgdata() { from glob import glob import json - import bb.compress.zstd + import gzip def process_postinst_on_target(pkg, mlprefix): pkgval = d.getVar('PKG_%s' % pkg)
@@ -1610,9 +1610,8 @@ fi sf.write('%s_%s: %d\n' % ('PKGSIZE', pkg, total_size)) - subdata_extended_file = pkgdatadir + "/extended/%s.json.zstd" % pkg - num_threads = int(d.getVar("BB_NUMBER_THREADS")) - with bb.compress.zstd.open(subdata_extended_file, "wt", encoding="utf-8", num_threads=num_threads) as f: + subdata_extended_file = pkgdatadir + "/extended/%s.json.gz" % pkg + with gzip.open(subdata_extended_file, "wt", encoding="utf-8") as f: json.dump(extended_data, f, sort_keys=True, separators=(",", ":")) # Symlinks needed for rprovides lookup
meta/lib/oe/packagedata.py +3 -4
@@ -59,12 +59,11 @@ def read_subpkgdata_dict(pkg, d): def read_subpkgdata_extended(pkg, d): import json - import bb.compress.zstd + import gzip - fn = d.expand("${PKGDATA_DIR}/extended/%s.json.zstd" % pkg) + fn = d.expand("${PKGDATA_DIR}/extended/%s.json.gz" % pkg) try: - num_threads = int(d.getVar("BB_NUMBER_THREADS")) - with bb.compress.zstd.open(fn, "rt", encoding="utf-8", num_threads=num_threads) as f: + with gzip.open(fn, "rt", encoding="utf-8") as f: return json.load(f) except FileNotFoundError: return None
Signature · 1 line
2.33.0
v2 cover Backport SPDX Support
In reply to Joshua WattBackports support to creates SPDX during the build from the master branch, with a few changes to make it compatible with dunfell. Primarily, the SPDX data is compressed with gzip instead of zstd to prevent needing the zstd host tools on the older distros supported by dunfell
v2:
Fixed SPDX symlinks still using .zst instead of .gz extension
Joshua Watt (5):
classes/create-spdx: Backport classes/package: Add extended packaged data
licenses: Add GPL+ licenses to map
create-spdx: Use gzip for compression classes/package: Use gzip for extended package data
meta/classes/create-spdx-2.2.bbclass | 1067 +++++ meta/classes/create-spdx.bbclass | 8 + meta/classes/package.bbclass | 39 +- meta/conf/licenses.conf | 7 + meta/files/spdx-licenses.json | 5937 ++++++++++++++++++++++++++ meta/lib/oe/packagedata.py | 11 + meta/lib/oe/sbom.py | 84 + meta/lib/oe/spdx.py | 357 ++ 8 files changed, 7507 insertions(+), 3 deletions(-) create mode 100644 meta/classes/create-spdx-2.2.bbclass create mode 100644 meta/classes/create-spdx.bbclass create mode 100644 meta/files/spdx-licenses.json create mode 100644 meta/lib/oe/sbom.py create mode 100644 meta/lib/oe/spdx.py
Signature · 1 line
2.33.0
PATCH v2 1/5 classes/create-spdx: Backport
Backports the create-spdx classes from the latest versions on master. This backport is a simple copy with no modifications, as its too complex to cherry-pick all the corresponding changes. This will give an appropriate base commit for subsequent changes and if necessary additional backport cherry-picks from master in the future.
--- meta/classes/create-spdx-2.2.bbclass | 1069 +++++ meta/classes/create-spdx.bbclass | 8 + meta/files/spdx-licenses.json | 5937 ++++++++++++++++++++++++++ meta/lib/oe/sbom.py | 84 + meta/lib/oe/spdx.py | 357 ++ 5 files changed, 7455 insertions(+) create mode 100644 meta/classes/create-spdx-2.2.bbclass create mode 100644 meta/classes/create-spdx.bbclass create mode 100644 meta/files/spdx-licenses.json create mode 100644 meta/lib/oe/sbom.py create mode 100644 meta/lib/oe/spdx.py
meta/classes/create-spdx-2.2.bbclass new file +1069 -0
@@ -0,0 +1,1069 @@ +# +# Copyright OpenEmbedded Contributors +# +# SPDX-License-Identifier: GPL-2.0-only +# + +DEPLOY_DIR_SPDX ??= "${DEPLOY_DIR}/spdx/${MACHINE}" + +# The product name that the CVE database uses. Defaults to BPN, but may need to +# be overriden per recipe (for example tiff.bb sets CVE_PRODUCT=libtiff). +CVE_PRODUCT ??= "${BPN}" +CVE_VERSION ??= "${PV}" + +SPDXDIR ??= "${WORKDIR}/spdx" +SPDXDEPLOY = "${SPDXDIR}/deploy" +SPDXWORK = "${SPDXDIR}/work" +SPDXIMAGEWORK = "${SPDXDIR}/image-work" +SPDXSDKWORK = "${SPDXDIR}/sdk-work" + +SPDX_TOOL_NAME ??= "oe-spdx-creator" +SPDX_TOOL_VERSION ??= "1.0" + +SPDXRUNTIMEDEPLOY = "${SPDXDIR}/runtime-deploy" + +SPDX_INCLUDE_SOURCES ??= "0" +SPDX_ARCHIVE_SOURCES ??= "0" +SPDX_ARCHIVE_PACKAGED ??= "0" + +SPDX_UUID_NAMESPACE ??= "sbom.openembedded.org" +SPDX_NAMESPACE_PREFIX ??= "http://spdx.org/spdxdoc" +SPDX_PRETTY ??= "0" + +SPDX_LICENSES ??= "${COREBASE}/meta/files/spdx-licenses.json" + +SPDX_CUSTOM_ANNOTATION_VARS ??= "" + +SPDX_ORG ??= "OpenEmbedded ()" +SPDX_SUPPLIER ??= "Organization: ${SPDX_ORG}" +SPDX_SUPPLIER[doc] = "The SPDX PackageSupplier field for SPDX packages created from \ + this recipe. For SPDX documents create using this class during the build, this \ + is the contact information for the person or organization who is doing the \ + build." + +def extract_licenses(filename): + import re + + lic_regex = re.compile(rb'^\W*SPDX-License-Identifier:\s*([ \w\d.()+-]+?)(?:\s+\W*)?$', re.MULTILINE) + + try: + with open(filename, 'rb') as f: + size = min(15000, os.stat(filename).st_size) + txt = f.read(size) + licenses = re.findall(lic_regex, txt) + if licenses: + ascii_licenses = [lic.decode('ascii') for lic in licenses] + return ascii_licenses + except Exception as e: + bb.warn(f"Exception reading {filename}: {e}") + return None + +def get_doc_namespace(d, doc): + import uuid + namespace_uuid = uuid.uuid5(uuid.NAMESPACE_DNS, d.getVar("SPDX_UUID_NAMESPACE")) + return "%s/%s-%s" % (d.getVar("SPDX_NAMESPACE_PREFIX"), doc.name, str(uuid.uuid5(namespace_uuid, doc.name))) + +def create_annotation(d, comment): + from datetime import datetime, timezone + + creation_time = datetime.now(tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + annotation = oe.spdx.SPDXAnnotation() + annotation.annotationDate = creation_time + annotation.annotationType = "OTHER" + annotation.annotator = "Tool: %s - %s" % (d.getVar("SPDX_TOOL_NAME"), d.getVar("SPDX_TOOL_VERSION")) + annotation.comment = comment + return annotation + +def recipe_spdx_is_native(d, recipe): + return any(a.annotationType == "OTHER" and + a.annotator == "Tool: %s - %s" % (d.getVar("SPDX_TOOL_NAME"), d.getVar("SPDX_TOOL_VERSION")) and + a.comment == "isNative" for a in recipe.annotations) + +def is_work_shared_spdx(d): + return bb.data.inherits_class('kernel', d) or ('work-shared' in d.getVar('WORKDIR')) + +def get_json_indent(d): + if d.getVar("SPDX_PRETTY") == "1": + return 2 + return None + +python() { + import json + if d.getVar("SPDX_LICENSE_DATA"): + return + + with open(d.getVar("SPDX_LICENSES"), "r") as f: + data = json.load(f) + # Transform the license array to a dictionary + data["licenses"] = {l["licenseId"]: l for l in data["licenses"]} + d.setVar("SPDX_LICENSE_DATA", data) +} + +def convert_license_to_spdx(lic, document, d, existing={}): + from pathlib import Path + import oe.spdx + + license_data = d.getVar("SPDX_LICENSE_DATA") + extracted = {} + + def add_extracted_license(ident, name): + nonlocal document + + if name in extracted: + return + + extracted_info = oe.spdx.SPDXExtractedLicensingInfo() + extracted_info.name = name + extracted_info.licenseId = ident + extracted_info.extractedText = None + + if name == "PD": + # Special-case this. + extracted_info.extractedText = "Software released to the public domain" + else: + # Seach for the license in COMMON_LICENSE_DIR and LICENSE_PATH + for directory in [d.getVar('COMMON_LICENSE_DIR')] + (d.getVar('LICENSE_PATH') or '').split(): + try: + with (Path(directory) / name).open(errors="replace") as f: + extracted_info.extractedText = f.read() + break + except FileNotFoundError: + pass + if extracted_info.extractedText is None: + # If it's not SPDX or PD, then NO_GENERIC_LICENSE must be set + filename = d.getVarFlag('NO_GENERIC_LICENSE', name) + if filename: + filename = d.expand("${S}/" + filename) + with open(filename, errors="replace") as f: + extracted_info.extractedText = f.read() + else: + bb.error("Cannot find any text for license %s" % name) + + extracted[name] = extracted_info + document.hasExtractedLicensingInfos.append(extracted_info) + + def convert(l): + if l == "(" or l == ")": + return l + + if l == "&": + return "AND" + + if l == "|": + return "OR" + + if l == "CLOSED": + return "NONE" + + spdx_license = d.getVarFlag("SPDXLICENSEMAP", l) or l + if spdx_license in license_data["licenses"]: + return spdx_license + + try: + spdx_license = existing[l] + except KeyError: + spdx_license = "LicenseRef-" + l + add_extracted_license(spdx_license, l) + + return spdx_license + + lic_split = lic.replace("(", " ( ").replace(")", " ) ").split() + + return ' '.join(convert(l) for l in lic_split) + +def process_sources(d): + pn = d.getVar('PN') + assume_provided = (d.getVar("ASSUME_PROVIDED") or "").split() + if pn in assume_provided: + for p in d.getVar("PROVIDES").split(): + if p != pn: + pn = p + break + + # glibc-locale: do_fetch, do_unpack and do_patch tasks have been deleted, + # so avoid archiving source here. + if pn.startswith('glibc-locale'): + return False + if d.getVar('PN') == "libtool-cross": + return False + if d.getVar('PN') == "libgcc-initial": + return False + if d.getVar('PN') == "shadow-sysroot": + return False + + # We just archive gcc-source for all the gcc related recipes + if d.getVar('BPN') in ['gcc', 'libgcc']: + bb.debug(1, 'spdx: There is bug in scan of %s is, do nothing' % pn) + return False + + return True + + +def add_package_files(d, doc, spdx_pkg, topdir, get_spdxid, get_types, *, archive=None, ignore_dirs=[], ignore_top_level_dirs=[]): + from pathlib import Path + import oe.spdx + import hashlib + + source_date_epoch = d.getVar("SOURCE_DATE_EPOCH") + if source_date_epoch: + source_date_epoch = int(source_date_epoch) + + sha1s = [] + spdx_files = [] + + file_counter = 1 + for subdir, dirs, files in os.walk(topdir): + dirs[:] = [d for d in dirs if d not in ignore_dirs] + if subdir == str(topdir): + dirs[:] = [d for d in dirs if d not in ignore_top_level_dirs] + + for file in files: + filepath = Path(subdir) / file + filename = str(filepath.relative_to(topdir)) + + if not filepath.is_symlink() and filepath.is_file(): + spdx_file = oe.spdx.SPDXFile() + spdx_file.SPDXID = get_spdxid(file_counter) + for t in get_types(filepath): + spdx_file.fileTypes.append(t) + spdx_file.fileName = filename + + if archive is not None: + with filepath.open("rb") as f: + info = archive.gettarinfo(fileobj=f) + info.name = filename + info.uid = 0 + info.gid = 0 + info.uname = "root" + info.gname = "root" + + if source_date_epoch is not None and info.mtime > source_date_epoch: + info.mtime = source_date_epoch + + archive.addfile(info, f) + + sha1 = bb.utils.sha1_file(filepath) + sha1s.append(sha1) + spdx_file.checksums.append(oe.spdx.SPDXChecksum( + algorithm="SHA1", + checksumValue=sha1, + )) + spdx_file.checksums.append(oe.spdx.SPDXChecksum( + algorithm="SHA256", + checksumValue=bb.utils.sha256_file(filepath), + )) + + if "SOURCE" in spdx_file.fileTypes: + extracted_lics = extract_licenses(filepath) + if extracted_lics: + spdx_file.licenseInfoInFiles = extracted_lics + + doc.files.append(spdx_file) + doc.add_relationship(spdx_pkg, "CONTAINS", spdx_file) + spdx_pkg.hasFiles.append(spdx_file.SPDXID) + + spdx_files.append(spdx_file) + + file_counter += 1 + + sha1s.sort() + verifier = hashlib.sha1() + for v in sha1s: + verifier.update(v.encode("utf-8")) + spdx_pkg.packageVerificationCode.packageVerificationCodeValue = verifier.hexdigest() + + return spdx_files + + +def add_package_sources_from_debug(d, package_doc, spdx_package, package, package_files, sources): + from pathlib import Path + import hashlib + import oe.packagedata + import oe.spdx + + debug_search_paths = [ + Path(d.getVar('PKGD')), + Path(d.getVar('STAGING_DIR_TARGET')), + Path(d.getVar('STAGING_DIR_NATIVE')), + Path(d.getVar('STAGING_KERNEL_DIR')), + ] + + pkg_data = oe.packagedata.read_subpkgdata_extended(package, d) + + if pkg_data is None: + return + + for file_path, file_data in pkg_data["files_info"].items(): + if not "debugsrc" in file_data: + continue + + for pkg_file in package_files: + if file_path.lstrip("/") == pkg_file.fileName.lstrip("/"): + break + else: + bb.fatal("No package file found for %s" % str(file_path)) + continue + + for debugsrc in file_data["debugsrc"]: + ref_id = "NOASSERTION" + for search in debug_search_paths: + if debugsrc.startswith("/usr/src/kernel"): + debugsrc_path = search / debugsrc.replace('/usr/src/kernel/', '') + else: + debugsrc_path = search / debugsrc.lstrip("/") + if not debugsrc_path.exists(): + continue + + file_sha256 = bb.utils.sha256_file(debugsrc_path) + + if file_sha256 in sources: + source_file = sources[file_sha256] + + doc_ref = package_doc.find_external_document_ref(source_file.doc.documentNamespace) + if doc_ref is None: + doc_ref = oe.spdx.SPDXExternalDocumentRef() + doc_ref.externalDocumentId = "DocumentRef-dependency-" + source_file.doc.name + doc_ref.spdxDocument = source_file.doc.documentNamespace + doc_ref.checksum.algorithm = "SHA1" + doc_ref.checksum.checksumValue = source_file.doc_sha1 + package_doc.externalDocumentRefs.append(doc_ref) + + ref_id = "%s:%s" % (doc_ref.externalDocumentId, source_file.file.SPDXID) + else: + bb.debug(1, "Debug source %s with SHA256 %s not found in any dependency" % (str(debugsrc_path), file_sha256)) + break + else: + bb.debug(1, "Debug source %s not found" % debugsrc) + + package_doc.add_relationship(pkg_file, "GENERATED_FROM", ref_id, comment=debugsrc) + +def collect_dep_recipes(d, doc, spdx_recipe): + from pathlib import Path + import oe.sbom + import oe.spdx + + deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX")) + + dep_recipes = [] + taskdepdata = d.getVar("BB_TASKDEPDATA", False) + deps = sorted(set( + dep[0] for dep in taskdepdata.values() if + dep[1] == "do_create_spdx" and dep[0] != d.getVar("PN") + )) + for dep_pn in deps: + dep_recipe_path = deploy_dir_spdx / "recipes" / ("recipe-%s.spdx.json" % dep_pn) + + spdx_dep_doc, spdx_dep_sha1 = oe.sbom.read_doc(dep_recipe_path) + + for pkg in spdx_dep_doc.packages: + if pkg.name == dep_pn: + spdx_dep_recipe = pkg + break + else: + continue + + dep_recipes.append(oe.sbom.DepRecipe(spdx_dep_doc, spdx_dep_sha1, spdx_dep_recipe)) + + dep_recipe_ref = oe.spdx.SPDXExternalDocumentRef() + dep_recipe_ref.externalDocumentId = "DocumentRef-dependency-" + spdx_dep_doc.name + dep_recipe_ref.spdxDocument = spdx_dep_doc.documentNamespace + dep_recipe_ref.checksum.algorithm = "SHA1" + dep_recipe_ref.checksum.checksumValue = spdx_dep_sha1 + + doc.externalDocumentRefs.append(dep_recipe_ref) + + doc.add_relationship( + "%s:%s" % (dep_recipe_ref.externalDocumentId, spdx_dep_recipe.SPDXID), + "BUILD_DEPENDENCY_OF", + spdx_recipe + ) + + return dep_recipes + +collect_dep_recipes[vardepsexclude] += "BB_TASKDEPDATA" +collect_dep_recipes[vardeps] += "DEPENDS" + +def collect_dep_sources(d, dep_recipes): + import oe.sbom + + sources = {} + for dep in dep_recipes: + # Don't collect sources from native recipes as they + # match non-native sources also. + if recipe_spdx_is_native(d, dep.recipe): + continue + recipe_files = set(dep.recipe.hasFiles) + + for spdx_file in dep.doc.files: + if spdx_file.SPDXID not in recipe_files: + continue + + if "SOURCE" in spdx_file.fileTypes: + for checksum in spdx_file.checksums: + if checksum.algorithm == "SHA256": + sources[checksum.checksumValue] = oe.sbom.DepSource(dep.doc, dep.doc_sha1, dep.recipe, spdx_file) + break + + return sources + +def add_download_packages(d, doc, recipe): + import os.path + from bb.fetch2 import decodeurl, CHECKSUM_LIST + import bb.process + import oe.spdx + import oe.sbom + + for download_idx, src_uri in enumerate(d.getVar('SRC_URI').split()): + f = bb.fetch2.FetchData(src_uri, d) + + for name in f.names: + package = oe.spdx.SPDXPackage() + package.name = "%s-source-%d" % (d.getVar("PN"), download_idx + 1) + package.SPDXID = oe.sbom.get_download_spdxid(d, download_idx + 1) + + if f.type == "file": + continue + + uri = f.type + proto = getattr(f, "proto", None) + if proto is not None: + uri = uri + "+" + proto + uri = uri + "://" + f.host + f.path + + if f.method.supports_srcrev(): + uri = uri + "@" + f.revisions[name] + + if f.method.supports_checksum(f): + for checksum_id in CHECKSUM_LIST: + if checksum_id.upper() not in oe.spdx.SPDXPackage.ALLOWED_CHECKSUMS: + continue + + expected_checksum = getattr(f, "%s_expected" % checksum_id) + if expected_checksum is None: + continue + + c = oe.spdx.SPDXChecksum() + c.algorithm = checksum_id.upper() + c.checksumValue = expected_checksum + package.checksums.append(c) + + package.downloadLocation = uri + doc.packages.append(package) + doc.add_relationship(doc, "DESCRIBES", package) + # In the future, we might be able to do more fancy dependencies, + # but this should be sufficient for now + doc.add_relationship(package, "BUILD_DEPENDENCY_OF", recipe) + +python do_create_spdx() { + from datetime import datetime, timezone + import oe.sbom + import oe.spdx + import uuid + from pathlib import Path + from contextlib import contextmanager + import oe.cve_check + + @contextmanager + def optional_tarfile(name, guard, mode="w"): + import tarfile + import bb.compress.zstd + + num_threads = int(d.getVar("BB_NUMBER_THREADS")) + + if guard: + name.parent.mkdir(parents=True, exist_ok=True) + with bb.compress.zstd.open(name, mode=mode + "b", num_threads=num_threads) as f: + with tarfile.open(fileobj=f, mode=mode + "|") as tf: + yield tf + else: + yield None + + + deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX")) + spdx_workdir = Path(d.getVar("SPDXWORK")) + include_sources = d.getVar("SPDX_INCLUDE_SOURCES") == "1" + archive_sources = d.getVar("SPDX_ARCHIVE_SOURCES") == "1" + archive_packaged = d.getVar("SPDX_ARCHIVE_PACKAGED") == "1" + + creation_time = datetime.now(tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + doc = oe.spdx.SPDXDocument() + + doc.name = "recipe-" + d.getVar("PN") + doc.documentNamespace = get_doc_namespace(d, doc) + doc.creationInfo.created = creation_time + doc.creationInfo.comment = "This document was created by analyzing recipe files during the build." + doc.creationInfo.licenseListVersion = d.getVar("SPDX_LICENSE_DATA")["licenseListVersion"] + doc.creationInfo.creators.append("Tool: OpenEmbedded Core create-spdx.bbclass") + doc.creationInfo.creators.append("Organization: %s" % d.getVar("SPDX_ORG")) + doc.creationInfo.creators.append("Person: N/A ()") + + recipe = oe.spdx.SPDXPackage() + recipe.name = d.getVar("PN") + recipe.versionInfo = d.getVar("PV") + recipe.SPDXID = oe.sbom.get_recipe_spdxid(d) + recipe.supplier = d.getVar("SPDX_SUPPLIER") + if bb.data.inherits_class("native", d) or bb.data.inherits_class("cross", d): + recipe.annotations.append(create_annotation(d, "isNative")) + + homepage = d.getVar("HOMEPAGE") + if homepage: + recipe.homepage = homepage + + license = d.getVar("LICENSE") + if license: + recipe.licenseDeclared = convert_license_to_spdx(license, doc, d) + + summary = d.getVar("SUMMARY") + if summary: + recipe.summary = summary + + description = d.getVar("DESCRIPTION") + if description: + recipe.description = description + + if d.getVar("SPDX_CUSTOM_ANNOTATION_VARS"): + for var in d.getVar('SPDX_CUSTOM_ANNOTATION_VARS').split(): + recipe.annotations.append(create_annotation(d, var + "=" + d.getVar(var))) + + # Some CVEs may be patched during the build process without incrementing the version number, + # so querying for CVEs based on the CPE id can lead to false positives. To account for this, + # save the CVEs fixed by patches to source information field in the SPDX. + patched_cves = oe.cve_check.get_patched_cves(d) + patched_cves = list(patched_cves) + patched_cves = ' '.join(patched_cves) + if patched_cves: + recipe.sourceInfo = "CVEs fixed: " + patched_cves + + cpe_ids = oe.cve_check.get_cpe_ids(d.getVar("CVE_PRODUCT"), d.getVar("CVE_VERSION")) + if cpe_ids: + for cpe_id in cpe_ids: + cpe = oe.spdx.SPDXExternalReference() + cpe.referenceCategory = "SECURITY" + cpe.referenceType = "http://spdx.org/rdf/references/cpe23Type" + cpe.referenceLocator = cpe_id + recipe.externalRefs.append(cpe) + + doc.packages.append(recipe) + doc.add_relationship(doc, "DESCRIBES", recipe) + + add_download_packages(d, doc, recipe) + + if process_sources(d) and include_sources: + recipe_archive = deploy_dir_spdx / "recipes" / (doc.name + ".tar.zst") + with optional_tarfile(recipe_archive, archive_sources) as archive: + spdx_get_src(d) + + add_package_files( + d, + doc, + recipe, + spdx_workdir, + lambda file_counter: "SPDXRef-SourceFile-%s-%d" % (d.getVar("PN"), file_counter), + lambda filepath: ["SOURCE"], + ignore_dirs=[".git"], + ignore_top_level_dirs=["temp"], + archive=archive, + ) + + if archive is not None: + recipe.packageFileName = str(recipe_archive.name) + + dep_recipes = collect_dep_recipes(d, doc, recipe) + + doc_sha1 = oe.sbom.write_doc(d, doc, "recipes", indent=get_json_indent(d)) + dep_recipes.append(oe.sbom.DepRecipe(doc, doc_sha1, recipe)) + + recipe_ref = oe.spdx.SPDXExternalDocumentRef() + recipe_ref.externalDocumentId = "DocumentRef-recipe-" + recipe.name + recipe_ref.spdxDocument = doc.documentNamespace + recipe_ref.checksum.algorithm = "SHA1" + recipe_ref.checksum.checksumValue = doc_sha1 + + sources = collect_dep_sources(d, dep_recipes) + found_licenses = {license.name:recipe_ref.externalDocumentId + ":" + license.licenseId for license in doc.hasExtractedLicensingInfos} + + if not recipe_spdx_is_native(d, recipe): + bb.build.exec_func("read_subpackage_metadata", d) + + pkgdest = Path(d.getVar("PKGDEST")) + for package in d.getVar("PACKAGES").split(): + if not oe.packagedata.packaged(package, d): + continue + + package_doc = oe.spdx.SPDXDocument() + pkg_name = d.getVar("PKG:%s" % package) or package + package_doc.name = pkg_name + package_doc.documentNamespace = get_doc_namespace(d, package_doc) + package_doc.creationInfo.created = creation_time + package_doc.creationInfo.comment = "This document was created by analyzing packages created during the build." + package_doc.creationInfo.licenseListVersion = d.getVar("SPDX_LICENSE_DATA")["licenseListVersion"] + package_doc.creationInfo.creators.append("Tool: OpenEmbedded Core create-spdx.bbclass") + package_doc.creationInfo.creators.append("Organization: %s" % d.getVar("SPDX_ORG")) + package_doc.creationInfo.creators.append("Person: N/A ()") + package_doc.externalDocumentRefs.append(recipe_ref) + + package_license = d.getVar("LICENSE:%s" % package) or d.getVar("LICENSE") + + spdx_package = oe.spdx.SPDXPackage() + + spdx_package.SPDXID = oe.sbom.get_package_spdxid(pkg_name) + spdx_package.name = pkg_name + spdx_package.versionInfo = d.getVar("PV") + spdx_package.licenseDeclared = convert_license_to_spdx(package_license, package_doc, d, found_licenses) + spdx_package.supplier = d.getVar("SPDX_SUPPLIER") + + package_doc.packages.append(spdx_package) + + package_doc.add_relationship(spdx_package, "GENERATED_FROM", "%s:%s" % (recipe_ref.externalDocumentId, recipe.SPDXID)) + package_doc.add_relationship(package_doc, "DESCRIBES", spdx_package) + + package_archive = deploy_dir_spdx / "packages" / (package_doc.name + ".tar.zst") + with optional_tarfile(package_archive, archive_packaged) as archive: + package_files = add_package_files( + d, + package_doc, + spdx_package, + pkgdest / package, + lambda file_counter: oe.sbom.get_packaged_file_spdxid(pkg_name, file_counter), + lambda filepath: ["BINARY"], + ignore_top_level_dirs=['CONTROL', 'DEBIAN'], + archive=archive, + ) + + if archive is not None: + spdx_package.packageFileName = str(package_archive.name) + + add_package_sources_from_debug(d, package_doc, spdx_package, package, package_files, sources) + + oe.sbom.write_doc(d, package_doc, "packages", indent=get_json_indent(d)) +} +# NOTE: depending on do_unpack is a hack that is necessary to get it's dependencies for archive the source +addtask do_create_spdx after do_package do_packagedata do_unpack before do_populate_sdk do_build do_rm_work + +SSTATETASKS += "do_create_spdx" +do_create_spdx[sstate-inputdirs] = "${SPDXDEPLOY}" +do_create_spdx[sstate-outputdirs] = "${DEPLOY_DIR_SPDX}" + +python do_create_spdx_setscene () { + sstate_setscene(d) +} +addtask do_create_spdx_setscene + +do_create_spdx[dirs] = "${SPDXWORK}" +do_create_spdx[cleandirs] = "${SPDXDEPLOY} ${SPDXWORK}" +do_create_spdx[depends] += "${PATCHDEPENDENCY}" +do_create_spdx[deptask] = "do_create_spdx" + +def collect_package_providers(d): + from pathlib import Path + import oe.sbom + import oe.spdx + import json + + deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX")) + + providers = {} + + taskdepdata = d.getVar("BB_TASKDEPDATA", False) + deps = sorted(set( + dep[0] for dep in taskdepdata.values() if dep[0] != d.getVar("PN") + )) + deps.append(d.getVar("PN")) + + for dep_pn in deps: + recipe_data = oe.packagedata.read_pkgdata(dep_pn, d) + + for pkg in recipe_data.get("PACKAGES", "").split(): + + pkg_data = oe.packagedata.read_subpkgdata_dict(pkg, d) + rprovides = set(n for n, _ in bb.utils.explode_dep_versions2(pkg_data.get("RPROVIDES", "")).items()) + rprovides.add(pkg) + + for r in rprovides: + providers[r] = pkg + + return providers + +collect_package_providers[vardepsexclude] += "BB_TASKDEPDATA" + +python do_create_runtime_spdx() { + from datetime import datetime, timezone + import oe.sbom + import oe.spdx + import oe.packagedata + from pathlib import Path + + deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX")) + spdx_deploy = Path(d.getVar("SPDXRUNTIMEDEPLOY")) + is_native = bb.data.inherits_class("native", d) or bb.data.inherits_class("cross", d) + + creation_time = datetime.now(tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + providers = collect_package_providers(d) + + if not is_native: + bb.build.exec_func("read_subpackage_metadata", d) + + dep_package_cache = {} + + pkgdest = Path(d.getVar("PKGDEST")) + for package in d.getVar("PACKAGES").split(): + localdata = bb.data.createCopy(d) + pkg_name = d.getVar("PKG:%s" % package) or package + localdata.setVar("PKG", pkg_name) + localdata.setVar('OVERRIDES', d.getVar("OVERRIDES", False) + ":" + package) + + if not oe.packagedata.packaged(package, localdata): + continue + + pkg_spdx_path = deploy_dir_spdx / "packages" / (pkg_name + ".spdx.json") + + package_doc, package_doc_sha1 = oe.sbom.read_doc(pkg_spdx_path) + + for p in package_doc.packages: + if p.name == pkg_name: + spdx_package = p + break + else: + bb.fatal("Package '%s' not found in %s" % (pkg_name, pkg_spdx_path)) + + runtime_doc = oe.spdx.SPDXDocument() + runtime_doc.name = "runtime-" + pkg_name + runtime_doc.documentNamespace = get_doc_namespace(localdata, runtime_doc) + runtime_doc.creationInfo.created = creation_time + runtime_doc.creationInfo.comment = "This document was created by analyzing package runtime dependencies." + runtime_doc.creationInfo.licenseListVersion = d.getVar("SPDX_LICENSE_DATA")["licenseListVersion"] + runtime_doc.creationInfo.creators.append("Tool: OpenEmbedded Core create-spdx.bbclass") + runtime_doc.creationInfo.creators.append("Organization: %s" % d.getVar("SPDX_ORG")) + runtime_doc.creationInfo.creators.append("Person: N/A ()") + + package_ref = oe.spdx.SPDXExternalDocumentRef() + package_ref.externalDocumentId = "DocumentRef-package-" + package + package_ref.spdxDocument = package_doc.documentNamespace + package_ref.checksum.algorithm = "SHA1" + package_ref.checksum.checksumValue = package_doc_sha1 + + runtime_doc.externalDocumentRefs.append(package_ref) + + runtime_doc.add_relationship( + runtime_doc.SPDXID, + "AMENDS", + "%s:%s" % (package_ref.externalDocumentId, package_doc.SPDXID) + ) + + deps = bb.utils.explode_dep_versions2(localdata.getVar("RDEPENDS") or "") + seen_deps = set() + for dep, _ in deps.items(): + if dep in seen_deps: + continue + + if dep not in providers: + continue + + dep = providers[dep] + + if not oe.packagedata.packaged(dep, localdata): + continue + + dep_pkg_data = oe.packagedata.read_subpkgdata_dict(dep, d) + dep_pkg = dep_pkg_data["PKG"] + + if dep in dep_package_cache: + (dep_spdx_package, dep_package_ref) = dep_package_cache[dep] + else: + dep_path = deploy_dir_spdx / "packages" / ("%s.spdx.json" % dep_pkg) + + spdx_dep_doc, spdx_dep_sha1 = oe.sbom.read_doc(dep_path) + + for pkg in spdx_dep_doc.packages: + if pkg.name == dep_pkg: + dep_spdx_package = pkg + break + else: + bb.fatal("Package '%s' not found in %s" % (dep_pkg, dep_path)) + + dep_package_ref = oe.spdx.SPDXExternalDocumentRef() + dep_package_ref.externalDocumentId = "DocumentRef-runtime-dependency-" + spdx_dep_doc.name + dep_package_ref.spdxDocument = spdx_dep_doc.documentNamespace + dep_package_ref.checksum.algorithm = "SHA1" + dep_package_ref.checksum.checksumValue = spdx_dep_sha1 + + dep_package_cache[dep] = (dep_spdx_package, dep_package_ref) + + runtime_doc.externalDocumentRefs.append(dep_package_ref) + + runtime_doc.add_relationship( + "%s:%s" % (dep_package_ref.externalDocumentId, dep_spdx_package.SPDXID), + "RUNTIME_DEPENDENCY_OF", + "%s:%s" % (package_ref.externalDocumentId, spdx_package.SPDXID) + ) + seen_deps.add(dep) + + oe.sbom.write_doc(d, runtime_doc, "runtime", spdx_deploy, indent=get_json_indent(d)) +} + +addtask do_create_runtime_spdx after do_create_spdx before do_build do_rm_work +SSTATETASKS += "do_create_runtime_spdx" +do_create_runtime_spdx[sstate-inputdirs] = "${SPDXRUNTIMEDEPLOY}" +do_create_runtime_spdx[sstate-outputdirs] = "${DEPLOY_DIR_SPDX}" + +python do_create_runtime_spdx_setscene () { + sstate_setscene(d) +} +addtask do_create_runtime_spdx_setscene + +do_create_runtime_spdx[dirs] = "${SPDXRUNTIMEDEPLOY}" +do_create_runtime_spdx[cleandirs] = "${SPDXRUNTIMEDEPLOY}" +do_create_runtime_spdx[rdeptask] = "do_create_spdx" + +def spdx_get_src(d): + """ + save patched source of the recipe in SPDX_WORKDIR. + """ + import shutil + spdx_workdir = d.getVar('SPDXWORK') + spdx_sysroot_native = d.getVar('STAGING_DIR_NATIVE') + pn = d.getVar('PN') + + workdir = d.getVar("WORKDIR") + + try: + # The kernel class functions require it to be on work-shared, so we dont change WORKDIR + if not is_work_shared_spdx(d): + # Change the WORKDIR to make do_unpack do_patch run in another dir. + d.setVar('WORKDIR', spdx_workdir) + # Restore the original path to recipe's native sysroot (it's relative to WORKDIR). + d.setVar('STAGING_DIR_NATIVE', spdx_sysroot_native) + + # The changed 'WORKDIR' also caused 'B' changed, create dir 'B' for the + # possibly requiring of the following tasks (such as some recipes's + # do_patch required 'B' existed). + bb.utils.mkdirhier(d.getVar('B')) + + bb.build.exec_func('do_unpack', d) + # Copy source of kernel to spdx_workdir + if is_work_shared_spdx(d): + share_src = d.getVar('WORKDIR') + d.setVar('WORKDIR', spdx_workdir) + d.setVar('STAGING_DIR_NATIVE', spdx_sysroot_native) + src_dir = spdx_workdir + "/" + d.getVar('PN')+ "-" + d.getVar('PV') + "-" + d.getVar('PR') + bb.utils.mkdirhier(src_dir) + if bb.data.inherits_class('kernel',d): + share_src = d.getVar('STAGING_KERNEL_DIR') + cmd_copy_share = "cp -rf " + share_src + "/* " + src_dir + "/" + cmd_copy_shared_res = os.popen(cmd_copy_share).read() + bb.note("cmd_copy_shared_result = " + cmd_copy_shared_res) + + git_path = src_dir + "/.git" + if os.path.exists(git_path): + shutils.rmtree(git_path) + + # Make sure gcc and kernel sources are patched only once + if not (d.getVar('SRC_URI') == "" or is_work_shared_spdx(d)): + bb.build.exec_func('do_patch', d) + + # Some userland has no source. + if not os.path.exists( spdx_workdir ): + bb.utils.mkdirhier(spdx_workdir) + finally: + d.setVar("WORKDIR", workdir) + +do_rootfs[recrdeptask] += "do_create_spdx do_create_runtime_spdx" +do_rootfs[cleandirs] += "${SPDXIMAGEWORK}" + +ROOTFS_POSTUNINSTALL_COMMAND =+ "image_combine_spdx ; " + +do_populate_sdk[recrdeptask] += "do_create_spdx do_create_runtime_spdx" +do_populate_sdk[cleandirs] += "${SPDXSDKWORK}" +POPULATE_SDK_POST_HOST_COMMAND:append:task-populate-sdk = " sdk_host_combine_spdx; " +POPULATE_SDK_POST_TARGET_COMMAND:append:task-populate-sdk = " sdk_target_combine_spdx; " + +python image_combine_spdx() { + import os + import oe.sbom + from pathlib import Path + from oe.rootfs import image_list_installed_packages + + image_name = d.getVar("IMAGE_NAME") + image_link_name = d.getVar("IMAGE_LINK_NAME") + imgdeploydir = Path(d.getVar("IMGDEPLOYDIR")) + img_spdxid = oe.sbom.get_image_spdxid(image_name) + packages = image_list_installed_packages(d) + + combine_spdx(d, image_name, imgdeploydir, img_spdxid, packages, Path(d.getVar("SPDXIMAGEWORK"))) + + def make_image_link(target_path, suffix): + if image_link_name: + link = imgdeploydir / (image_link_name + suffix) + if link != target_path: + link.symlink_to(os.path.relpath(target_path, link.parent)) + + spdx_tar_path = imgdeploydir / (image_name + ".spdx.tar.zst") + make_image_link(spdx_tar_path, ".spdx.tar.zst") +} + +python sdk_host_combine_spdx() { + sdk_combine_spdx(d, "host") +} + +python sdk_target_combine_spdx() { + sdk_combine_spdx(d, "target") +} + +def sdk_combine_spdx(d, sdk_type): + import oe.sbom + from pathlib import Path + from oe.sdk import sdk_list_installed_packages + + sdk_name = d.getVar("SDK_NAME") + "-" + sdk_type + sdk_deploydir = Path(d.getVar("SDKDEPLOYDIR")) + sdk_spdxid = oe.sbom.get_sdk_spdxid(sdk_name) + sdk_packages = sdk_list_installed_packages(d, sdk_type == "target") + combine_spdx(d, sdk_name, sdk_deploydir, sdk_spdxid, sdk_packages, Path(d.getVar('SPDXSDKWORK'))) + +def combine_spdx(d, rootfs_name, rootfs_deploydir, rootfs_spdxid, packages, spdx_workdir): + import os + import oe.spdx + import oe.sbom + import io + import json + from datetime import timezone, datetime + from pathlib import Path + import tarfile + import bb.compress.zstd + + creation_time = datetime.now(tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX")) + source_date_epoch = d.getVar("SOURCE_DATE_EPOCH") + + doc = oe.spdx.SPDXDocument() + doc.name = rootfs_name + doc.documentNamespace = get_doc_namespace(d, doc) + doc.creationInfo.created = creation_time + doc.creationInfo.comment = "This document was created by analyzing the source of the Yocto recipe during the build." + doc.creationInfo.licenseListVersion = d.getVar("SPDX_LICENSE_DATA")["licenseListVersion"] + doc.creationInfo.creators.append("Tool: OpenEmbedded Core create-spdx.bbclass") + doc.creationInfo.creators.append("Organization: %s" % d.getVar("SPDX_ORG")) + doc.creationInfo.creators.append("Person: N/A ()") + + image = oe.spdx.SPDXPackage() + image.name = d.getVar("PN") + image.versionInfo = d.getVar("PV") + image.SPDXID = rootfs_spdxid + image.supplier = d.getVar("SPDX_SUPPLIER") + + doc.packages.append(image) + + for name in sorted(packages.keys()): + pkg_spdx_path = deploy_dir_spdx / "packages" / (name + ".spdx.json") + pkg_doc, pkg_doc_sha1 = oe.sbom.read_doc(pkg_spdx_path) + + for p in pkg_doc.packages: + if p.name == name: + pkg_ref = oe.spdx.SPDXExternalDocumentRef() + pkg_ref.externalDocumentId = "DocumentRef-%s" % pkg_doc.name + pkg_ref.spdxDocument = pkg_doc.documentNamespace + pkg_ref.checksum.algorithm = "SHA1" + pkg_ref.checksum.checksumValue = pkg_doc_sha1 + + doc.externalDocumentRefs.append(pkg_ref) + doc.add_relationship(image, "CONTAINS", "%s:%s" % (pkg_ref.externalDocumentId, p.SPDXID)) + break + else: + bb.fatal("Unable to find package with name '%s' in SPDX file %s" % (name, pkg_spdx_path)) + + runtime_spdx_path = deploy_dir_spdx / "runtime" / ("runtime-" + name + ".spdx.json") + runtime_doc, runtime_doc_sha1 = oe.sbom.read_doc(runtime_spdx_path) + + runtime_ref = oe.spdx.SPDXExternalDocumentRef() + runtime_ref.externalDocumentId = "DocumentRef-%s" % runtime_doc.name + runtime_ref.spdxDocument = runtime_doc.documentNamespace + runtime_ref.checksum.algorithm = "SHA1" + runtime_ref.checksum.checksumValue = runtime_doc_sha1 + + # "OTHER" isn't ideal here, but I can't find a relationship that makes sense + doc.externalDocumentRefs.append(runtime_ref) + doc.add_relationship( + image, + "OTHER", + "%s:%s" % (runtime_ref.externalDocumentId, runtime_doc.SPDXID), + comment="Runtime dependencies for %s" % name + ) + + image_spdx_path = spdx_workdir / (rootfs_name + ".spdx.json") + + with image_spdx_path.open("wb") as f: + doc.to_json(f, sort_keys=True, indent=get_json_indent(d)) + + num_threads = int(d.getVar("BB_NUMBER_THREADS")) + + visited_docs = set() + + index = {"documents": []} + + spdx_tar_path = rootfs_deploydir / (rootfs_name + ".spdx.tar.zst") + with bb.compress.zstd.open(spdx_tar_path, "w", num_threads=num_threads) as f: + with tarfile.open(fileobj=f, mode="w|") as tar: + def collect_spdx_document(path): + nonlocal tar + nonlocal deploy_dir_spdx + nonlocal source_date_epoch + nonlocal index + + if path in visited_docs: + return + + visited_docs.add(path) + + with path.open("rb") as f: + doc, sha1 = oe.sbom.read_doc(f) + f.seek(0) + + if doc.documentNamespace in visited_docs: + return + + bb.note("Adding SPDX document %s" % path) + visited_docs.add(doc.documentNamespace) + info = tar.gettarinfo(fileobj=f) + + info.name = doc.name + ".spdx.json" + info.uid = 0 + info.gid = 0 + info.uname = "root" + info.gname = "root" + + if source_date_epoch is not None and info.mtime > int(source_date_epoch): + info.mtime = int(source_date_epoch) + + tar.addfile(info, f) + + index["documents"].append({ + "filename": info.name, + "documentNamespace": doc.documentNamespace, + "sha1": sha1, + }) + + for ref in doc.externalDocumentRefs: + ref_path = deploy_dir_spdx / "by-namespace" / ref.spdxDocument.replace("/", "_") + collect_spdx_document(ref_path) + + collect_spdx_document(image_spdx_path) + + index["documents"].sort(key=lambda x: x["filename"]) + + index_str = io.BytesIO(json.dumps( + index, + sort_keys=True, + indent=get_json_indent(d), + ).encode("utf-8")) + + info = tarfile.TarInfo() + info.name = "index.json" + info.size = len(index_str.getvalue()) + info.uid = 0 + info.gid = 0 + info.uname = "root" + info.gname = "root" + + tar.addfile(info, fileobj=index_str)
meta/classes/create-spdx.bbclass new file +8 -0
@@ -0,0 +1,8 @@ +# +# Copyright OpenEmbedded Contributors +# +# SPDX-License-Identifier: GPL-2.0-only +# +# Include this class when you don't care what version of SPDX you get; it will +# be updated to the latest stable version that is supported +inherit create-spdx-2.2
meta/files/spdx-licenses.json new file +5937 -0
1,923 lines not on this page · raw message
meta/lib/oe/sbom.py new file +84 -0
84 lines not on this page · raw message
meta/lib/oe/spdx.py new file +357 -0
357 lines not on this page · raw message
The rest of this diff is not shown. View the raw message.
Signature · 1 line
2.33.0
Part of the diff is not shown here. View the raw message.
PATCH v2 2/5 classes/package: Add extended packaged data
In reply to Joshua WattAdds extended package data which is encoded as JSON which allows it to encode more structure than the "flat" package data files. The extended data might be much larger than the standard package data, so it is not read by default and instead requires oe.packagedata.read_subpkgdata_extended() to be called
Currently, the file sizes and ELF debug sources are saved off into the extended package data
(cherry picked from commit db9cf430e54ae68da80fbc3fba80ce88d8df164d)
--- meta/classes/package.bbclass | 40 +++++++++++++++++++++++++++++++++--- meta/lib/oe/packagedata.py | 12 +++++++++++ 2 files changed, 49 insertions(+), 3 deletions(-)
meta/classes/package.bbclass +37 -3
@@ -1140,6 +1140,14 @@ python split_and_strip_files () { # Modified the file so clear the cache cpath.updatecache(file) + def strip_pkgd_prefix(f): + nonlocal dvar + + if f.startswith(dvar): + return f[len(dvar):] + + return f + # # First lets process debug splitting #
@@ -1153,6 +1161,8 @@ python split_and_strip_files () { for file in staticlibs: results.append( (file,source_info(file, d)) ) + d.setVar("PKGDEBUGSOURCES", {strip_pkgd_prefix(f): sorted(s) for f, s in results}) + sources = set() for r in results: sources.update(r[1])
@@ -1460,6 +1470,7 @@ PKGDATA_VARS = "PN PE PV PR PKGE PKGV PKGR LICENSE DESCRIPTION SUMMARY RDEPENDS python emit_pkgdata() { from glob import glob import json + import bb.compress.zstd def process_postinst_on_target(pkg, mlprefix): pkgval = d.getVar('PKG_%s' % pkg)
@@ -1532,6 +1543,8 @@ fi with open(data_file, 'w') as fd: fd.write("PACKAGES: %s\n" % packages) + pkgdebugsource = d.getVar("PKGDEBUGSOURCES") or [] + pn = d.getVar('PN') global_variants = (d.getVar('MULTILIB_GLOBAL_VARIANTS') or "").split() variants = (d.getVar('MULTILIB_VARIANTS') or "").split()
@@ -1551,17 +1564,32 @@ fi pkgval = pkg d.setVar('PKG_%s' % pkg, pkg) + extended_data = { + "files_info": {} + } + pkgdestpkg = os.path.join(pkgdest, pkg) files = {} + files_extra = {} total_size = 0 seen = set() for f in pkgfiles[pkg]: - relpth = os.path.relpath(f, pkgdestpkg) + fpath = os.sep + os.path.relpath(f, pkgdestpkg) + fstat = os.lstat(f) - files[os.sep + relpth] = fstat.st_size + files[fpath] = fstat.st_size + + extended_data["files_info"].setdefault(fpath, {}) + extended_data["files_info"][fpath]['size'] = fstat.st_size + if fstat.st_ino not in seen: seen.add(fstat.st_ino) total_size += fstat.st_size + + if fpath in pkgdebugsource: + extended_data["files_info"][fpath]['debugsrc'] = pkgdebugsource[fpath] + del pkgdebugsource[fpath] + d.setVar('FILES_INFO', json.dumps(files, sort_keys=True)) process_postinst_on_target(pkg, d.getVar("MLPREFIX"))
@@ -1582,6 +1610,11 @@ fi sf.write('%s_%s: %d\n' % ('PKGSIZE', pkg, total_size)) + subdata_extended_file = pkgdatadir + "/extended/%s.json.zstd" % pkg + num_threads = int(d.getVar("BB_NUMBER_THREADS")) + with bb.compress.zstd.open(subdata_extended_file, "wt", encoding="utf-8", num_threads=num_threads) as f: + json.dump(extended_data, f, sort_keys=True, separators=(",", ":")) + # Symlinks needed for rprovides lookup rprov = d.getVar('RPROVIDES_%s' % pkg) or d.getVar('RPROVIDES') if rprov:
@@ -1612,7 +1645,8 @@ fi write_extra_runtime_pkgs(global_variants, packages, pkgdatadir) } -emit_pkgdata[dirs] = "${PKGDESTWORK}/runtime ${PKGDESTWORK}/runtime-reverse ${PKGDESTWORK}/runtime-rprovides" +emit_pkgdata[dirs] = "${PKGDESTWORK}/runtime ${PKGDESTWORK}/runtime-reverse ${PKGDESTWORK}/runtime-rprovides ${PKGDESTWORK}/extended" +emit_pkgdata[vardepsexclude] = "BB_NUMBER_THREADS" ldconfig_postinst_fragment() { if [ x"$D" = "x" ]; then
meta/lib/oe/packagedata.py +12 -0
@@ -57,6 +57,18 @@ def read_subpkgdata_dict(pkg, d): ret[newvar] = subd[var] return ret +def read_subpkgdata_extended(pkg, d): + import json + import bb.compress.zstd + + fn = d.expand("${PKGDATA_DIR}/extended/%s.json.zstd" % pkg) + try: + num_threads = int(d.getVar("BB_NUMBER_THREADS")) + with bb.compress.zstd.open(fn, "rt", encoding="utf-8", num_threads=num_threads) as f: + return json.load(f) + except FileNotFoundError: + return None + def _pkgmap(d): """Return a dictionary mapping package to recipe name."""
Signature · 1 line
2.33.0
PATCH v2 3/5 licenses: Add GPL+ licenses to map
In reply to Joshua WattAdds some missing licenses mappings related to the "+" versions of the GPL licenses
--- meta/conf/licenses.conf | 7 +++++++ 1 file changed, 7 insertions(+)
meta/conf/licenses.conf +7 -0
@@ -22,21 +22,28 @@ SPDXLICENSEMAP[GPLv1.0] = "GPL-1.0" SPDXLICENSEMAP[GPL-1.0-only] = "GPL-1.0" SPDXLICENSEMAP[GPL-2] = "GPL-2.0" SPDXLICENSEMAP[GPLv2] = "GPL-2.0" +SPDXLICENSEMAP[GPLv2+] = "GPL-2.0+" SPDXLICENSEMAP[GPLv2.0] = "GPL-2.0" +SPDXLICENSEMAP[GPLv2.0+] = "GPL-2.0+" SPDXLICENSEMAP[GPL-2.0-only] = "GPL-2.0" SPDXLICENSEMAP[GPL-3] = "GPL-3.0" SPDXLICENSEMAP[GPLv3] = "GPL-3.0" +SPDXLICENSEMAP[GPLv3+] = "GPL-3.0+" SPDXLICENSEMAP[GPLv3.0] = "GPL-3.0" +SPDXLICENSEMAP[GPLv3.0+] = "GPL-3.0+" SPDXLICENSEMAP[GPL-3.0-only] = "GPL-3.0" #LGPL variations SPDXLICENSEMAP[LGPLv2] = "LGPL-2.0" +SPDXLICENSEMAP[LGPLv2+] = "LGPL-2.0+" SPDXLICENSEMAP[LGPLv2.0] = "LGPL-2.0" SPDXLICENSEMAP[LGPL-2.0-only] = "LGPL-2.0" SPDXLICENSEMAP[LGPL2.1] = "LGPL-2.1" SPDXLICENSEMAP[LGPLv2.1] = "LGPL-2.1" +SPDXLICENSEMAP[LGPLv2.1+] = "LGPL-2.1+" SPDXLICENSEMAP[LGPL-2.1-only] = "LGPL-2.1" SPDXLICENSEMAP[LGPLv3] = "LGPL-3.0" +SPDXLICENSEMAP[LGPLv3+] = "LGPL-3.0+" SPDXLICENSEMAP[LGPL-3.0-only] = "LGPL-3.0" #MPL variations
Signature · 1 line
2.33.0
PATCH v2 4/5 create-spdx: Use gzip for compression
In reply to Joshua WattThe master version of the SPDX classes uses zstd for efficient compression, but it relies on the zstd tool to be present on the host system. Since dunfell supports older distros, we don't want to add this tool as an additional requirement so switch to using gzip instead.
--- meta/classes/create-spdx-2.2.bbclass | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-)
meta/classes/create-spdx-2.2.bbclass +9 -11
@@ -466,13 +466,11 @@ python do_create_spdx() { @contextmanager def optional_tarfile(name, guard, mode="w"): import tarfile - import bb.compress.zstd - - num_threads = int(d.getVar("BB_NUMBER_THREADS")) + import gzip if guard: name.parent.mkdir(parents=True, exist_ok=True) - with bb.compress.zstd.open(name, mode=mode + "b", num_threads=num_threads) as f: + with gzip.open(name, mode=mode + "b") as f: with tarfile.open(fileobj=f, mode=mode + "|") as tf: yield tf else:
@@ -550,7 +548,7 @@ python do_create_spdx() { add_download_packages(d, doc, recipe) if process_sources(d) and include_sources: - recipe_archive = deploy_dir_spdx / "recipes" / (doc.name + ".tar.zst") + recipe_archive = deploy_dir_spdx / "recipes" / (doc.name + ".tar.gz") with optional_tarfile(recipe_archive, archive_sources) as archive: spdx_get_src(d)
@@ -618,7 +616,7 @@ python do_create_spdx() { package_doc.add_relationship(spdx_package, "GENERATED_FROM", "%s:%s" % (recipe_ref.externalDocumentId, recipe.SPDXID)) package_doc.add_relationship(package_doc, "DESCRIBES", spdx_package) - package_archive = deploy_dir_spdx / "packages" / (package_doc.name + ".tar.zst") + package_archive = deploy_dir_spdx / "packages" / (package_doc.name + ".tar.gz") with optional_tarfile(package_archive, archive_packaged) as archive: package_files = add_package_files( d,
@@ -899,8 +897,8 @@ python image_combine_spdx() { if link != target_path: link.symlink_to(os.path.relpath(target_path, link.parent)) - spdx_tar_path = imgdeploydir / (image_name + ".spdx.tar.zst") - make_image_link(spdx_tar_path, ".spdx.tar.zst") + spdx_tar_path = imgdeploydir / (image_name + ".spdx.tar.gz") + make_image_link(spdx_tar_path, ".spdx.tar.gz") } python sdk_host_combine_spdx() {
@@ -931,7 +929,7 @@ def combine_spdx(d, rootfs_name, rootfs_deploydir, rootfs_spdxid, packages, spdx from datetime import timezone, datetime from pathlib import Path import tarfile - import bb.compress.zstd + import gzip creation_time = datetime.now(tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX"))
@@ -1002,8 +1000,8 @@ def combine_spdx(d, rootfs_name, rootfs_deploydir, rootfs_spdxid, packages, spdx index = {"documents": []} - spdx_tar_path = rootfs_deploydir / (rootfs_name + ".spdx.tar.zst") - with bb.compress.zstd.open(spdx_tar_path, "w", num_threads=num_threads) as f: + spdx_tar_path = rootfs_deploydir / (rootfs_name + ".spdx.tar.gz") + with gzip.open(spdx_tar_path, "w") as f: with tarfile.open(fileobj=f, mode="w|") as tar: def collect_spdx_document(path): nonlocal tar
Signature · 1 line
2.33.0
PATCH v2 5/5 classes/package: Use gzip for extended package data
In reply to Joshua WattThe master version of extended package data uses zstd for efficient compression, but it relies on the zstd tool to be present on the host system. Since dunfell supports older distros, we don't want to add this tool as an additional requirement so switch to using gzip instead.
--- meta/classes/package.bbclass | 7 +++---- meta/lib/oe/packagedata.py | 7 +++---- 2 files changed, 6 insertions(+), 8 deletions(-)
meta/classes/package.bbclass +3 -4
@@ -1470,7 +1470,7 @@ PKGDATA_VARS = "PN PE PV PR PKGE PKGV PKGR LICENSE DESCRIPTION SUMMARY RDEPENDS python emit_pkgdata() { from glob import glob import json - import bb.compress.zstd + import gzip def process_postinst_on_target(pkg, mlprefix): pkgval = d.getVar('PKG_%s' % pkg)
@@ -1610,9 +1610,8 @@ fi sf.write('%s_%s: %d\n' % ('PKGSIZE', pkg, total_size)) - subdata_extended_file = pkgdatadir + "/extended/%s.json.zstd" % pkg - num_threads = int(d.getVar("BB_NUMBER_THREADS")) - with bb.compress.zstd.open(subdata_extended_file, "wt", encoding="utf-8", num_threads=num_threads) as f: + subdata_extended_file = pkgdatadir + "/extended/%s.json.gz" % pkg + with gzip.open(subdata_extended_file, "wt", encoding="utf-8") as f: json.dump(extended_data, f, sort_keys=True, separators=(",", ":")) # Symlinks needed for rprovides lookup
meta/lib/oe/packagedata.py +3 -4
@@ -59,12 +59,11 @@ def read_subpkgdata_dict(pkg, d): def read_subpkgdata_extended(pkg, d): import json - import bb.compress.zstd + import gzip - fn = d.expand("${PKGDATA_DIR}/extended/%s.json.zstd" % pkg) + fn = d.expand("${PKGDATA_DIR}/extended/%s.json.gz" % pkg) try: - num_threads = int(d.getVar("BB_NUMBER_THREADS")) - with bb.compress.zstd.open(fn, "rt", encoding="utf-8", num_threads=num_threads) as f: + with gzip.open(fn, "rt", encoding="utf-8") as f: return json.load(f) except FileNotFoundError: return None
Signature · 1 line
2.33.0