Check compiled files to filter kernel CVEs
v6 cover Check compiled files to filter kernel CVEs
From: Daniel Turull <[email protected]>
Since kernel.org became a CNA, more information is available in the published CVEs, including details about which files are affected by a given CVE.
I have rewritten the original patch after the feedback received, including only the basic functionality in the build (extracting the sources as a text file and inside spdx) and created a postprocessing script that enrich the cve-summary.
To filter out CVEs that are not applicable, we extract the files used during the kernel compilation and compare it with the metadata in the CVE.
To enabled, add in your local.conf SPDX_INCLUDE_COMPILED_SOURCES. It will generated spdx files with only the used sources. If only the kernel is wanted use in your local.conf: SPDX_INCLUDE_COMPILED_SOURCES:pn-linux-yocto = "1".
This could use as a base to run the vulnerability check independently and run an external tool to filter the CVEs.
v1: initial proposal
v2:
- rewrite kernel_vulns to fetch similarly as cve-update-db-native
- add functionality into cve_check.py, for the classes that uses oe.get_patched_cves function
- add linux-vulns into the cve-check results
- add only compiled files in the spdx, so the check can be done outside the build
- include compiled files into spdx when CVE_CHECK_KERNEL_CONFIG and SPDX_INCLUDE_SOURCES is enabled
v3:
- make inclusion of compiled files generic for SPDX, so other systems that has knowledge of used files can also make more accurate sboms
- have the functions to extract files in kernel.bbclass. For other recipes that in the future want to use this feature can add the function in their recipe or in a build bbclass.
- move order of patches
- explicitly have the save_compiled_files added only when having the CVE_CHECK_KERNEL_CONFIG
- add first kernel cves in cve_check, so manual CVE_STATUS is preserved
- add CVE_STATUS for false positives
v4:
- Refactor and reduce series to 3 patches, one for spdx, one for the kernel, and one standalone script
v5:
- Use debug information from do_package to extract compiled files
- Read spdx file from the kernel in the script to process files
v6:
- Export debug sources as compressed json in pkgdata
- CVE script is kept intact
Daniel Turull
Daniel Turull (3):
package: export debugsources in PKGDESTWORK as json
spdx: add option to include only compiled sources
improve_kernel_cve_report: add script for postprocesing of kernel CVE
data
meta/classes/create-spdx-2.2.bbclass | 9 + meta/classes/spdx-common.bbclass | 3 + meta/lib/oe/package.py | 46 ++ meta/lib/oe/spdx30_tasks.py | 10 + meta/lib/oe/spdx_common.py | 41 ++ scripts/contrib/improve_kernel_cve_report.py | 467 +++++++++++++++++++ 6 files changed, 576 insertions(+) create mode 100755 scripts/contrib/improve_kernel_cve_report.py
PATCH v6 1/3 package: export debugsources in PKGDESTWORK as json
From: Daniel Turull <[email protected]>
The source information used during packaging can be use from other tasks to have more detailed information on the files used during the compilation and improve SPDX accuracy.
Source files used during compilation are store as compressed zstd json in pkgdata/debugsources/$PN-debugsources.json.zstd
Format:
{ binary1: [src1, src2, ...], binary2: [src1, src2, ...] }
I checked the sstate size, and it slightly increase using core-image-full-cmdline:
Before: 2454884 B
After: 2456860 B (+1976 B or 0,08%)
--- meta/lib/oe/package.py | 46 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+)
meta/lib/oe/package.py +46 -0
@@ -1049,6 +1049,49 @@ def copydebugsources(debugsrcdir, sources, d): if os.path.exists(p) and not os.listdir(p): os.rmdir(p) +def save_debugsources_info(debugsrcdir, sources_raw, d): + import json + import bb.compress.zstd + if debugsrcdir and sources_raw: + debugsources_file = d.expand("${PKGDESTWORK}/debugsources/${PN}-debugsources.json.zstd") + debugsources_dir = os.path.dirname(debugsources_file) + if not os.path.isdir(debugsources_dir): + bb.utils.mkdirhier(debugsources_dir) + bb.utils.remove(debugsources_file) + + workdir = d.getVar("WORKDIR") + sdir = d.getVar("S") + pn = d.getVar('PN') + pv = d.getVar('PV') + sources_dict = {} + for file, src_files in sources_raw: + file_clean = file.replace(f"{workdir}/package/","") + sources_clean = [ + src.replace(f"/usr/src/debug/{pn}/", "") + .replace(f"{sdir}/", "") + .replace(f"/usr/src/kernel/", "") + .replace(f"/usr/src/{pn}/", "") + .replace(f"{pn}-{pv}/", "") + .replace(f"{pv}/", "") + for src in src_files + if not any(keyword in src for keyword in ("<internal>", "<built-in>")) and not src.endswith("/") + ] + sources_dict[file_clean] = sorted(sources_clean) + num_threads = int(d.getVar("BB_NUMBER_THREADS")) + with bb.compress.zstd.open(debugsources_file, "wt", encoding="utf-8", num_threads=num_threads) as f: + json.dump(sources_dict, f, sort_keys=True) + +def read_debugsources_file(d): + import json + import bb.compress.zstd + try: + fn = d.expand("${PKGDESTWORK}/debugsources/${PN}-debugsources.json.zstd") + num_threads = int(d.getVar("BB_NUMBER_THREADS")) + with bb.compress.zstd.open(fn, "rt", encoding="utf-8", num_threads=num_threads) as f: + return json.load(f) + except FileNotFoundError: + bb.debug(1, f"File not found: {fn}") + return None def process_split_and_strip_files(d): cpath = oe.cachedpath.CachedPath()
@@ -1280,6 +1323,9 @@ def process_split_and_strip_files(d): # Process the dv["srcdir"] if requested... # This copies and places the referenced sources for later debugging... copydebugsources(dv["srcdir"], sources, d) + + # Save source info to be accessible to other tasks + save_debugsources_info(dv["srcdir"], results, d) # # End of debug splitting #
Re: [PATCH v6 1/3] package: export debugsources in PKGDESTWORK as json
Quoted 76 lines · On Wed, 2025-06-04 at 13:21 +0200, [email protected] wrote:
From: Daniel Turull <[email protected]>
The source information used during packaging can be use from other tasks to have more detailed information on the files used during the compilation and improve SPDX accuracy.
Source files used during compilation are store as compressed zstd json in pkgdata/debugsources/$PN-debugsources.json.zstd
Format:
{ binary1: [src1, src2, ...], binary2: [src1, src2, ...] }I checked the sstate size, and it slightly increase using core-image-full-cmdline:
Before: 2454884 B
After: 2456860 B (+1976 B or 0,08%)CC Richard Purdie <[email protected]>Signed-off-by Daniel Turull <[email protected]>---
meta/lib/oe/package.py | 46 ++++++++++++++++++++++++++++++++++++++++++1 file changed, 46 insertions(+)meta/lib/oe/package.py +0
-0@@ -1049,6 +1049,49 @@ def copydebugsources(debugsrcdir, sources, d):if
os.path.exists(p)and notos.listdir(p):
os.rmdir(p)meta/lib/oe/package.py +43
-0+def save_debugsources_info(debugsrcdir, sources_raw, d): + import json + import bb.compress.zstd + if debugsrcdir and sources_raw: + debugsources_file = d.expand("${PKGDESTWORK}/debugsources/${PN}-debugsources.json.zstd") + debugsources_dir = os.path.dirname(debugsources_file) + if not os.path.isdir(debugsources_dir): + bb.utils.mkdirhier(debugsources_dir) + bb.utils.remove(debugsources_file) + + workdir = d.getVar("WORKDIR") + sdir = d.getVar("S") + pn = d.getVar('PN') + pv = d.getVar('PV') + sources_dict = {} + for file, src_files in sources_raw: + file_clean = file.replace(f"{workdir}/package/","") + sources_clean = [ + src.replace(f"/usr/src/debug/{pn}/", "") + .replace(f"{sdir}/", "") + .replace(f"/usr/src/kernel/", "") + .replace(f"/usr/src/{pn}/", "") + .replace(f"{pn}-{pv}/", "") + .replace(f"{pv}/", "") + for src in src_files + if not any(keyword in src for keyword in ("<internal>", "<built-in>")) and not src.endswith("/") + ] + sources_dict[file_clean] = sorted(sources_clean) + num_threads = int(d.getVar("BB_NUMBER_THREADS")) + with bb.compress.zstd.open(debugsources_file, "wt", encoding="utf-8", num_threads=num_threads) as f: + json.dump(sources_dict, f, sort_keys=True) + +def read_debugsources_file(d): + import json + import bb.compress.zstd + try: + fn = d.expand("${PKGDESTWORK}/debugsources/${PN}-debugsources.json.zstd") + num_threads = int(d.getVar("BB_NUMBER_THREADS")) + with bb.compress.zstd.open(fn, "rt", encoding="utf-8", num_threads=num_threads) as f: + return json.load(f) + except FileNotFoundError: + bb.debug(1, f"File not found: {fn}") + return None
defprocess_split_and_strip_files(d):
cpath =oe.cachedpath.CachedPath()
meta/lib/oe/package.py def process_split_and_strip_files(d): · line 1331
Quoted 9 lines
meta/lib/oe/package.py +3
-0@@ -1280,6 +1323,9 @@ def process_split_and_strip_files(d): # Process the dv["srcdir"] if requested... # This copies and places the referenced sources for later debugging... copydebugsources(dv["srcdir"], sources, d) + + # Save source info to be accessible to other tasks + save_debugsources_info(dv["srcdir"], results, d) # # End of debug splitting #
This looks good, as I as about to merge it, I realised there is one issue. bitbake.conf has these entries:
meta/conf/bitbake.conf:oe.packagedata.emit_pkgdata[vardepsexclude] = "BB_NUMBER_THREADS" meta/conf/bitbake.conf:oe.packagedata.read_subpkgdata_extended[vardepsexclude] = "BB_NUMBER_THREADS"
and I think you need to replicate these for your new functions. The reason is to stop sstate being dependency on the number of threads used.
We have had work in process to move these definitions alongside the function declarations but I'm not sure we've solved that yet so adding to bitbake.conf is the easiest thing to do for now.
Cheers,
Richard
Re: [PATCH v6 1/3] package: export debugsources in PKGDESTWORK as json
Thanks, I’ll do that and resubmit with the changes. Should I resubmit the whole series or only this patch? Cheers,
Daniel
________________________________
From: Richard Purdie <[email protected]>
Sent: Thursday, June 5, 2025 12:27:35 PM
To: Daniel Turull <[email protected]>; [email protected] <[email protected]>
Cc: Joshua Watt <[email protected]>
Subject: Re: [PATCH v6 1/3] package: export debugsources in PKGDESTWORK as json
Quoted 76 lines · On Wed, 2025-06-04 at 13:21 +0200, [email protected] wrote:
From: Daniel Turull <[email protected]>
The source information used during packaging can be use from other tasks to have more detailed information on the files used during the compilation and improve SPDX accuracy.
Source files used during compilation are store as compressed zstd json in pkgdata/debugsources/$PN-debugsources.json.zstd
Format:
{ binary1: [src1, src2, ...], binary2: [src1, src2, ...] }I checked the sstate size, and it slightly increase using core-image-full-cmdline:
Before: 2454884 B
After: 2456860 B (+1976 B or 0,08%)CC Richard Purdie <[email protected]>Signed-off-by Daniel Turull <[email protected]>--- meta/lib/oe/package.py | 46 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+)meta/lib/oe/package.py +43
-0@@ -1049,6 +1049,49 @@ def copydebugsources(debugsrcdir, sources, d): if os.path.exists(p) and not os.listdir(p): os.rmdir(p) +def save_debugsources_info(debugsrcdir, sources_raw, d): + import json + import bb.compress.zstd + if debugsrcdir and sources_raw: + debugsources_file = d.expand("${PKGDESTWORK}/debugsources/${PN}-debugsources.json.zstd") + debugsources_dir = os.path.dirname(debugsources_file) + if not os.path.isdir(debugsources_dir): + bb.utils.mkdirhier(debugsources_dir) + bb.utils.remove(debugsources_file) + + workdir = d.getVar("WORKDIR") + sdir = d.getVar("S") + pn = d.getVar('PN') + pv = d.getVar('PV') + sources_dict = {} + for file, src_files in sources_raw: + file_clean = file.replace(f"{workdir}/package/","") + sources_clean = [ + src.replace(f"/usr/src/debug/{pn}/", "") + .replace(f"{sdir}/", "") + .replace(f"/usr/src/kernel/", "") + .replace(f"/usr/src/{pn}/", "") + .replace(f"{pn}-{pv}/", "") + .replace(f"{pv}/", "") + for src in src_files + if not any(keyword in src for keyword in ("<internal>", "<built-in>")) and not src.endswith("/") + ] + sources_dict[file_clean] = sorted(sources_clean) + num_threads = int(d.getVar("BB_NUMBER_THREADS")) + with bb.compress.zstd.open(debugsources_file, "wt", encoding="utf-8", num_threads=num_threads) as f: + json.dump(sources_dict, f, sort_keys=True) + +def read_debugsources_file(d): + import json + import bb.compress.zstd + try: + fn = d.expand("${PKGDESTWORK}/debugsources/${PN}-debugsources.json.zstd") + num_threads = int(d.getVar("BB_NUMBER_THREADS")) + with bb.compress.zstd.open(fn, "rt", encoding="utf-8", num_threads=num_threads) as f: + return json.load(f) + except FileNotFoundError: + bb.debug(1, f"File not found: {fn}") + return None def process_split_and_strip_files(d): cpath = oe.cachedpath.CachedPath()
meta/lib/oe/package.py def process_split_and_strip_files(d): · line 1331
Quoted 9 lines
meta/lib/oe/package.py +3
-0@@ -1280,6 +1323,9 @@ def process_split_and_strip_files(d): # Process the dv["srcdir"] if requested... # This copies and places the referenced sources for later debugging... copydebugsources(dv["srcdir"], sources, d) + + # Save source info to be accessible to other tasks + save_debugsources_info(dv["srcdir"], results, d) # # End of debug splitting #
This looks good, as I as about to merge it, I realised there is one issue. bitbake.conf has these entries:
meta/conf/bitbake.conf:oe.packagedata.emit_pkgdata[vardepsexclude] = "BB_NUMBER_THREADS" meta/conf/bitbake.conf:oe.packagedata.read_subpkgdata_extended[vardepsexclude] = "BB_NUMBER_THREADS"
and I think you need to replicate these for your new functions. The reason is to stop sstate being dependency on the number of threads used.
We have had work in process to move these definitions alongside the function declarations but I'm not sure we've solved that yet so adding to bitbake.conf is the easiest thing to do for now.
Cheers,
Richard
Re: [PATCH v6 1/3] package: export debugsources in PKGDESTWORK as json
Quoted 2 lines · Daniel Turull
Thanks, I’ll do that and resubmit with the changes. Should I resubmit the whole series or only this patch?
Just this one is fine. I'm trying to get this sorted, then we can focus on the others. Joshua is the person I'll look to on the SPDX piece, then the wider community on the script.
Cheers,
Richard
PATCH v6 2/3 spdx: add option to include only compiled sources
In reply to [email protected]From: Daniel Turull <[email protected]>
When SPDX_INCLUDE_COMPILED_SOURCES is enabled, only include the source code files that are used during compilation.
It uses debugsource information generated during do_package.
This enables an external tool to use the SPDX information to disregard vulnerabilities that are not compiled.
As example, when used with the default config with linux-yocto, the spdx size is reduced from 156MB to 61MB.
Tested with bitbake world on oe-core.
--- meta/classes/create-spdx-2.2.bbclass | 9 ++++++ meta/classes/spdx-common.bbclass | 3 ++ meta/lib/oe/spdx30_tasks.py | 10 +++++++ meta/lib/oe/spdx_common.py | 41 ++++++++++++++++++++++++++++ 4 files changed, 63 insertions(+)
meta/classes/create-spdx-2.2.bbclass +9 -0
@@ -137,6 +137,11 @@ def add_package_files(d, doc, spdx_pkg, topdir, get_spdxid, get_types, *, archiv spdx_files = [] file_counter = 1 + + check_compiled_sources = d.getVar("SPDX_INCLUDE_COMPILED_SOURCES") == "1" + if check_compiled_sources: + compiled_sources, types = oe.spdx_common.get_compiled_sources(d) + bb.debug(1, f"Total compiled files: {len(compiled_sources)}") for subdir, dirs, files in os.walk(topdir): dirs[:] = [d for d in dirs if d not in ignore_dirs] if subdir == str(topdir):
@@ -147,6 +152,10 @@ def add_package_files(d, doc, spdx_pkg, topdir, get_spdxid, get_types, *, archiv filename = str(filepath.relative_to(topdir)) if not filepath.is_symlink() and filepath.is_file(): + # Check if file is compiled + if check_compiled_sources: + if not oe.spdx_common.is_compiled_source(filename, compiled_sources, types): + continue spdx_file = oe.spdx.SPDXFile() spdx_file.SPDXID = get_spdxid(file_counter) for t in get_types(filepath):
meta/classes/spdx-common.bbclass +3 -0
@@ -26,6 +26,7 @@ SPDX_TOOL_VERSION ??= "1.0" SPDXRUNTIMEDEPLOY = "${SPDXDIR}/runtime-deploy" SPDX_INCLUDE_SOURCES ??= "0" +SPDX_INCLUDE_COMPILED_SOURCES ??= "0" SPDX_UUID_NAMESPACE ??= "sbom.openembedded.org" SPDX_NAMESPACE_PREFIX ??= "http://spdx.org/spdxdocs"
@@ -40,6 +41,8 @@ SPDX_MULTILIB_SSTATE_ARCHS ??= "${SSTATE_ARCHS}" python () { from oe.cve_check import extend_cve_status extend_cve_status(d) + if d.getVar("SPDX_INCLUDE_COMPILED_SOURCES") == "1": + d.setVar("SPDX_INCLUDE_SOURCES", "1") } def create_spdx_source_deps(d):
meta/lib/oe/spdx30_tasks.py +10 -0
@@ -156,6 +156,11 @@ def add_package_files( bb.note(f"Skip {topdir}") return spdx_files + check_compiled_sources = d.getVar("SPDX_INCLUDE_COMPILED_SOURCES") == "1" + if check_compiled_sources: + compiled_sources, types = oe.spdx_common.get_compiled_sources(d) + bb.debug(1, f"Total compiled files: {len(compiled_sources)}") + for subdir, dirs, files in os.walk(topdir, onerror=walk_error): dirs[:] = [d for d in dirs if d not in ignore_dirs] if subdir == str(topdir):
@@ -171,6 +176,11 @@ def add_package_files( filename = str(filepath.relative_to(topdir)) file_purposes = get_purposes(filepath) + # Check if file is compiled + if check_compiled_sources: + if not oe.spdx_common.is_compiled_source(filename, compiled_sources, types): + continue + spdx_file = objset.new_file( get_spdxid(file_counter), filename,
meta/lib/oe/spdx_common.py +41 -0
44 lines not on this page · raw message
Re: [PATCH v6 2/3] spdx: add option to include only compiled sources
Quoted 109 lines · On Wed, Jun 4, 2025 at 5:21 AM <[email protected]> wrote:
From: Daniel Turull <[email protected]>
When
SPDX_INCLUDE_COMPILED_SOURCESis enabled, only include the source code files that are used during compilation.It uses debugsource information generated during
do_package.This enables an external tool to use the SPDX information to disregard vulnerabilities that are not compiled.
As example, when used with the default config with linux-yocto, the spdx size is reduced from 156MB to 61MB.
Tested with bitbake world on oe-core.
CC Quentin Schulz <[email protected]>CC Joshua Watt <[email protected]>CC Peter Marko <[email protected]>Signed-off-by Daniel Turull <[email protected]>--- meta/classes/create-spdx-2.2.bbclass | 9 ++++++ meta/classes/spdx-common.bbclass | 3 ++ meta/lib/oe/spdx30_tasks.py | 10 +++++++ meta/lib/oe/spdx_common.py | 41 ++++++++++++++++++++++++++++ 4 files changed, 63 insertions(+)meta/classes/create-spdx-2.2.bbclass +9
-0@@ -137,6 +137,11 @@ def add_package_files(d, doc, spdx_pkg, topdir, get_spdxid, get_types, *, archiv spdx_files = [] file_counter = 1 + + check_compiled_sources = d.getVar("SPDX_INCLUDE_COMPILED_SOURCES") == "1" + if check_compiled_sources: + compiled_sources, types = oe.spdx_common.get_compiled_sources(d) + bb.debug(1, f"Total compiled files: {len(compiled_sources)}") for subdir, dirs, files in os.walk(topdir): dirs[:] = [d for d in dirs if d not in ignore_dirs] if subdir == str(topdir):@@ -147,6 +152,10 @@ def add_package_files(d, doc, spdx_pkg, topdir, get_spdxid, get_types, *, archiv filename = str(filepath.relative_to(topdir)) if not filepath.is_symlink() and filepath.is_file(): + # Check if file is compiled + if check_compiled_sources: + if not oe.spdx_common.is_compiled_source(filename, compiled_sources, types): + continue spdx_file = oe.spdx.SPDXFile() spdx_file.SPDXID = get_spdxid(file_counter) for t in get_types(filepath):meta/classes/spdx-common.bbclass +3
-0@@ -26,6 +26,7 @@ SPDX_TOOL_VERSION ??= "1.0" SPDXRUNTIMEDEPLOY = "${SPDXDIR}/runtime-deploy" SPDX_INCLUDE_SOURCES ??= "0" +SPDX_INCLUDE_COMPILED_SOURCES ??= "0" SPDX_UUID_NAMESPACE ??= "sbom.openembedded.org" SPDX_NAMESPACE_PREFIX ??= "http://spdx.org/spdxdocs"@@ -40,6 +41,8 @@ SPDX_MULTILIB_SSTATE_ARCHS ??= "${SSTATE_ARCHS}" python () { from oe.cve_check import extend_cve_status extend_cve_status(d) + if d.getVar("SPDX_INCLUDE_COMPILED_SOURCES") == "1": + d.setVar("SPDX_INCLUDE_SOURCES", "1") } def create_spdx_source_deps(d):meta/lib/oe/spdx30_tasks.py +10
-0@@ -156,6 +156,11 @@ def add_package_files( bb.note(f"Skip {topdir}") return spdx_files + check_compiled_sources = d.getVar("SPDX_INCLUDE_COMPILED_SOURCES") == "1" + if check_compiled_sources: + compiled_sources, types = oe.spdx_common.get_compiled_sources(d) + bb.debug(1, f"Total compiled files: {len(compiled_sources)}") + for subdir, dirs, files in os.walk(topdir, onerror=walk_error): dirs[:] = [d for d in dirs if d not in ignore_dirs] if subdir == str(topdir):@@ -171,6 +176,11 @@ def add_package_files( filename = str(filepath.relative_to(topdir)) file_purposes = get_purposes(filepath) + # Check if file is compiled + if check_compiled_sources: + if not oe.spdx_common.is_compiled_source(filename, compiled_sources, types): + continue + spdx_file = objset.new_file( get_spdxid(file_counter), filename,meta/lib/oe/spdx_common.py +0
-0@@ -242,3 +242,44 @@ def fetch_data_to_uri(fd, name): uri = uri + "@" + fd.revision
meta/lib/oe/spdx_common.py def fetch_data_to_uri(fd, name): · line 252
Quoted 10 lines
meta/lib/oe/spdx_common.py +8
-0return uri + +def is_compiled_source (filename, compiled_sources, types): + """ + Check if the file is a compiled file + """ + import os + # If we don't have compiled source, we assume all are compiled. + if len(compiled_sources) == 0:
idiomatically, this would be:
if not compiled_sources:
return True
meta/lib/oe/spdx_common.py def fetch_data_to_uri(fd, name): · line 254
Quoted 2 lines
meta/lib/oe/spdx_common.py +2
-0+ return True + # We remove the top directory, to match the format in compiled sources
This feels a little bit magic, can you explain why removing the top directory is necessary and or correct to always perform?
meta/lib/oe/spdx_common.py def fetch_data_to_uri(fd, name): · line 255
Quoted 1 line
meta/lib/oe/spdx_common.py +1
-0+ relative = filename[filename.find("/")+1:]
you use .find() a lot, but I think split() is more idiomatic:
_, relative = filename.split("/", 1)
meta/lib/oe/spdx_common.py def fetch_data_to_uri(fd, name): · line 257
Quoted 2 lines
meta/lib/oe/spdx_common.py +2
-0+ basename = os.path.basename(filename) + # We return always true if the file type is not in the list of compiled files
Why is this?
meta/lib/oe/spdx_common.py def fetch_data_to_uri(fd, name): · line 258
Quoted 1 line
meta/lib/oe/spdx_common.py +1
-0+ if basename[basename.find("."):] not in types:
if "." not in basename or basename.split(".", 1)[1] not in types:
return True
Quoted 6 lines
meta/lib/oe/spdx_common.py +6
-0+ return True + # Check that the file is in the list + return relative in compiled_sources + +def get_compiled_sources(d): + """
meta/lib/oe/spdx_common.py def fetch_data_to_uri(fd, name): · line 274
Quoted 10 lines
meta/lib/oe/spdx_common.py +10
-0+ Get list of compiled sources from debug information and normalize the paths + """ + import itertools + source_info = oe.package.read_debugsources_file(d) + if not source_info: + bb.debug(1, "Do not have debugsources.list. Skipping") + return [], [] + + # Sources are not split now in SPDX, so we aggregate them + sources = list(set(itertools.chain.from_iterable(source_info.values())))
sources should be a set, since you are using the "in" operator, which is much for efficient for sets than lists (especially when they are large)
meta/lib/oe/spdx_common.py def fetch_data_to_uri(fd, name): · line 281
Quoted 7 lines
meta/lib/oe/spdx_common.py +7
-0+ # Check extensions of files + types = [] + for src in sources: + basename = os.path.basename(src) + # We check that the basename has an extension + if basename.find(".") > 0: + ext = basename[basename.find("."):]
Similarly:
if "." in basename:
stem, ext = basename.split(".", 1)meta/lib/oe/spdx_common.py def fetch_data_to_uri(fd, name): · line 283
Quoted 2 lines
meta/lib/oe/spdx_common.py +2
-0+ if ext not in types and len(ext)>0: + types.append(ext)
Making types a set would be more efficient, e.g.
types = set()
...
if ext:
types.add(ext)
The "in" operator is much more efficient for sets than lists
Quoted 2 lines
meta/lib/oe/spdx_common.py +2
-0+ bb.debug(1, f"Num of sources: {len(sources)} and types: {len(types)} {str(types)}") + return sources, types
RE: [PATCH v6 2/3] spdx: add option to include only compiled sources
Thanks Joshua for the comments. I'll resend the patch with the fixes after I have verified that it works with a world build.
Quoted 150 lines
-----Original Message-----
From: Joshua Watt <[email protected]>
Sent: Thursday, 5 June 2025 17:15
To: Daniel Turull <[email protected]>
Cc: [email protected]; Quentin Schulz <[email protected]>; Peter Marko <[email protected]>
Subject: Re: [PATCH v6 2/3] spdx: add option to include only compiled sourcesQuoted 4 lines · On Wed, Jun 4, 2025 at 5:21 AM <[email protected]> wrote:
From: Daniel Turull <[email protected]>
When
SPDX_INCLUDE_COMPILED_SOURCESis enabled, only include thesource
meta/classes/create-spdx-2.2.bbclass def add_package_files(d, doc, spdx_pkg, topdir,
Quoted 29 lines
code files that are used during compilation.
It uses debugsource information generated during
do_package.This enables an external tool to use the SPDX information to disregard vulnerabilities that are not compiled.
As example, when used with the default config with linux-yocto, the spdx size is reduced from 156MB to 61MB.
Tested with bitbake world on oe-core.
CC Quentin Schulz <[email protected]>CC Joshua Watt <[email protected]>CC Peter Marko <[email protected]>Signed-off-by Daniel Turull <[email protected]>--- meta/classes/create-spdx-2.2.bbclass | 9 ++++++ meta/classes/spdx-common.bbclass | 3 ++ meta/lib/oe/spdx30_tasks.py | 10 +++++++ meta/lib/oe/spdx_common.py | 41 ++++++++++++++++++++++++++++ 4 files changed, 63 insertions(+)meta/classes/create-spdx-2.2.bbclass +0
-0@@ -137,6 +137,11 @@ def add_package_files(d, doc, spdx_pkg, topdir,
get_spdxid,get_types, *, archivmeta/classes/create-spdx-2.2.bbclass def add_package_files(d, doc, spdx_pkg, topdir, get_spdxid, get_types, *, archiv
Quoted 5 lines
meta/classes/create-spdx-2.2.bbclass +2
-0spdx_files = [] file_counter = 1 + + check_compiled_sources =
d.getVar("SPDX_INCLUDE_COMPILED_SOURCES")== "1"meta/classes/create-spdx-2.2.bbclass def add_package_files(d, doc, spdx_pkg, topdir, · line 143
Quoted 2 lines
meta/classes/create-spdx-2.2.bbclass +2
-0+ if check_compiled_sources: + compiled_sources, types =
oe.spdx_common.get_compiled_sources(d)meta/classes/create-spdx-2.2.bbclass def add_package_files(d, doc, spdx_pkg, topdir,
Quoted 5 lines
meta/classes/create-spdx-2.2.bbclass +1
-0+ bb.debug(1, f"Total compiled files: {len(compiled_sources)}") for subdir, dirs, files in os.walk(topdir): dirs[:] = [d for d in dirs if d not in ignore_dirs] if subdir == str(topdir):@@ -147,6 +152,10 @@ def add_package_files(d, doc, spdx_pkg, topdir,
get_spdxid,get_types, *, archivmeta/classes/create-spdx-2.2.bbclass def add_package_files(d, doc, spdx_pkg, topdir, get_spdxid, get_types, *, archiv
Quoted 6 lines
meta/classes/create-spdx-2.2.bbclass +3
-0filename = str(filepath.relative_to(topdir)) if not filepath.is_symlink() and filepath.is_file(): + # Check if file is compiled + if check_compiled_sources: + if not oe.spdx_common.is_compiled_source(filename,
compiled_sources, types):Quoted 10 lines
meta/classes/create-spdx-2.2.bbclass +1
-0+ continue spdx_file = oe.spdx.SPDXFile() spdx_file.SPDXID = get_spdxid(file_counter) for t in get_types(filepath):meta/classes/spdx-common.bbclass +0
-0meta/classes/spdx-common.bbclass SPDX_TOOL_VERSION ??= "1.0" · line 32
Quoted 7 lines
meta/classes/spdx-common.bbclass +1
-0@@ -26,6 +26,7 @@ SPDX_TOOL_VERSION ??= "1.0" SPDXRUNTIMEDEPLOY = "${SPDXDIR}/runtime-deploy" SPDX_INCLUDE_SOURCES ??= "0" +SPDX_INCLUDE_COMPILED_SOURCES ??= "0" SPDX_UUID_NAMESPACE ??= "sbom.openembedded.org" SPDX_NAMESPACE_PREFIX ??="http://spdx.o/ rg%2Fspdxdocs&data=05%7C02%7Cdaniel.turull%40ericsson.com%7Cc6b5feb a17be4428746508dda443bebe%7C92e84cebfbfd47abbe52080c6b87953f%7C 0%7C0%7C638847333069033663%7CUnknown%7CTWFpbGZsb3d8eyJFbXB0e U1hcGkiOnRydWUsIlYiOiIwLjAuMDAwMCIsIlAiOiJXaW4zMiIsIkFOIjoiTWFpbCI sIldUIjoyfQ%3D%3D%7C0%7C%7C%7C&sdata=JujsQFQMyjDumPohQKZ8C3Vp ECOPPTtwOZE9qqiEaZ0%3D&reserved=0"
meta/classes/spdx-common.bbclass SPDX_MULTILIB_SSTATE_ARCHS ??=
Quoted 1 line
meta/classes/spdx-common.bbclass +0
-0@@ -40,6 +41,8 @@ SPDX_MULTILIB_SSTATE_ARCHS ??="${SSTATE_ARCHS}"
Quoted 13 lines
meta/classes/spdx-common.bbclass +2
-0python () { from oe.cve_check import extend_cve_status extend_cve_status(d) + if d.getVar("SPDX_INCLUDE_COMPILED_SOURCES") == "1": + d.setVar("SPDX_INCLUDE_SOURCES", "1") } def create_spdx_source_deps(d):meta/lib/oe/spdx30_tasks.py +0
-0meta/lib/oe/spdx30_tasks.py def add_package_files( · line 159
Quoted 4 lines
meta/lib/oe/spdx30_tasks.py +1
-0@@ -156,6 +156,11 @@ def add_package_files( bb.note(f"Skip {topdir}") return spdx_files + check_compiled_sources =
d.getVar("SPDX_INCLUDE_COMPILED_SOURCES")== "1"meta/lib/oe/spdx30_tasks.py def add_package_files( · line 161
Quoted 2 lines
meta/lib/oe/spdx30_tasks.py +2
-0+ if check_compiled_sources: + compiled_sources, types =
oe.spdx_common.get_compiled_sources(d)meta/lib/oe/spdx30_tasks.py def add_package_files( · line 181
Quoted 12 lines
meta/lib/oe/spdx30_tasks.py +5
-0+ bb.debug(1, f"Total compiled files: {len(compiled_sources)}") + for subdir, dirs, files in os.walk(topdir, onerror=walk_error): dirs[:] = [d for d in dirs if d not in ignore_dirs] if subdir == str(topdir):@@ -171,6 +176,11 @@ def add_package_files( filename = str(filepath.relative_to(topdir)) file_purposes = get_purposes(filepath) + # Check if file is compiled + if check_compiled_sources: + if not oe.spdx_common.is_compiled_source(filename,
compiled_sources, types):Quoted 11 lines
meta/lib/oe/spdx30_tasks.py +2
-0+ continue + spdx_file = objset.new_file( get_spdxid(file_counter), filename,meta/lib/oe/spdx_common.py +0
-0@@ -242,3 +242,44 @@ def fetch_data_to_uri(fd, name): uri = uri + "@" + fd.revisionmeta/lib/oe/spdx_common.py def fetch_data_to_uri(fd, name): · line 252
Quoted 10 lines
meta/lib/oe/spdx_common.py +8
-0return uri + +def is_compiled_source (filename, compiled_sources, types): + """ + Check if the file is a compiled file + """ + import os + # If we don't have compiled source, we assume all are compiled. + if len(compiled_sources) == 0:idiomatically, this would be:
Quoted 2 lines
if not
compiled_sources:
return True
Thanks. I'll change it.
Quoted 6 lines
meta/lib/oe/spdx_common.py def fetch_data_to_uri(fd, name): · line 255
Quoted 3 lines
meta/lib/oe/spdx_common.py +3
-0+ return True + # We remove the top directory, to match the format in compiled + sourcesThis feels a little bit magic, can you explain why removing the top directory is necessary and or correct to always perform?
From the data that we extract in the do_packaging, the files could be in a different place, mostly for the kernel, therefore I have normalized them removing the top directory.
For example for linux-yocto, if I take tcp_ipv4.c SPDX path:linux-yocto-6.12.30+git/net/ipv4/tcp_ipv4.c Packaging path from debugsources without normalization:
"/usr/src/kernel/net/ipv4/tcp_ipv4.c", Packaging with normalization: net/ipv4/tcp_ipv4.c
For ncurses:
SPDX: git/ncurses/tinfo/comp_scan.c
Packaging: "/usr/src/debug/ncurses/git/ncurses/tinfo/comp_scan.c" Packaging with normalization: ncurses/tinfo/comp_scan.c
Quoted 5 lines
meta/lib/oe/spdx_common.py def fetch_data_to_uri(fd, name): · line 255
Quoted 1 line
meta/lib/oe/spdx_common.py +1
-0+ relative = filename[filename.find("/")+1:]you use .find() a lot, but I think
split()is more idiomatic:_, relative =
filename.split("/", 1)
I'll change it for this one.
Quoted 5 lines
meta/lib/oe/spdx_common.py def fetch_data_to_uri(fd, name): · line 259
Quoted 3 lines
meta/lib/oe/spdx_common.py +3
-0+ basename = os.path.basename(filename) + # We return always true if the file type is not in the list of + compiled filesWhy is this?
For example Makefiles, python files or config files that are not actually compiled, but should be included since are part of the build
Quoted 24 lines
meta/lib/oe/spdx_common.py def fetch_data_to_uri(fd, name): · line 258
Quoted 1 line
meta/lib/oe/spdx_common.py +1
-0+ if basename[basename.find("."):] not in types:if "." not in basename or
basename.split(".", 1)[1] not in types:
return Truemeta/lib/oe/spdx_common.py def fetch_data_to_uri(fd, name):
Quoted 7 lines
meta/lib/oe/spdx_common.py +7
-0+ return True + # Check that the file is in the list + return relative in compiled_sources + +def get_compiled_sources(d): + """ + Get list of compiled sources from debug information and normalize thepaths
meta/lib/oe/spdx_common.py def fetch_data_to_uri(fd, name):
Quoted 10 lines
meta/lib/oe/spdx_common.py +10
-0+ """ + import itertools + source_info = oe.package.read_debugsources_file(d) + if not source_info: + bb.debug(1, "Do not have debugsources.list. Skipping") + return [], [] + + # Sources are not split now in SPDX, so we aggregate them + sources = + list(set(itertools.chain.from_iterable(source_info.values())))
Quoted 3 lines
sources should be a set, since you are using the "in" operator, which is much for efficient for sets than lists (especially when they are large)
I'll change it. That's a good tip.
Quoted 11 lines
meta/lib/oe/spdx_common.py def fetch_data_to_uri(fd, name): · line 281
Quoted 8 lines
meta/lib/oe/spdx_common.py +7
-0+ # Check extensions of files + types = [] + for src in sources: + basename = os.path.basename(src) + # We check that the basename has an extension + if basename.find(".") > 0: + ext = basename[basename.find("."):]Similarly:
Quoted 7 lines
if "." in basename:
stem, ext = basename.split(".", 1)meta/lib/oe/spdx_common.py def fetch_data_to_uri(fd, name): · line 283
Quoted 2 lines
meta/lib/oe/spdx_common.py +2
-0+ if ext not in types and len(ext)>0: + types.append(ext)Making types a set would be more efficient, e.g.
Thanks for the tip. I'll replace the lists with sets.
Quoted 11 lines
types =
set()
...if ext:
types.add(ext)The "in" operator is much more efficient for sets than lists
Quoted 1 line
+ bb.debug(1, f"Num of sources: {len(sources)} and types: {len(types)}
{str(types)}")
Quoted 1 line
+ return sources, types
PATCH v6 3/3 improve_kernel_cve_report: add script for postprocesing of kernel CVE data
In reply to [email protected]From: Daniel Turull <[email protected]>
Adding postprocessing script to process data from linux CNA that includes more accurate metadata and it is updated directly by the source.
Example of enhanced CVE from a report from cve-check:
{
"id": "CVE-2024-26710",
"status": "Ignored",
"link": "https://nvd.nist.gov/vuln/detail/CVE-2024-26710",
"summary": "In the Linux kernel, the following vulnerability [...]",
"scorev2": "0.0",
"scorev3": "5.5",
"scorev4": "0.0",
"modified": "2025-03-17T15:36:11.620",
"vector": "LOCAL",
"vectorString": "CVSS:3.1/AV:L/AC:L/PR:L/UI:N/S:U/C:N/I:N/A:H",
"detail": "not-applicable-config",
"description": "Source code not compiled by config. ['arch/powerpc/include/asm/thread_info.h']"
},
And same from a report generated with vex:
{
"id": "CVE-2024-26710",
"status": "Ignored",
"link": "https://nvd.nist.gov/vuln/detail/CVE-2024-26710",
"detail": "not-applicable-config",
"description": "Source code not compiled by config. ['arch/powerpc/include/asm/thread_info.h']"
},
For unpatched CVEs, provide more context in the description:
Tested with 6.12.22 kernel
{
"id": "CVE-2025-39728",
"status": "Unpatched",
"link": "https://nvd.nist.gov/vuln/detail/CVE-2025-39728", "summary": "In the Linux kernel, the following vulnerability has been [...], "scorev2": "0.0",
"scorev3": "0.0",
"scorev4": "0.0",
"modified": "2025-04-21T14:23:45.950",
"vector": "UNKNOWN",
"vectorString": "UNKNOWN",
"detail": "version-in-range",
"description": "Needs backporting (fixed from 6.12.23)"
},
--- scripts/contrib/improve_kernel_cve_report.py | 467 +++++++++++++++++++ 1 file changed, 467 insertions(+) create mode 100755 scripts/contrib/improve_kernel_cve_report.py
scripts/contrib/improve_kernel_cve_report.py new file +467 -0
@@ -0,0 +1,467 @@ +#! /usr/bin/env python3 +# +# Copyright OpenEmbedded Contributors +# +# The script uses another source of CVE information from linux-vulns +# to enrich the cve-summary from cve-check or vex. +# It can also use the list of compiled files from the kernel spdx to ignore CVEs +# that are not affected since the files are not compiled. +# +# It creates a new json file with updated CVE information +# +# Compiled files can be extracted adding the following in local.conf +# SPDX_INCLUDE_COMPILED_SOURCES:pn-linux-yocto = "1" +# +# Tested with the following CVE sources: +# - https://git.kernel.org/pub/scm/linux/security/vulns.git +# - https://github.com/CVEProject/cvelistV5 +# +# Example: +# python3 ./openembedded-core/scripts/contrib/improve_kernel_cve_report.py --spdx tmp/deploy/spdx/3.0.1/qemux86_64/recipes/recipe-linux-yocto.spdx.json --kernel-version 6.12.27 --datadir ./vulns +# python3 ./openembedded-core/scripts/contrib/improve_kernel_cve_report.py --spdx tmp/deploy/spdx/3.0.1/qemux86_64/recipes/recipe-linux-yocto.spdx.json --datadir ./vulns --old-cve-report build/tmp/log/cve/cve-summary.json +# +# SPDX-License-Identifier: GPLv2 + +import argparse +import json +import sys +import logging +import glob +import os +import pathlib +from packaging.version import Version + +def is_linux_cve(cve_info): + '''Return true is the CVE belongs to Linux''' + if not "affected" in cve_info["containers"]["cna"]: + return False + for affected in cve_info["containers"]["cna"]["affected"]: + if not "product" in affected: + return False + if affected["product"] == "Linux" and affected["vendor"] == "Linux": + return True + return False + +def get_kernel_cves(datadir, compiled_files, version): + """ + Get CVEs for the kernel + """ + cves = {} + + check_config = len(compiled_files) > 0 + + base_version = Version(f"{version.major}.{version.minor}") + + # Check all CVES from kernel vulns + pattern = os.path.join(datadir, '**', "CVE-*.json") + cve_files = glob.glob(pattern, recursive=True) + not_applicable_config = 0 + fixed_as_later_backport = 0 + vulnerable = 0 + not_vulnerable = 0 + for cve_file in sorted(cve_files): + cve_info = {} + with open(cve_file, "r", encoding='ISO-8859-1') as f: + cve_info = json.load(f) + + if len(cve_info) == 0: + logging.error("Not valid data in %s. Aborting", cve_file) + break + + if not is_linux_cve(cve_info): + continue + cve_id = os.path.basename(cve_file)[:-5] + description = cve_info["containers"]["cna"]["descriptions"][0]["value"] + if cve_file.find("rejected") >= 0: + logging.debug("%s is rejected by the CNA", cve_id) + cves[cve_id] = { + "id": cve_id, + "status": "Ignored", + "detail": "rejected", + "summary": description, + "description": f"Rejected by CNA" + } + continue + if any(elem in cve_file for elem in ["review", "reverved", "testing"]): + continue + + is_vulnerable, first_affected, last_affected, better_match_first, better_match_last, affected_versions = get_cpe_applicability(cve_info, version) + + logging.debug("%s: %s (%s - %s) (%s - %s)", cve_id, is_vulnerable, better_match_first, better_match_last, first_affected, last_affected) + + if is_vulnerable is None: + logging.warning("%s doesn't have good metadata", cve_id) + if is_vulnerable: + is_affected = True + affected_files = [] + if check_config: + is_affected, affected_files = check_kernel_compiled_files(compiled_files, cve_info) + + if not is_affected and len(affected_files) > 0: + logging.debug( + "%s - not applicable configuration since affected files not compiled: %s", + cve_id, affected_files) + cves[cve_id] = { + "id": cve_id, + "status": "Ignored", + "detail": "not-applicable-config", + "summary": description, + "description": f"Source code not compiled by config. {affected_files}" + } + not_applicable_config +=1 + # Check if we have backport + else: + if not better_match_last: + fixed_in = last_affected + else: + fixed_in = better_match_last + logging.debug("%s needs backporting (fixed from %s)", cve_id, fixed_in) + cves[cve_id] = { + "id": cve_id, + "status": "Unpatched", + "detail": "version-in-range", + "summary": description, + "description": f"Needs backporting (fixed from {fixed_in})" + } + vulnerable += 1 + if (better_match_last and + Version(f"{better_match_last.major}.{better_match_last.minor}") == base_version): + fixed_as_later_backport += 1 + # Not vulnerable + else: + if not first_affected: + logging.debug("%s - not known affected %s", + cve_id, + better_match_last) + cves[cve_id] = { + "id": cve_id, + "status": "Patched", + "detail": "version-not-in-range", + "summary": description, + "description": "No CPE match" + } + not_vulnerable += 1 + continue + backport_base = Version(f"{better_match_last.major}.{better_match_last.minor}") + if version < first_affected: + logging.debug('%s - fixed-version: only affects %s onwards', + cve_id, + first_affected) + cves[cve_id] = { + "id": cve_id, + "status": "Patched", + "detail": "fixed-version", + "summary": description, + "description": f"only affects {first_affected} onwards" + } + not_vulnerable += 1 + elif last_affected <= version: + logging.debug("%s - fixed-version: Fixed from version %s", + cve_id, + last_affected) + cves[cve_id] = { + "id": cve_id, + "status": "Patched", + "detail": "fixed-version", + "summary": description, + "description": f"fixed-version: Fixed from version {last_affected}" + } + not_vulnerable += 1 + elif backport_base == base_version: + logging.debug("%s - cpe-stable-backport: Backported in %s", + cve_id, + better_match_last) + cves[cve_id] = { + "id": cve_id, + "status": "Patched", + "detail": "cpe-stable-backport", + "summary": description, + "description": f"Backported in {better_match_last}" + } + not_vulnerable += 1 + else: + logging.debug("%s - version not affected %s", cve_id, str(affected_versions)) + cves[cve_id] = { + "id": cve_id, + "status": "Patched", + "detail": "version-not-in-range", + "summary": description, + "description": f"Range {affected_versions}" + } + not_vulnerable += 1 + + logging.info("Total CVEs ignored due to not applicable config: %d", not_applicable_config) + logging.info("Total CVEs not vulnerable due version-not-in-range: %d", not_vulnerable) + logging.info("Total vulnerable CVEs: %d", vulnerable) + + logging.info("Total CVEs already backported in %s: %s", base_version, + fixed_as_later_backport) + return cves + +def read_spdx(spdx_file): + '''Open SPDX file and extract compiled files''' + with open(spdx_file, 'r', encoding='ISO-8859-1') as f: + spdx = json.load(f) + if "spdxVersion" in spdx: + if spdx["spdxVersion"] == "SPDX-2.2": + return read_spdx2(spdx) + if "@graph" in spdx: + return read_spdx3(spdx) + return [] + +def read_spdx2(spdx): + ''' + Read spdx2 compiled files from spdx + ''' + cfiles = [] + if 'files' not in spdx: + return cfiles + for item in spdx['files']: + for ftype in item['fileTypes']: + if ftype == "SOURCE": + filename = item["fileName"][item["fileName"].find("/")+1:] + cfiles.append(filename) + return cfiles + +def read_spdx3(spdx): + ''' + Read spdx3 compiled files from spdx + ''' + cfiles = [] + for item in spdx["@graph"]: + if "software_primaryPurpose" not in item: + continue + if item["software_primaryPurpose"] == "source": + filename = item['name'][item['name'].find("/")+1:] + cfiles.append(filename) + return cfiles + +def check_kernel_compiled_files(compiled_files, cve_info): + """ + Return if a CVE affected us depending on compiled files + """ + files_affected = [] + is_affected = False + + for item in cve_info['containers']['cna']['affected']: + if "programFiles" in item: + for f in item['programFiles']: + if f not in files_affected: + files_affected.append(f) + + if len(files_affected) > 0: + for f in files_affected: + if f in compiled_files: + logging.debug("File match: %s", f) + is_affected = True + return is_affected, files_affected + +def get_cpe_applicability(cve_info, v): + ''' + Check if version is affected and return affected versions + ''' + base_branch = Version(f"{v.major}.{v.minor}") + affected = [] + if not 'cpeApplicability' in cve_info["containers"]["cna"]: + return None, None, None, None, None, None + + for nodes in cve_info["containers"]["cna"]["cpeApplicability"]: + for node in nodes.values(): + vulnerable = False + matched_branch = False + first_affected = Version("5000") + last_affected = Version("0") + better_match_first = Version("0") + better_match_last = Version("5000") + + if len(node[0]['cpeMatch']) == 0: + first_affected = None + last_affected = None + better_match_first = None + better_match_last = None + + for cpe_match in node[0]['cpeMatch']: + version_start_including = Version("0") + version_end_excluding = Version("0") + if 'versionStartIncluding' in cpe_match: + version_start_including = Version(cpe_match['versionStartIncluding']) + else: + version_start_including = Version("0") + # if versionEndExcluding is missing we are in a branch, which is not fixed. + if "versionEndExcluding" in cpe_match: + version_end_excluding = Version(cpe_match["versionEndExcluding"]) + else: + # if versionEndExcluding is missing we are in a branch, which is not fixed. + version_end_excluding = Version( + f"{version_start_including.major}.{version_start_including.minor}.5000" + ) + affected.append(f" {version_start_including}-{version_end_excluding}") + # Detect if versionEnd is in fixed in base branch. It has precedence over the rest + branch_end = Version(f"{version_end_excluding.major}.{version_end_excluding.minor}") + if branch_end == base_branch: + if version_start_including <= v < version_end_excluding: + vulnerable = cpe_match['vulnerable'] + # If we don't match in our branch, we are not vulnerable, + # since we have a backport + matched_branch = True + better_match_first = version_start_including + better_match_last = version_end_excluding + if version_start_including <= v < version_end_excluding and not matched_branch: + if version_end_excluding < better_match_last: + better_match_first = max(version_start_including, better_match_first) + better_match_last = min(better_match_last, version_end_excluding) + vulnerable = cpe_match['vulnerable'] + matched_branch = True + + first_affected = min(version_start_including, first_affected) + last_affected = max(version_end_excluding, last_affected) + # Not a better match, we use the first and last affected instead of the fake .5000 + if vulnerable and better_match_last == Version(f"{base_branch}.5000"): + better_match_last = last_affected + better_match_first = first_affected + return vulnerable, first_affected, last_affected, better_match_first, better_match_last, affected + +def copy_data(old, new): + '''Update dictionary with new entries, while keeping the old ones''' + for k in new.keys(): + old[k] = new[k] + return old + +# Function taken from cve_check.bbclass. Adapted to cve fields +def cve_update(cve_data, cve, entry): + # If no entry, just add it + if cve not in cve_data: + cve_data[cve] = entry + return + # If we are updating, there might be change in the status + if cve_data[cve]['status'] == "Unknown": + cve_data[cve] = copy_data(cve_data[cve], entry) + return + if cve_data[cve]['status'] == entry['status']: + return + if entry['status'] == "Unpatched" and cve_data[cve]['status'] == "Patched": + logging.warning("CVE entry %s update from Patched to Unpatched from the scan result", cve) + cve_data[cve] = copy_data(cve_data[cve], entry) + return + if entry['status'] == "Patched" and cve_data[cve]['status'] == "Unpatched": + logging.warning("CVE entry %s update from Unpatched to Patched from the scan result", cve) + cve_data[cve] = copy_data(cve_data[cve], entry) + return + # If we have an "Ignored", it has a priority + if cve_data[cve]['status'] == "Ignored": + logging.debug("CVE %s not updating because Ignored", cve) + return + # If we have an "Ignored", it has a priority + if entry['status'] == "Ignored": + cve_data[cve] = copy_data(cve_data[cve], entry) + logging.debug("CVE entry %s updated from Unpatched to Ignored", cve) + return + logging.warning("Unhandled CVE entry update for %s %s from %s %s to %s", + cve, cve_data[cve]['status'], cve_data[cve]['detail'], entry['status'], entry['detail']) + +def main(): + parser = argparse.ArgumentParser( + description="Update cve-summary with kernel compiled files and kernel CVE information" + ) + parser.add_argument( + "-s", + "--spdx", + help="SPDX2/3 for the kernel. Needs to include compiled sources", + ) + parser.add_argument( + "--datadir", + type=pathlib.Path, + help="Directory where CVE data is", + required=True + ) + parser.add_argument( + "--old-cve-report", + help="CVE report to update. (Optional)", + ) + parser.add_argument( + "--kernel-version", + help="Kernel version. Needed if old cve_report is not provided (Optional)", + type=Version + ) + parser.add_argument( + "--new-cve-report", + help="Output file", + default="cve-summary-enhance.json" + ) + parser.add_argument( + "-D", + "--debug", + help='Enable debug ', + action="store_true") + + args = parser.parse_args() + + if args.debug: + log_level=logging.DEBUG + else: + log_level=logging.INFO + logging.basicConfig(format='[%(filename)s:%(lineno)d] %(message)s', level=log_level) + + if not args.kernel_version and not args.old_cve_report: + parser.error("either --kernel-version or --old-cve-report are needed") + return -1 + + # by default we don't check the compiled files, unless provided + compiled_files = [] + if args.spdx: + compiled_files = read_spdx(args.spdx) + logging.info("Total compiled files %d", len(compiled_files)) + + if args.old_cve_report: + with open(args.old_cve_report, encoding='ISO-8859-1') as f: + cve_report = json.load(f) + else: + #If summary not provided, we create one + cve_report = { + "version": "1", + "package": [ + { + "name": "linux-yocto", + "version": str(args.kernel_version), + "products": [ + { + "product": "linux_kernel", + "cvesInRecord": "Yes" + } + ], + "issue": [] + } + ] + } + + for pkg in cve_report['package']: + is_kernel = False + for product in pkg['products']: + if product['product'] == "linux_kernel": + is_kernel=True + if not is_kernel: + continue + + kernel_cves = get_kernel_cves(args.datadir, + compiled_files, + Version(pkg["version"])) + logging.info("Total kernel cves from kernel CNA: %s", len(kernel_cves)) + cves = {issue["id"]: issue for issue in pkg["issue"]} + logging.info("Total kernel before processing cves: %s", len(cves)) + + for cve in kernel_cves: + cve_update(cves, cve, kernel_cves[cve]) + + pkg["issue"] = [] + for cve in sorted(cves): + pkg["issue"].extend([cves[cve]]) + logging.info("Total kernel cves after processing: %s", len(pkg['issue'])) + + with open(args.new_cve_report, "w", encoding='ISO-8859-1') as f: + json.dump(cve_report, f, indent=2) + + return 0 + +if __name__ == "__main__": + sys.exit(main()) +