cve-check: enrich annotation of CVEs
PATCH 1/3 cve-check: enrich annotation of CVEs
Previously the information passed between different function of the cve-check class included only tables of patched, unpatched, ignored vulnerabilities and the general status of the recipe.
The VEX work requires more information, and we need to pass them between different functions, so that it can be enriched as the analysis progresses. Instead of multiple tables, use a single one with annotations for each CVE encountered. For example, a patched CVE will have:
{"abbrev-status": "Patched", "status": "version-not-in-range"}
abbrev-status contains the general status (Patched, Unpatched, Ignored and Unknown that will be added in the VEX code) status contains more detailed information that can come from CVE_STATUS and the analysis.
Additional fields of the annotation include for example the name of the patch file fixing a given CVE.
--- meta/classes/cve-check.bbclass | 208 +++++++++++++++++---------------- meta/lib/oe/cve_check.py | 12 +- 2 files changed, 113 insertions(+), 107 deletions(-)
meta/classes/cve-check.bbclass +108 -100
@@ -188,10 +188,10 @@ python do_cve_check () { patched_cves = get_patched_cves(d) except FileNotFoundError: bb.fatal("Failure in searching patches") - ignored, patched, unpatched, status = check_cves(d, patched_cves) - if patched or unpatched or (d.getVar("CVE_CHECK_COVERAGE") == "1" and status): - cve_data = get_cve_info(d, patched + unpatched + ignored) - cve_write_data(d, patched, unpatched, ignored, cve_data, status) + cve_data, status = check_cves(d, patched_cves) + if len(cve_data) or (d.getVar("CVE_CHECK_COVERAGE") == "1" and status): + get_cve_info(d, cve_data) + cve_write_data(d, cve_data, status) else: bb.note("No CVE database found, skipping CVE check")
@@ -294,7 +294,51 @@ ROOTFS_POSTPROCESS_COMMAND:prepend = "${@'cve_check_write_rootfs_manifest ' if d do_rootfs[recrdeptask] += "${@'do_cve_check' if d.getVar('CVE_CHECK_CREATE_MANIFEST') == '1' else ''}" do_populate_sdk[recrdeptask] += "${@'do_cve_check' if d.getVar('CVE_CHECK_CREATE_MANIFEST') == '1' else ''}" -def check_cves(d, patched_cves): +def cve_is_ignored(d, cve_data, cve): + if cve not in cve_data: + return False + if cve_data[cve]['abbrev-status'] == "Ignored": + return True + return False + +def cve_is_patched(d, cve_data, cve): + if cve not in cve_data: + return False + if cve_data[cve]['abbrev-status'] == "Patched": + return True + return False + +def cve_update(d, cve_data, cve, entry): + # If no entry, just add it + if cve not in cve_data: + cve_data[cve] = entry + return + # If we are updating, there might be change in the status + bb.debug("Trying CVE entry update for %s from %s to %s" % (cve, cve_data[cve]['abbrev-status'], entry['abbrev-status'])) + if cve_data[cve]['abbrev-status'] == "Unknown": + cve_data[cve] = entry + return + if cve_data[cve]['abbrev-status'] == entry['abbrev-status']: + return + # Update like in {'abbrev-status': 'Patched', 'status': 'version-not-in-range'} to {'abbrev-status': 'Unpatched', 'status': 'version-in-range'} + if entry['abbrev-status'] == "Unpatched" and cve_data[cve]['abbrev-status'] == "Patched": + if entry['status'] == "version-in-range" and cve_data[cve]['status'] == "version-not-in-range": + # New result from the scan, vulnerable + cve_data[cve] = entry + bb.debug("CVE entry %s update from Patched to Unpatched from the scan result" % cve) + return + if entry['abbrev-status'] == "Patched" and cve_data[cve]['abbrev-status'] == "Unpatched": + if entry['status'] == "version-not-in-range" and cve_data[cve]['status'] == "version-in-range": + # Range does not match the scan, but we already have a vulnerable match, ignore + bb.debug("CVE entry %s update from Patched to Unpatched from the scan result - not applying" % cve) + return + # If we have an "Ignored", it has a priority + if cve_data[cve]['abbrev-status'] == "Ignored": + bb.debug("CVE %s not updating because Ignored" % cve) + return + bb.warn("Unhandled CVE entry update for %s from %s to %s" % (cve, cve_data[cve], entry)) + +def check_cves(d, cve_data): """ Connect to the NVD database and find unpatched cves. """
@@ -304,28 +348,19 @@ def check_cves(d, patched_cves): real_pv = d.getVar("PV") suffix = d.getVar("CVE_VERSION_SUFFIX") - cves_unpatched = [] - cves_ignored = [] cves_status = [] cves_in_recipe = False # CVE_PRODUCT can contain more than one product (eg. curl/libcurl) products = d.getVar("CVE_PRODUCT").split() # If this has been unset then we're not scanning for CVEs here (for example, image recipes) if not products: - return ([], [], [], []) + return ([], []) pv = d.getVar("CVE_VERSION").split("+git")[0] # If the recipe has been skipped/ignored we return empty lists if pn in d.getVar("CVE_CHECK_SKIP_RECIPE").split(): bb.note("Recipe has been skipped by cve-check") - return ([], [], [], []) - - # Convert CVE_STATUS into ignored CVEs and check validity - cve_ignore = [] - for cve in (d.getVarFlags("CVE_STATUS") or {}): - decoded_status, _, _ = decode_cve_status(d, cve) - if decoded_status == "Ignored": - cve_ignore.append(cve) + return ([], []) import sqlite3 db_file = d.expand("file:${CVE_CHECK_DB_FILE}?mode=ro")
@@ -344,11 +379,10 @@ def check_cves(d, patched_cves): for cverow in cve_cursor: cve = cverow[0] - if cve in cve_ignore: + if cve_is_ignored(d, cve_data, cve): bb.note("%s-%s ignores %s" % (product, pv, cve)) - cves_ignored.append(cve) continue - elif cve in patched_cves: + elif cve_is_patched(d, cve_data, cve): bb.note("%s has been patched" % (cve)) continue # Write status once only for each product
@@ -364,7 +398,7 @@ def check_cves(d, patched_cves): for row in product_cursor: (_, _, _, version_start, operator_start, version_end, operator_end) = row #bb.debug(2, "Evaluating row " + str(row)) - if cve in cve_ignore: + if cve_is_ignored(d, cve_data, cve): ignored = True version_start = convert_cve_version(version_start)
@@ -403,16 +437,16 @@ def check_cves(d, patched_cves): if vulnerable: if ignored: bb.note("%s is ignored in %s-%s" % (cve, pn, real_pv)) - cves_ignored.append(cve) + cve_update(d, cve_data, cve, {"abbrev-status": "Ignored"}) else: bb.note("%s-%s is vulnerable to %s" % (pn, real_pv, cve)) - cves_unpatched.append(cve) + cve_update(d, cve_data, cve, {"abbrev-status": "Unpatched", "status": "version-in-range"}) break product_cursor.close() if not vulnerable: bb.note("%s-%s is not vulnerable to %s" % (pn, real_pv, cve)) - patched_cves.add(cve) + cve_update(d, cve_data, cve, {"abbrev-status": "Patched", "status": "version-not-in-range"}) cve_cursor.close() if not cves_in_product:
@@ -420,48 +454,45 @@ def check_cves(d, patched_cves): cves_status.append([product, False]) conn.close() - diff_ignore = list(set(cve_ignore) - set(cves_ignored)) - if diff_ignore: - oe.qa.handle_error("cve_status_not_in_db", "Found CVE (%s) with CVE_STATUS set that are not found in database for this component" % " ".join(diff_ignore), d) if not cves_in_recipe: bb.note("No CVE records for products in recipe %s" % (pn)) - return (list(cves_ignored), list(patched_cves), cves_unpatched, cves_status) + return (cve_data, cves_status) -def get_cve_info(d, cves): +def get_cve_info(d, cve_data): """ Get CVE information from the database. """ import sqlite3 - cve_data = {} db_file = d.expand("file:${CVE_CHECK_DB_FILE}?mode=ro") conn = sqlite3.connect(db_file, uri=True) - for cve in cves: + for cve in cve_data: cursor = conn.execute("SELECT * FROM NVD WHERE ID IS ?", (cve,)) for row in cursor: - cve_data[row[0]] = {} - cve_data[row[0]]["summary"] = row[1] - cve_data[row[0]]["scorev2"] = row[2] - cve_data[row[0]]["scorev3"] = row[3] - cve_data[row[0]]["modified"] = row[4] - cve_data[row[0]]["vector"] = row[5] - cve_data[row[0]]["vectorString"] = row[6] + # The CVE itdelf has been added already + if row[0] not in cve_data: + bb.note("CVE record %s not present" % row[0]) + continue + #cve_data[row[0]] = {} + cve_data[row[0]]["NVD-summary"] = row[1] + cve_data[row[0]]["NVD-scorev2"] = row[2] + cve_data[row[0]]["NVD-scorev3"] = row[3] + cve_data[row[0]]["NVD-modified"] = row[4] + cve_data[row[0]]["NVD-vector"] = row[5] + cve_data[row[0]]["NVD-vectorString"] = row[6] cursor.close() conn.close() - return cve_data -def cve_write_data_text(d, patched, unpatched, ignored, cve_data): +def cve_write_data_text(d, cve_data): """ Write CVE information in WORKDIR; and to CVE_CHECK_DIR, and CVE manifest if enabled. """ - from oe.cve_check import decode_cve_status - cve_file = d.getVar("CVE_CHECK_LOG") fdir_name = d.getVar("FILE_DIRNAME") layer = fdir_name.split("/")[-3]
@@ -478,7 +509,7 @@ def cve_write_data_text(d, patched, unpatched, ignored, cve_data): return # Early exit, the text format does not report packages without CVEs - if not patched+unpatched+ignored: + if not len(cve_data): return nvd_link = "https://nvd.nist.gov/vuln/detail/"
@@ -487,36 +518,27 @@ def cve_write_data_text(d, patched, unpatched, ignored, cve_data): bb.utils.mkdirhier(os.path.dirname(cve_file)) for cve in sorted(cve_data): - is_patched = cve in patched - is_ignored = cve in ignored - - status = "Unpatched" - if (is_patched or is_ignored) and not report_all: - continue - if is_ignored: - status = "Ignored" - elif is_patched: - status = "Patched" - else: - # default value of status is Unpatched - unpatched_cves.append(cve) - write_string += "LAYER: %s\n" % layer write_string += "PACKAGE NAME: %s\n" % d.getVar("PN") write_string += "PACKAGE VERSION: %s%s\n" % (d.getVar("EXTENDPE"), d.getVar("PV")) write_string += "CVE: %s\n" % cve - write_string += "CVE STATUS: %s\n" % status - _, detail, description = decode_cve_status(d, cve) - if detail: - write_string += "CVE DETAIL: %s\n" % detail - if description: - write_string += "CVE DESCRIPTION: %s\n" % description - write_string += "CVE SUMMARY: %s\n" % cve_data[cve]["summary"] - write_string += "CVSS v2 BASE SCORE: %s\n" % cve_data[cve]["scorev2"] - write_string += "CVSS v3 BASE SCORE: %s\n" % cve_data[cve]["scorev3"] - write_string += "VECTOR: %s\n" % cve_data[cve]["vector"] - write_string += "VECTORSTRING: %s\n" % cve_data[cve]["vectorString"] + write_string += "CVE STATUS: %s\n" % cve_data[cve]["abbrev-status"] + + if 'status' in cve_data[cve]: + write_string += "CVE DETAIL: %s\n" % cve_data[cve]["status"] + if 'justification' in cve_data[cve]: + write_string += "CVE DESCRIPTION: %s\n" % cve_data[cve]["justification"] + + if "NVD-summary" in cve_data[cve]: + write_string += "CVE SUMMARY: %s\n" % cve_data[cve]["NVD-summary"] + write_string += "CVSS v2 BASE SCORE: %s\n" % cve_data[cve]["NVD-scorev2"] + write_string += "CVSS v3 BASE SCORE: %s\n" % cve_data[cve]["NVD-scorev3"] + write_string += "VECTOR: %s\n" % cve_data[cve]["NVD-vector"] + write_string += "VECTORSTRING: %s\n" % cve_data[cve]["NVD-vectorString"] + write_string += "MORE INFORMATION: %s%s\n\n" % (nvd_link, cve) + if cve_data[cve]["abbrev-status"] == "Unpatched": + unpatched_cves.append(cve) if unpatched_cves and d.getVar("CVE_CHECK_SHOW_WARNINGS") == "1": bb.warn("Found unpatched CVE (%s), for more information check %s" % (" ".join(unpatched_cves),cve_file))
@@ -568,13 +590,11 @@ def cve_check_write_json_output(d, output, direct_file, deploy_file, manifest_fi with open(index_path, "a+") as f: f.write("%s\n" % fragment_path) -def cve_write_data_json(d, patched, unpatched, ignored, cve_data, cve_status): +def cve_write_data_json(d, cve_data, cve_status): """ Prepare CVE data for the JSON format, then write it. """ - from oe.cve_check import decode_cve_status - output = {"version":"1", "package": []} nvd_link = "https://nvd.nist.gov/vuln/detail/"
@@ -592,8 +612,6 @@ def cve_write_data_json(d, patched, unpatched, ignored, cve_data, cve_status): if include_layers and layer not in include_layers: return - unpatched_cves = [] - product_data = [] for s in cve_status: p = {"product": s[0], "cvesInRecord": "Yes"}
@@ -608,39 +626,29 @@ def cve_write_data_json(d, patched, unpatched, ignored, cve_data, cve_status): "version" : package_version, "products": product_data } + cve_list = [] for cve in sorted(cve_data): - is_patched = cve in patched - is_ignored = cve in ignored - status = "Unpatched" - if (is_patched or is_ignored) and not report_all: - continue - if is_ignored: - status = "Ignored" - elif is_patched: - status = "Patched" - else: - # default value of status is Unpatched - unpatched_cves.append(cve) - issue_link = "%s%s" % (nvd_link, cve) cve_item = { "id" : cve, - "summary" : cve_data[cve]["summary"], - "scorev2" : cve_data[cve]["scorev2"], - "scorev3" : cve_data[cve]["scorev3"], - "vector" : cve_data[cve]["vector"], - "vectorString" : cve_data[cve]["vectorString"], - "status" : status, - "link": issue_link + "status" : cve_data[cve]["abbrev-status"], + "link": issue_link, } - _, detail, description = decode_cve_status(d, cve) - if detail: - cve_item["detail"] = detail - if description: - cve_item["description"] = description + if 'NVD-summary' in cve_data[cve]: + cve_item["summary"] = cve_data[cve]["NVD-summary"] + cve_item["scorev2"] = cve_data[cve]["NVD-scorev2"] + cve_item["scorev3"] = cve_data[cve]["NVD-scorev3"] + cve_item["vector"] = cve_data[cve]["NVD-vector"] + cve_item["vectorString"] = cve_data[cve]["NVD-vectorString"] + if 'status' in cve_data[cve]: + cve_item["detail"] = cve_data[cve]["status"] + if 'justification' in cve_data[cve]: + cve_item["description"] = cve_data[cve]["justification"] + if 'resource' in cve_data[cve]: + cve_item["patch-file"] = cve_data[cve]["resource"] cve_list.append(cve_item) package_data["issue"] = cve_list
@@ -652,12 +660,12 @@ def cve_write_data_json(d, patched, unpatched, ignored, cve_data, cve_status): cve_check_write_json_output(d, output, direct_file, deploy_file, manifest_file) -def cve_write_data(d, patched, unpatched, ignored, cve_data, status): +def cve_write_data(d, cve_data, status): """ Write CVE data in each enabled format. """ if d.getVar("CVE_CHECK_FORMAT_TEXT") == "1": - cve_write_data_text(d, patched, unpatched, ignored, cve_data) + cve_write_data_text(d, cve_data) if d.getVar("CVE_CHECK_FORMAT_JSON") == "1": - cve_write_data_json(d, patched, unpatched, ignored, cve_data, status) + cve_write_data_json(d, cve_data, status)
meta/lib/oe/cve_check.py +5 -7
@@ -88,7 +88,7 @@ def get_patched_cves(d): # (cve_match regular expression) cve_file_name_match = re.compile(r".*(CVE-\d{4}-\d+)", re.IGNORECASE) - patched_cves = set() + patched_cves = {} patches = oe.patch.src_patches(d) bb.debug(2, "Scanning %d patches for CVEs" % len(patches)) for url in patches:
@@ -98,7 +98,7 @@ def get_patched_cves(d): fname_match = cve_file_name_match.search(patch_file) if fname_match: cve = fname_match.group(1).upper() - patched_cves.add(cve) + patched_cves[cve] = {"abbrev-status": "Patched", "status": "fix-file-included", "resource": patch_file} bb.debug(2, "Found %s from patch file name %s" % (cve, patch_file)) # Remote patches won't be present and compressed patches won't be
@@ -124,7 +124,7 @@ def get_patched_cves(d): cves = patch_text[match.start()+5:match.end()] for cve in cves.split(): bb.debug(2, "Patch %s solves %s" % (patch_file, cve)) - patched_cves.add(cve) + patched_cves[cve] = {"abbrev-status": "Patched", "status": "fix-file-included", "resource": patch_file} text_match = True if not fname_match and not text_match:
@@ -132,10 +132,8 @@ def get_patched_cves(d): # Search for additional patched CVEs for cve in (d.getVarFlags("CVE_STATUS") or {}): - decoded_status, _, _ = decode_cve_status(d, cve) - if decoded_status == "Patched": - bb.debug(2, "CVE %s is additionally patched" % cve) - patched_cves.add(cve) + decoded_status, detail, description = decode_cve_status(d, cve) + patched_cves[cve] = {"abbrev-status": decoded_status, "status": detail, "justification": description} return patched_cves
Signature · 1 line
2.43.0
PATCH 2/3 vex.bbclass: add a new class
The "vex" class generates the minimum information that is necessary for VEX generation by an external CVE checking tool. It is a drop-in replacement of "cve-check". It uses the same variables from recipes to make the migration and backporting easier.
The goal if this class is to allow generation of the CVE list of an image or distribution on-demand, including the latest information from vulnerability databases. Vulnerability data changes every day, so a status generated at build becomes out-of-date very soon.
Research done for this work shows that the current VEX formats (CSAF and OpenVEX) do not provide enough information to generate such rolling information. Instead, we extract the needed data from recipe annotations (package names, CPEs, versions, CVE patches applied...) and store for later use in the format that is an extension of the CVE-check JSON output format.
This output can be then used (separately or with SPDX of the same build) by an external tool to generate the vulnerability annotation and VEX statements in standard formats.
--- meta/classes/vex.bbclass | 316 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 316 insertions(+) create mode 100644 meta/classes/vex.bbclass
meta/classes/vex.bbclass new file +316 -0
@@ -0,0 +1,316 @@ +# +# Copyright OpenEmbedded Contributors +# +# SPDX-License-Identifier: MIT +# + +# This class is used to check recipes against public CVEs. +# +# In order to use this class just inherit the class in the +# local.conf file and it will add the cve_check task for +# every recipe. The task can be used per recipe, per image, +# or using the special cases "world" and "universe". The +# cve_check task will print a warning for every unpatched +# CVE found and generate a file in the recipe WORKDIR/cve +# directory. If an image is build it will generate a report +# in DEPLOY_DIR_IMAGE for all the packages used. +# +# Example: +# bitbake -c cve_check openssl +# bitbake core-image-sato +# bitbake -k -c cve_check universe +# +# DISCLAIMER +# +# This class/tool is meant to be used as support and not +# the only method to check against CVEs. Running this tool +# doesn't guarantee your packages are free of CVEs. + +# The product name that the CVE database uses defaults to BPN, but may need to +# be overriden per recipe (for example tiff.bb sets CVE_PRODUCT=libtiff). +CVE_PRODUCT ??= "${BPN}" +CVE_VERSION ??= "${PV}" + +CVE_CHECK_SUMMARY_DIR ?= "${LOG_DIR}/cve" + +CVE_CHECK_SUMMARY_FILE_NAME_JSON = "cve-summary.json" +CVE_CHECK_SUMMARY_INDEX_PATH = "${CVE_CHECK_SUMMARY_DIR}/cve-summary-index.txt" + +CVE_CHECK_DIR ??= "${DEPLOY_DIR}/cve" +CVE_CHECK_RECIPE_FILE_JSON ?= "${CVE_CHECK_DIR}/${PN}_cve.json" +CVE_CHECK_MANIFEST_JSON ?= "${IMGDEPLOYDIR}/${IMAGE_NAME}.json" + +# Report Patched or Ignored CVEs +CVE_CHECK_REPORT_PATCHED ??= "1" + +CVE_CHECK_SHOW_WARNINGS ??= "1" + +# Skip CVE Check for packages (PN) +CVE_CHECK_SKIP_RECIPE ?= "" + +# Replace NVD DB check status for a given CVE. Each of CVE has to be mentioned +# separately with optional detail and description for this status. +# +# CVE_STATUS[CVE-1234-0001] = "not-applicable-platform: Issue only applies on Windows" +# CVE_STATUS[CVE-1234-0002] = "fixed-version: Fixed externally" +# +# Settings the same status and reason for multiple CVEs is possible +# via CVE_STATUS_GROUPS variable. +# +# CVE_STATUS_GROUPS = "CVE_STATUS_WIN CVE_STATUS_PATCHED" +# +# CVE_STATUS_WIN = "CVE-1234-0001 CVE-1234-0003" +# CVE_STATUS_WIN[status] = "not-applicable-platform: Issue only applies on Windows" +# CVE_STATUS_PATCHED = "CVE-1234-0002 CVE-1234-0004" +# CVE_STATUS_PATCHED[status] = "fixed-version: Fixed externally" +# +# All possible CVE statuses could be found in cve-check-map.conf +# CVE_CHECK_STATUSMAP[not-applicable-platform] = "Ignored" +# CVE_CHECK_STATUSMAP[fixed-version] = "Patched" +# +# CVE_CHECK_IGNORE is deprecated and CVE_STATUS has to be used instead. +# Keep CVE_CHECK_IGNORE until other layers migrate to new variables +CVE_CHECK_IGNORE ?= "" + +# Layers to be excluded +CVE_CHECK_LAYER_EXCLUDELIST ??= "" + +# Layers to be included +CVE_CHECK_LAYER_INCLUDELIST ??= "" + + +# set to "alphabetical" for version using single alphabetical character as increment release +CVE_VERSION_SUFFIX ??= "" + +python () { + # Fallback all CVEs from CVE_CHECK_IGNORE to CVE_STATUS + cve_check_ignore = d.getVar("CVE_CHECK_IGNORE") + if cve_check_ignore: + bb.warn("CVE_CHECK_IGNORE is deprecated in favor of CVE_STATUS") + for cve in (d.getVar("CVE_CHECK_IGNORE") or "").split(): + d.setVarFlag("CVE_STATUS", cve, "ignored") + + # Process CVE_STATUS_GROUPS to set multiple statuses and optional detail or description at once + for cve_status_group in (d.getVar("CVE_STATUS_GROUPS") or "").split(): + cve_group = d.getVar(cve_status_group) + if cve_group is not None: + for cve in cve_group.split(): + d.setVarFlag("CVE_STATUS", cve, d.getVarFlag(cve_status_group, "status")) + else: + bb.warn("CVE_STATUS_GROUPS contains undefined variable %s" % cve_status_group) +} + +def generate_json_report(d, out_path, link_path): + if os.path.exists(d.getVar("CVE_CHECK_SUMMARY_INDEX_PATH")): + import json + from oe.cve_check import cve_check_merge_jsons, update_symlinks + + bb.note("Generating JSON CVE summary") + index_file = d.getVar("CVE_CHECK_SUMMARY_INDEX_PATH") + summary = {"version":"1", "package": []} + with open(index_file) as f: + filename = f.readline() + while filename: + with open(filename.rstrip()) as j: + data = json.load(j) + cve_check_merge_jsons(summary, data) + filename = f.readline() + + summary["package"].sort(key=lambda d: d['name']) + + with open(out_path, "w") as f: + json.dump(summary, f, indent=2) + + update_symlinks(out_path, link_path) + +python vex_save_summary_handler () { + import shutil + import datetime + from oe.cve_check import update_symlinks + + cvelogpath = d.getVar("CVE_CHECK_SUMMARY_DIR") + + bb.utils.mkdirhier(cvelogpath) + timestamp = datetime.datetime.now().strftime('%Y%m%d%H%M%S') + + json_summary_link_name = os.path.join(cvelogpath, d.getVar("CVE_CHECK_SUMMARY_FILE_NAME_JSON")) + json_summary_name = os.path.join(cvelogpath, "cve-summary-%s.json" % (timestamp)) + generate_json_report(d, json_summary_name, json_summary_link_name) + bb.plain("Complete CVE JSON report summary created at: %s" % json_summary_link_name) +} + +addhandler vex_save_summary_handler +vex_save_summary_handler[eventmask] = "bb.event.BuildCompleted" + +python do_generate_vex () { + """ + Check recipe for patched and unpatched CVEs + """ + from oe.cve_check import get_patched_cves + + try: + patched_cves = get_patched_cves(d) + cves_status = [] + products = d.getVar("CVE_PRODUCT").split() + for product in products: + if ":" in product: + _, product = product.split(":", 1) + cves_status.append([product, False]) + + except FileNotFoundError: + bb.fatal("Failure in searching patches") + + cve_write_data_json(d, patched_cves, cves_status) +} + +addtask generate_vex before do_build + +python vex_cleanup () { + """ + Delete the file used to gather all the CVE information. + """ + bb.utils.remove(e.data.getVar("CVE_CHECK_SUMMARY_INDEX_PATH")) +} + +addhandler vex_cleanup +vex_cleanup[eventmask] = "bb.event.BuildCompleted" + +python vex_write_rootfs_manifest () { + """ + Create CVE manifest when building an image + """ + + import json + from oe.rootfs import image_list_installed_packages + from oe.cve_check import cve_check_merge_jsons, update_symlinks + + deploy_file_json = d.getVar("CVE_CHECK_RECIPE_FILE_JSON") + if os.path.exists(deploy_file_json): + bb.utils.remove(deploy_file_json) + + # Create a list of relevant recipies + recipies = set() + for pkg in list(image_list_installed_packages(d)): + pkg_info = os.path.join(d.getVar('PKGDATA_DIR'), + 'runtime-reverse', pkg) + pkg_data = oe.packagedata.read_pkgdatafile(pkg_info) + recipies.add(pkg_data["PN"]) + + bb.note("Writing rootfs CVE manifest") + deploy_dir = d.getVar("IMGDEPLOYDIR") + link_name = d.getVar("IMAGE_LINK_NAME") + + json_data = {"version":"1", "package": []} + text_data = "" + + save_pn = d.getVar("PN") + + for pkg in recipies: + # To be able to use the CVE_CHECK_RECIPE_FILE_JSON variable we have to evaluate + # it with the different PN names set each time. + d.setVar("PN", pkg) + + pkgfilepath = d.getVar("CVE_CHECK_RECIPE_FILE_JSON") + if os.path.exists(pkgfilepath): + with open(pkgfilepath) as j: + data = json.load(j) + cve_check_merge_jsons(json_data, data) + + d.setVar("PN", save_pn) + + link_path = os.path.join(deploy_dir, "%s.json" % link_name) + manifest_name = d.getVar("CVE_CHECK_MANIFEST_JSON") + + with open(manifest_name, "w") as f: + json.dump(json_data, f, indent=2) + + update_symlinks(manifest_name, link_path) + bb.plain("Image CVE JSON report stored in: %s" % manifest_name) +} + +ROOTFS_POSTPROCESS_COMMAND:prepend = "vex_write_rootfs_manifest; " +do_rootfs[recrdeptask] += "do_generate_vex " +do_populate_sdk[recrdeptask] += "do_generate_vex " + +def cve_write_data_json(d, cve_data, cve_status): + """ + Prepare CVE data for the JSON format, then write it. + Done for each recipe. + """ + + from oe.cve_check import get_cpe_ids + import json + + output = {"version":"1", "package": []} + nvd_link = "https://nvd.nist.gov/vuln/detail/" + + fdir_name = d.getVar("FILE_DIRNAME") + layer = fdir_name.split("/")[-3] + + include_layers = d.getVar("CVE_CHECK_LAYER_INCLUDELIST").split() + exclude_layers = d.getVar("CVE_CHECK_LAYER_EXCLUDELIST").split() + + if exclude_layers and layer in exclude_layers: + return + + if include_layers and layer not in include_layers: + return + + product_data = [] + for s in cve_status: + p = {"product": s[0], "cvesInRecord": "Yes"} + if s[1] == False: + p["cvesInRecord"] = "No" + product_data.append(p) + product_data = list({p['product']:p for p in product_data}.values()) + + package_version = "%s%s" % (d.getVar("EXTENDPE"), d.getVar("PV")) + cpes = get_cpe_ids(d.getVar("CVE_PRODUCT"), d.getVar("CVE_VERSION")) + package_data = { + "name" : d.getVar("PN"), + "layer" : layer, + "version" : package_version, + "products": product_data, + "cpes": cpes + } + + cve_list = [] + + for cve in sorted(cve_data): + issue_link = "%s%s" % (nvd_link, cve) + + cve_item = { + "id" : cve, + "status" : cve_data[cve]["abbrev-status"], + "link": issue_link, + } + if 'NVD-summary' in cve_data[cve]: + cve_item["summary"] = cve_data[cve]["NVD-summary"] + cve_item["scorev2"] = cve_data[cve]["NVD-scorev2"] + cve_item["scorev3"] = cve_data[cve]["NVD-scorev3"] + cve_item["vector"] = cve_data[cve]["NVD-vector"] + cve_item["vectorString"] = cve_data[cve]["NVD-vectorString"] + if 'status' in cve_data[cve]: + cve_item["detail"] = cve_data[cve]["status"] + if 'justification' in cve_data[cve]: + cve_item["description"] = cve_data[cve]["justification"] + if 'resource' in cve_data[cve]: + cve_item["patch-file"] = cve_data[cve]["resource"] + cve_list.append(cve_item) + + package_data["issue"] = cve_list + output["package"].append(package_data) + + deploy_file = d.getVar("CVE_CHECK_RECIPE_FILE_JSON") + + write_string = json.dumps(output, indent=2) + + cvelogpath = d.getVar("CVE_CHECK_SUMMARY_DIR") + index_path = d.getVar("CVE_CHECK_SUMMARY_INDEX_PATH") + bb.utils.mkdirhier(cvelogpath) + fragment_file = os.path.basename(deploy_file) + fragment_path = os.path.join(cvelogpath, fragment_file) + with open(fragment_path, "w") as f: + f.write(write_string) + with open(index_path, "a+") as f: + f.write("%s\n" % fragment_path)
Signature · 1 line
2.43.0
PATCH 3/3 classes/kernel.bbclass: update CVE_PRODUCT
In reply to Marta RybczynskaAdd linux:linux to CVE_PRODUCT. linux:linux is used by the kernel CNA in raw CVE entries. We can't use just linux, because of conflicts with CPE entries of multiple distributions.
--- meta/classes-recipe/kernel.bbclass | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-)
meta/classes-recipe/kernel.bbclass +4 -1
@@ -21,7 +21,10 @@ PACKAGE_WRITE_DEPS += "depmodwrapper-cross" do_deploy[depends] += "depmodwrapper-cross:do_populate_sysroot gzip-native:do_populate_sysroot" do_clean[depends] += "make-mod-scripts:do_clean" -CVE_PRODUCT ?= "linux_kernel" +# CPE entries from NVD use linux_kernel, but the raw CVE entries from the kernel CNA have +# vendor: linux and product: linux. Note that multiple distributions use "linux" as a product +# name, so we need to fill vendor to avoid false positives +CVE_PRODUCT ?= "linux_kernel linux:linux" S = "${STAGING_KERNEL_DIR}" B = "${WORKDIR}/build"
Signature · 1 line
2.43.0
Hi Marta,
Thanks for the great work on this topic.
I have left 3 comments below.
Thanks for considering them.
Peter
Quoted 71 lines
-----Original Message-----
From: [email protected] <openembedded- [email protected]> On Behalf Of Marta Rybczynska via lists.openembedded.org
Sent: Monday, July 15, 2024 12:20
To: [email protected]
Cc: Marta Rybczynska <[email protected]>; Samantha Jalabert <[email protected]>
Subject: [OE-core] [PATCH 1/3] cve-check: enrich annotation of CVEsPreviously the information passed between different function of the cve-check class included only tables of patched, unpatched, ignored vulnerabilities and the general status of the recipe.
The VEX work requires more information, and we need to pass them between different functions, so that it can be enriched as the analysis progresses. Instead of multiple tables, use a single one with annotations for each CVE encountered. For example, a patched CVE will have:
{"abbrev-status": "Patched", "status": "version-not-in-range"}
abbrev-status contains the general status (Patched, Unpatched, Ignored and Unknown that will be added in the VEX code) status contains more detailed information that can come from
CVE_STATUSand the analysis.Additional fields of the annotation include for example the name of the patch file fixing a given CVE.
Signed-off-by Marta Rybczynska <[email protected]>Signed-off-by Samantha Jalabert <[email protected]>--- meta/classes/cve-check.bbclass | 208 +++++++++++++++++---------------- meta/lib/oe/cve_check.py | 12 +- 2 files changed, 113 insertions(+), 107 deletions(-)meta/classes/cve-check.bbclass +0
-2@@ -188,10 +188,10 @@ python do_cve_check () { patched_cves = get_patched_cves(d) except FileNotFoundError: bb.fatal("Failure in searching patches") - ignored, patched, unpatched, status = check_cves(d, patched_cves) - if patched or unpatched or (d.getVar("CVE_CHECK_COVERAGE") == "1"and status):
meta/classes/cve-check.bbclass +2
-2- cve_data = get_cve_info(d, patched + unpatched + ignored) - cve_write_data(d, patched, unpatched, ignored, cve_data, status) + cve_data, status = check_cves(d, patched_cves) + if len(cve_data) or (d.getVar("CVE_CHECK_COVERAGE") == "1" andstatus):
meta/classes/cve-check.bbclass +2
-0+ get_cve_info(d, cve_data) + cve_write_data(d, cve_data, status) else: bb.note("No CVE database found, skipping CVE check")@@ -294,7 +294,51 @@ ROOTFS_POSTPROCESS_COMMAND:prepend ="${@'cve_check_write_rootfs_manifest ' if d
do_rootfs[recrdeptask] += "${@'do_cve_check' ifd.getVar('CVE_CHECK_CREATE_MANIFEST')== '1' else ''}"
do_populate_sdk[recrdeptask] += "${@'do_cve_check' ifd.getVar('CVE_CHECK_CREATE_MANIFEST')== '1' else ''}"meta/classes/cve-check.bbclass +5
-1-def check_cves(d, patched_cves): +def cve_is_ignored(d, cve_data, cve): + if cve not in cve_data: + return False + if cve_data[cve]['abbrev-status'] == "Ignored": + return True
meta/classes/cve-check.bbclass ROOTFS_POSTPROCESS_COMMAND:prepend = "${@'cve_check_write_rootfs_manifest ' if d · line 311
Quoted 10 lines
meta/classes/cve-check.bbclass +10
-0+ return False + +def cve_is_patched(d, cve_data, cve): + if cve not in cve_data: + return False + if cve_data[cve]['abbrev-status'] == "Patched": + return True + return False + +def cve_update(d, cve_data, cve, entry):
I think that there is a fundamental change in behavior here.
Previously we were taking (NVD) DB as base and only vulnerable CVEs were compared annotated with CVE_STATUS or our presence of CVE patches. Now we take the CVE_STATUS and CVE patches as base and add entries from DB only if they were not annotated yet.
I am not arguing against it, I actually like it much more as we will be able to insert also CVEs not in DB into our reports.
But I have two comments on this:
* this should be explicitly described in commit message
* this makes global cve includes spill into all recipe reports, so this commit series should also get rid of cve-extra-exclusions.inc file finally (or at least add a comment into it with this sideeffect)
Quoted 100 lines
meta/classes/cve-check.bbclass +6
-0+ # If no entry, just add it + if cve not in cve_data: + cve_data[cve] = entry + return + # If we are updating, there might be change in the status + bb.debug("Trying CVE entry update for %s from %s to %s" % (cve,cve_data[cve]['abbrev-status'], entry['abbrev-status']))
meta/classes/cve-check.bbclass +6
-0+ if cve_data[cve]['abbrev-status'] == "Unknown": + cve_data[cve] = entry + return + if cve_data[cve]['abbrev-status'] == entry['abbrev-status']: + return + # Update like in {'abbrev-status': 'Patched', 'status': 'version-not-in-range'}to {'abbrev-status': 'Unpatched', 'status': 'version-in-range'}
+ if entry['abbrev-status'] == "Unpatched" and cve_data[cve]['abbrev-
status'] == "Patched":
+ if entry['status'] == "version-in-range" and cve_data[cve]['status'] ==
"version-not-in-range":meta/classes/cve-check.bbclass +3
-0+ # New result from the scan, vulnerable + cve_data[cve] = entry + bb.debug("CVE entry %s update from Patched to Unpatched from thescan result" % cve)
meta/classes/cve-check.bbclass +2
-0+ return + if entry['abbrev-status'] == "Patched" and cve_data[cve]['abbrev-status']== "Unpatched":
+ if entry['status'] == "version-not-in-range" and cve_data[cve]['status'] ==
"version-in-range":
+ # Range does not match the scan, but we already have a vulnerable
match, ignore
+ bb.debug("CVE entry %s update from Patched to Unpatched from the
scan result - not applying" % cve)meta/classes/cve-check.bbclass +6
-0+ return + # If we have an "Ignored", it has a priority + if cve_data[cve]['abbrev-status'] == "Ignored": + bb.debug("CVE %s not updating because Ignored" % cve) + return + bb.warn("Unhandled CVE entry update for %s from %s to %s" % (cve,cve_data[cve], entry))
meta/classes/cve-check.bbclass +4
-11+ +def check_cves(d, cve_data): """ Connect to the NVD database and find unpatched cves. """@@ -304,28 +348,19 @@ def check_cves(d, patched_cves): real_pv = d.getVar("PV") suffix = d.getVar("CVE_VERSION_SUFFIX") - cves_unpatched = [] - cves_ignored = [] cves_status = [] cves_in_recipe = False # CVE_PRODUCT can contain more than one product (eg. curl/libcurl) products = d.getVar("CVE_PRODUCT").split() # If this has been unset then we're not scanning for CVEs here (for example, image recipes) if not products: - return ([], [], [], []) + return ([], []) pv = d.getVar("CVE_VERSION").split("+git")[0] # If the recipe has been skipped/ignored we return empty lists if pn in d.getVar("CVE_CHECK_SKIP_RECIPE").split(): bb.note("Recipe has been skipped by cve-check") - return ([], [], [], []) - - # Convert CVE_STATUS into ignored CVEs and check validity - cve_ignore = [] - for cve in (d.getVarFlags("CVE_STATUS") or {}): - decoded_status, _, _ = decode_cve_status(d, cve) - if decoded_status == "Ignored": - cve_ignore.append(cve) + return ([], []) import sqlite3db_file = d.expand("file:${CVE_CHECK_DB_FILE}?mode=ro")meta/classes/cve-check.bbclass +2
-3@@ -344,11 +379,10 @@ def check_cves(d, patched_cves): for cverow in cve_cursor: cve = cverow[0] - if cve in cve_ignore: + if cve_is_ignored(d, cve_data, cve): bb.note("%s-%s ignores %s" % (product, pv, cve)) - cves_ignored.append(cve) continue - elif cve in patched_cves: + elif cve_is_patched(d, cve_data, cve): bb.note("%s has been patched" % (cve)) continue # Write status once only for each product@@ -364,7 +398,7 @@ def check_cves(d, patched_cves): for row in product_cursor: (_, _, _, version_start, operator_start, version_end, operator_end) =row
#bb.debug(2, "Evaluating row " + str(row))meta/classes/cve-check.bbclass +1
-1- if cve in cve_ignore: + if cve_is_ignored(d, cve_data, cve): ignored = True version_start = convert_cve_version(version_start)
meta/classes/cve-check.bbclass def check_cves(d, patched_cves): · line 440
Quoted 5 lines
meta/classes/cve-check.bbclass +1
-1@@ -403,16 +437,16 @@ def check_cves(d, patched_cves): if vulnerable: if ignored: bb.note("%s is ignored in %s-%s" % (cve, pn, real_pv)) - cves_ignored.append(cve) + cve_update(d, cve_data, cve, {"abbrev-status": "Ignored"})
I think that this case happens only when we ignore the CVE via CVE_STATUS. So calling this is not necessary as it is already prefilled from for it, correct? In case it would not be prefilled, then it would drop our enrichment...
meta/classes/cve-check.bbclass def check_cves(d, patched_cves):
Quoted 5 lines
meta/classes/cve-check.bbclass +1
-1else: bb.note("%s-%s is vulnerable to %s" % (pn, real_pv, cve)) - cves_unpatched.append(cve) + cve_update(d, cve_data, cve, {"abbrev-status": "Unpatched", "status": "version-in-range"})
Could you please include the new statuses in cve-check-map.conf
(e.g. version-in-range, version-not-in-range, fix-file-included)
I think it's important to have all possible statuses there to be able to correct the DB manually.
Quoted 299 lines
meta/classes/cve-check.bbclass +1
-1break product_cursor.close() if not vulnerable: bb.note("%s-%s is not vulnerable to %s" % (pn, real_pv, cve)) - patched_cves.add(cve) + cve_update(d, cve_data, cve, {"abbrev-status": "Patched", "status": "version-not-in-range"}) cve_cursor.close()if not
cves_in_product:meta/classes/cve-check.bbclass +0
-3@@ -420,48 +454,45 @@ def check_cves(d, patched_cves): cves_status.append([product, False]) conn.close() - diff_ignore = list(set(cve_ignore) - set(cves_ignored)) - if diff_ignore: - oe.qa.handle_error("cve_status_not_in_db", "Found CVE (%s) with
CVE_STATUSset that are not found in database for this component" % " ".join(diff_ignore), d)if not
cves_in_recipe:
bb.note("No CVE records for products in recipe %s" % (pn))- return (
list(cves_ignored),list(patched_cves),cves_unpatched,cves_status)meta/classes/cve-check.bbclass +15
-14+ return (cve_data, cves_status) -def get_cve_info(d, cves): +def get_cve_info(d, cve_data): """ Get CVE information from the database. """ import sqlite3 - cve_data = {} db_file = d.expand("file:${CVE_CHECK_DB_FILE}?mode=ro") conn = sqlite3.connect(db_file, uri=True) - for cve in cves: + for cve in cve_data: cursor = conn.execute("SELECT * FROM NVD WHERE ID IS ?", (cve,)) for row in cursor: - cve_data[row[0]] = {} - cve_data[row[0]]["summary"] = row[1] - cve_data[row[0]]["scorev2"] = row[2] - cve_data[row[0]]["scorev3"] = row[3] - cve_data[row[0]]["modified"] = row[4] - cve_data[row[0]]["vector"] = row[5] - cve_data[row[0]]["vectorString"] = row[6] + # The CVE itdelf has been added already + if row[0] not in cve_data: + bb.note("CVE record %s not present" % row[0]) + continue + #cve_data[row[0]] = {} + cve_data[row[0]]["NVD-summary"] = row[1] + cve_data[row[0]]["NVD-scorev2"] = row[2] + cve_data[row[0]]["NVD-scorev3"] = row[3] + cve_data[row[0]]["NVD-modified"] = row[4] + cve_data[row[0]]["NVD-vector"] = row[5] + cve_data[row[0]]["NVD-vectorString"] = row[6] cursor.close() conn.close() - return cve_data -def cve_write_data_text(d, patched, unpatched, ignored, cve_data): +def cve_write_data_text(d, cve_data): """ Write CVE information in WORKDIR; and to CVE_CHECK_DIR, and CVE manifest if enabled. """ - from oe.cve_check import decode_cve_status - cve_file = d.getVar("CVE_CHECK_LOG") fdir_name = d.getVar("FILE_DIRNAME") layer = fdir_name.split("/")[-3]@@ -478,7 +509,7 @@ def cve_write_data_text(d, patched, unpatched,ignored,
cve_data):
return# Early exit, the text format does not report packages without CVEs
meta/classes/cve-check.bbclass +1
-1- if not patched+unpatched+ignored: + if not len(cve_data): return nvd_link = "https://nvd.nist.gov/vuln/detail/"@@ -487,36 +518,27 @@ def cve_write_data_text(d, patched, unpatched,ignored,
cve_data):
bb.utils.mkdirhier(os.path.dirname(cve_file))for cve in
sorted(cve_data):meta/classes/cve-check.bbclass +0
-14- is_patched = cve in patched - is_ignored = cve in ignored - - status = "Unpatched" - if (is_patched or is_ignored) and not report_all: - continue - if is_ignored: - status = "Ignored" - elif is_patched: - status = "Patched" - else: - # default value of status is Unpatched - unpatched_cves.append(cve) - write_string += "LAYER: %s\n" % layer write_string += "PACKAGE NAME: %s\n" % d.getVar("PN") write_string += "PACKAGE VERSION: %s%s\n" % (d.getVar("EXTENDPE"),
d.getVar("PV"))write_string += "CVE: %s\n" % cvemeta/classes/cve-check.bbclass +6
-11- write_string += "CVE STATUS: %s\n" % status - _, detail, description = decode_cve_status(d, cve) - if detail: - write_string += "CVE DETAIL: %s\n" % detail - if description: - write_string += "CVE DESCRIPTION: %s\n" % description - write_string += "CVE SUMMARY: %s\n" % cve_data[cve]["summary"] - write_string += "CVSS v2 BASE SCORE: %s\n" % cve_data[cve]["scorev2"] - write_string += "CVSS v3 BASE SCORE: %s\n" % cve_data[cve]["scorev3"] - write_string += "VECTOR: %s\n" % cve_data[cve]["vector"] - write_string += "VECTORSTRING: %s\n" % cve_data[cve]["vectorString"] + write_string += "CVE STATUS: %s\n" % cve_data[cve]["abbrev-status"] + + if 'status' in cve_data[cve]: + write_string += "CVE DETAIL: %s\n" % cve_data[cve]["status"] + if 'justification' in cve_data[cve]: + write_string += "CVE DESCRIPTION: %s\n" %cve_data[cve]["justification"]
meta/classes/cve-check.bbclass +3
-0+ + if "NVD-summary" in cve_data[cve]: + write_string += "CVE SUMMARY: %s\n" % cve_data[cve]["NVD-summary"]
+write_string+= "CVSS v2 BASE SCORE: %s\n" % cve_data[cve]["NVD-
scorev2"]
+write_string+= "CVSS v3 BASE SCORE: %s\n" % cve_data[cve]["NVD-
scorev3"]meta/classes/cve-check.bbclass +2
-0+ write_string += "VECTOR: %s\n" % cve_data[cve]["NVD-vector"] + write_string += "VECTORSTRING: %s\n" % cve_data[cve]["NVD-vectorString"]
meta/classes/cve-check.bbclass +3
-0+ write_string += "MORE INFORMATION: %s%s\n\n" % (nvd_link, cve) + if cve_data[cve]["abbrev-status"] == "Unpatched": + unpatched_cves.append(cve) if unpatched_cves and d.getVar("CVE_CHECK_SHOW_WARNINGS") == "1": bb.warn("Found unpatched CVE (%s), for more information check %s" %(" ".join(unpatched_cves),cve_file))
meta/classes/cve-check.bbclass +0
-0@@ -568,13 +590,11 @@ def cve_check_write_json_output(d, output,
direct_file,deploy_file,manifest_fi
withopen(index_path, "a+")as f:
f.write("%s\n" % fragment_path)-def
cve_write_data_json(d, patched, unpatched, ignored, cve_data, cve_status):meta/classes/cve-check.bbclass +1
-2+def cve_write_data_json(d, cve_data, cve_status): """ Prepare CVE data for the JSON format, then write it. """ - from oe.cve_check import decode_cve_status - output = {"version":"1", "package": []} nvd_link = "https://nvd.nist.gov/vuln/detail/"@@ -592,8 +612,6 @@ def cve_write_data_json(d, patched, unpatched,ignored,
cve_data,cve_status):
ifinclude_layersand layer not ininclude_layers:
returnmeta/classes/cve-check.bbclass +0
-2- unpatched_cves = [] - product_data = [] for s in cve_status: p = {"product": s[0], "cvesInRecord": "Yes"}@@ -608,39 +626,29 @@ def cve_write_data_json(d, patched, unpatched,ignored,
cve_data,cve_status):
"version" :package_version,
"products":product_data
}meta/classes/cve-check.bbclass +15
-25+ cve_list = [] for cve in sorted(cve_data): - is_patched = cve in patched - is_ignored = cve in ignored - status = "Unpatched" - if (is_patched or is_ignored) and not report_all: - continue - if is_ignored: - status = "Ignored" - elif is_patched: - status = "Patched" - else: - # default value of status is Unpatched - unpatched_cves.append(cve) - issue_link = "%s%s" % (nvd_link, cve) cve_item = { "id" : cve, - "summary" : cve_data[cve]["summary"], - "scorev2" : cve_data[cve]["scorev2"], - "scorev3" : cve_data[cve]["scorev3"], - "vector" : cve_data[cve]["vector"], - "vectorString" : cve_data[cve]["vectorString"], - "status" : status, - "link": issue_link + "status" : cve_data[cve]["abbrev-status"], + "link": issue_link, } - _, detail, description = decode_cve_status(d, cve) - if detail: - cve_item["detail"] = detail - if description: - cve_item["description"] = description + if 'NVD-summary' in cve_data[cve]: + cve_item["summary"] = cve_data[cve]["NVD-summary"] + cve_item["scorev2"] = cve_data[cve]["NVD-scorev2"] + cve_item["scorev3"] = cve_data[cve]["NVD-scorev3"] + cve_item["vector"] = cve_data[cve]["NVD-vector"] + cve_item["vectorString"] = cve_data[cve]["NVD-vectorString"] + if 'status' in cve_data[cve]: + cve_item["detail"] = cve_data[cve]["status"] + if 'justification' in cve_data[cve]: + cve_item["description"] = cve_data[cve]["justification"] + if 'resource' in cve_data[cve]: + cve_item["patch-file"] = cve_data[cve]["resource"] cve_list.append(cve_item) package_data["issue"] = cve_list@@ -652,12 +660,12 @@ def cve_write_data_json(d, patched, unpatched,ignored,
cve_data,cve_status):cve_check_write_json_output(d, output,
direct_file,deploy_file,manifest_file)meta/classes/cve-check.bbclass +3
-3-def cve_write_data(d, patched, unpatched, ignored, cve_data, status): +def cve_write_data(d, cve_data, status): """ Write CVE data in each enabled format. """ if d.getVar("CVE_CHECK_FORMAT_TEXT") == "1": - cve_write_data_text(d, patched, unpatched, ignored, cve_data) + cve_write_data_text(d, cve_data) if d.getVar("CVE_CHECK_FORMAT_JSON") == "1": - cve_write_data_json(d, patched, unpatched, ignored, cve_data, status) + cve_write_data_json(d, cve_data, status)meta/lib/oe/cve_check.py +2
-2@@ -88,7 +88,7 @@ def get_patched_cves(d): # (cve_match regular expression) cve_file_name_match = re.compile(r".*(CVE-\d{4}-\d+)", re.IGNORECASE) - patched_cves = set() + patched_cves = {} patches = oe.patch.src_patches(d) bb.debug(2, "Scanning %d patches for CVEs" % len(patches)) for url in patches:@@ -98,7 +98,7 @@ def get_patched_cves(d): fname_match = cve_file_name_match.search(patch_file) if fname_match: cve = fname_match.group(1).upper() - patched_cves.add(cve) + patched_cves[cve] = {"abbrev-status": "Patched", "status": "fix-file- included", "resource": patch_file} bb.debug(2, "Found %s from patch file name %s" % (cve, patch_file))# Remote patches won't be present and compressed patches won't be
meta/lib/oe/cve_check.py +1
-1@@ -124,7 +124,7 @@ def get_patched_cves(d): cves = patch_text[match.start()+5:match.end()] for cve in cves.split(): bb.debug(2, "Patch %s solves %s" % (patch_file, cve)) - patched_cves.add(cve) + patched_cves[cve] = {"abbrev-status": "Patched", "status": "fix-file- included", "resource": patch_file} text_match = Trueif not
fname_matchand nottext_match:meta/lib/oe/cve_check.py +2
-5@@ -132,10 +132,8 @@ def get_patched_cves(d): # Search for additional patched CVEs for cve in (d.getVarFlags("CVE_STATUS") or {}): - decoded_status, _, _ = decode_cve_status(d, cve) - if decoded_status == "Patched": - bb.debug(2, "CVE %s is additionally patched" % cve) - patched_cves.add(cve) + decoded_status, detail, description = decode_cve_status(d, cve) + patched_cves[cve] = {"abbrev-status": decoded_status, "status": detail, "justification": description} return patched_cves --2.43.0
Quoted 93 lines · Peter
Hi Marta,
Thanks for the great work on this topic.
I have left 3 comments below.Thanks for considering them.
PeterQuoted 30 lines
-----Original Message-----
From: [email protected] <openembedded- [email protected]> On Behalf Of Marta Rybczynska via lists.openembedded.org
Sent: Monday, July 15, 2024 12:20
To: [email protected]
Cc: Marta Rybczynska <[email protected]>; Samantha Jalabert <[email protected]>
Subject: [OE-core] [PATCH 1/3] cve-check: enrich annotation of CVEsPreviously the information passed between different function of the cve-check class included only tables of patched, unpatched, ignored vulnerabilities and the general status of the recipe.
The VEX work requires more information, and we need to pass them between different functions, so that it can be enriched as the analysis progresses. Instead of multiple tables, use a single one with annotations for each CVE encountered. For example, a patched CVE will have:
{"abbrev-status": "Patched", "status": "version-not-in-range"}
abbrev-status contains the general status (Patched, Unpatched, Ignored and Unknown that will be added in the VEX code) status contains more detailed information that can come from
CVE_STATUSand the analysis.Additional fields of the annotation include for example the name of the patch file fixing a given CVE.
Review
Quoted 8 lines
Signed-off-by Marta Rybczynska <[email protected]>Signed-off-by Samantha Jalabert <[email protected]>--- meta/classes/cve-check.bbclass | 208 +++++++++++++++++---------------- meta/lib/oe/cve_check.py | 12 +- 2 files changed, 113 insertions(+), 107 deletions(-)(quoted diff) +0
-0b/meta/classes/cve-check.bbclass
meta/classes/cve-check.bbclass python do_cve_check () { · line 191
Quoted 8 lines
index
93a2a1413d..f177223568100644meta/classes/cve-check.bbclass +0
-1@@ -188,10 +188,10 @@ python do_cve_check () { patched_cves = get_patched_cves(d) except FileNotFoundError: bb.fatal("Failure in searching patches") - ignored, patched, unpatched, status = check_cves(d,
patched_cves)meta/classes/cve-check.bbclass python do_cve_check () { · line 192
Quoted 1 line
meta/classes/cve-check.bbclass +0
-1- if patched or unpatched or (d.getVar("CVE_CHECK_COVERAGE")== "1"
Quoted 2 lines
and status):
-cve_data= get_cve_info(d, patched + unpatched +ignored)
Quoted 1 line
- cve_write_data(d, patched, unpatched, ignored,
cve_data, status)meta/classes/cve-check.bbclass
Quoted 2 lines
meta/classes/cve-check.bbclass +2
-0+ cve_data, status = check_cves(d, patched_cves) + if len(cve_data) or (d.getVar("CVE_CHECK_COVERAGE") == "1"and
Quoted 27 lines
status):
meta/classes/cve-check.bbclass +2
-0+ get_cve_info(d, cve_data) + cve_write_data(d, cve_data, status) else: bb.note("No CVE database found, skipping CVE check")@@ -294,7 +294,51 @@ ROOTFS_POSTPROCESS_COMMAND:prepend ="${@'cve_check_write_rootfs_manifest ' if d
do_rootfs[recrdeptask] += "${@'do_cve_check' ifd.getVar('CVE_CHECK_CREATE_MANIFEST')== '1' else ''}"
do_populate_sdk[recrdeptask] += "${@'do_cve_check' ifd.getVar('CVE_CHECK_CREATE_MANIFEST')== '1' else ''}"meta/classes/cve-check.bbclass +13
-1-def check_cves(d, patched_cves): +def cve_is_ignored(d, cve_data, cve): + if cve not in cve_data: + return False + if cve_data[cve]['abbrev-status'] == "Ignored": + return True + return False + +def cve_is_patched(d, cve_data, cve): + if cve not in cve_data: + return False + if cve_data[cve]['abbrev-status'] == "Patched": + return True + return False
Quoted 9 lines
meta/classes/cve-check.bbclass
Quoted 2 lines
meta/classes/cve-check.bbclass +2
-0+ +def cve_update(d, cve_data, cve, entry):I think that there is a fundamental change in behavior here. Previously we were taking (NVD) DB as base and only vulnerable CVEs were compared annotated with
CVE_STATUSor our presence of CVE patches. Now we take theCVE_STATUSand CVE patches as base and add entries from DB only if they were not annotated yet.
This was a little more complicated than that. get_patched_cves() was taking a part of CVE_STATUS at the beginning of the process, then applying the NVD database.
The change is to import the totality and then update the status in the process. Now, the entries in CVE_STATUS had priority before, and they still have.
Now it is explicit, before it was hidden in the code. I do not see changes in the end result, do you have a case in mind?
Quoted 5 lines
I am not arguing against it, I actually like it much more as we will be able to insert also CVEs not in DB into our reports.
But I have two comments on this:
* this should be explicitly described in commit message
Yes, we can add it here. The logic will be documented more in the standalone tool, because with the CVE database, the update rule is even more complex.
(this is also good, it forces to write the list of priorities)
Quoted 4 lines
* this makes global cve includes spill into all recipe reports, so this commit series should also get rid of cve-extra-exclusions.inc file finally (or at least add a comment into it with this sideeffect)
This is the plan. I do not want to push a series removing cve-extra-inclusions.inc with this one, but maybe, as you suggest, we add a warning to this file and then remove with a separate series?
Quoted 128 lines
Quoted 8 lines
meta/classes/cve-check.bbclass +6
-0+ # If no entry, just add it + if cve not in cve_data: + cve_data[cve] = entry + return + # If we are updating, there might be change in the status + bb.debug("Trying CVE entry update for %s from %s to %s" % (cve,cve_data[cve]['abbrev-status'], entry['abbrev-status']))
meta/classes/cve-check.bbclass ROOTFS_POSTPROCESS_COMMAND:prepend = "${@'cve_check_write_rootfs_manifest ' if d
Quoted 6 lines
meta/classes/cve-check.bbclass +6
-0+ if cve_data[cve]['abbrev-status'] == "Unknown": + cve_data[cve] = entry + return + if cve_data[cve]['abbrev-status'] == entry['abbrev-status']: + return + # Update like in {'abbrev-status': 'Patched', 'status':'version-not-in-range'}
Quoted 4 lines
to {'abbrev-status': 'Unpatched', 'status': 'version-in-range'}
+ if entry['abbrev-status'] == "Unpatched" and cve_data[cve]['abbrev-
status'] == "Patched":
+ if entry['status'] == "version-in-range" andcve_data[cve]['status'] ==
meta/classes/cve-check.bbclass ROOTFS_POSTPROCESS_COMMAND:prepend = "${@'cve_check_write_rootfs_manifest ' if d
Quoted 4 lines
"version-not-in-range":
meta/classes/cve-check.bbclass +3
-0+ # New result from the scan, vulnerable + cve_data[cve] = entry + bb.debug("CVE entry %s update from Patched to Unpatchedfrom the
meta/classes/cve-check.bbclass
Quoted 3 lines
scan result" % cve)
meta/classes/cve-check.bbclass +2
-0+ return + if entry['abbrev-status'] == "Patched" andcve_data[cve]['abbrev-status']
Quoted 2 lines
== "Unpatched":
+ if entry['status'] == "version-not-in-range" andcve_data[cve]['status'] ==
Quoted 2 lines
"version-in-range":
+ # Range does not match the scan, but we already have avulnerable
Quoted 2 lines
match, ignore
+ bb.debug("CVE entry %s update from Patched to Unpatchedfrom the
Quoted 14 lines
scan result - not applying" % cve)
meta/classes/cve-check.bbclass +6
-0+ return + # If we have an "Ignored", it has a priority + if cve_data[cve]['abbrev-status'] == "Ignored": + bb.debug("CVE %s not updating because Ignored" % cve) + return + bb.warn("Unhandled CVE entry update for %s from %s to %s" % (cve,cve_data[cve], entry))
meta/classes/cve-check.bbclass +2
-0+ +def check_cves(d, cve_data): """ Connect to the NVD database and find unpatched cves. """meta/classes/cve-check.bbclass def check_cves(d, patched_cves): · line 355
Quoted 10 lines
meta/classes/cve-check.bbclass +0
-2@@ -304,28 +348,19 @@ def check_cves(d, patched_cves): real_pv = d.getVar("PV") suffix = d.getVar("CVE_VERSION_SUFFIX") - cves_unpatched = [] - cves_ignored = [] cves_status = [] cves_in_recipe = False # CVE_PRODUCT can contain more than one product (eg. curl/libcurl) products = d.getVar("CVE_PRODUCT").split() # If this has been unset then we're not scanning for CVEs here (forexample,
Quoted 36 lines
meta/classes/cve-check.bbclass +2
-9image recipes) if not products: - return ([], [], [], []) + return ([], []) pv = d.getVar("CVE_VERSION").split("+git")[0] # If the recipe has been skipped/ignored we return empty lists if pn in d.getVar("CVE_CHECK_SKIP_RECIPE").split(): bb.note("Recipe has been skipped by cve-check") - return ([], [], [], []) - - # Convert CVE_STATUS into ignored CVEs and check validity - cve_ignore = [] - for cve in (d.getVarFlags("CVE_STATUS") or {}): - decoded_status, _, _ = decode_cve_status(d, cve) - if decoded_status == "Ignored": - cve_ignore.append(cve) + return ([], []) import sqlite3db_file = d.expand("file:${CVE_CHECK_DB_FILE}?mode=ro")meta/classes/cve-check.bbclass +2
-3@@ -344,11 +379,10 @@ def check_cves(d, patched_cves): for cverow in cve_cursor: cve = cverow[0] - if cve in cve_ignore: + if cve_is_ignored(d, cve_data, cve): bb.note("%s-%s ignores %s" % (product, pv, cve)) - cves_ignored.append(cve) continue - elif cve in patched_cves: + elif cve_is_patched(d, cve_data, cve): bb.note("%s has been patched" % (cve)) continue # Write status once only for each productmeta/classes/cve-check.bbclass def check_cves(d, patched_cves): · line 399
Quoted 2 lines
meta/classes/cve-check.bbclass +0
-0@@ -364,7 +398,7 @@ def check_cves(d, patched_cves): for row in product_cursor: (_, _, _, version_start, operator_start, version_end,
operator_end) =meta/classes/cve-check.bbclass def check_cves(d, patched_cves): · line 439
Quoted 11 lines
row
#bb.debug(2, "Evaluating row " + str(row))meta/classes/cve-check.bbclass +1
-1- if cve in cve_ignore: + if cve_is_ignored(d, cve_data, cve): ignored = True version_start = convert_cve_version(version_start)@@ -403,16 +437,16 @@ def check_cves(d, patched_cves): if vulnerable: if ignored: bb.note("%s is ignored in %s-%s" % (cve, pn,
real_pv))meta/classes/cve-check.bbclass def check_cves(d, patched_cves): · line 440
Quoted 2 lines
meta/classes/cve-check.bbclass +1
-1- cves_ignored.append(cve) + cve_update(d, cve_data, cve, {"abbrev-status":"Ignored"})
I think that this case happens only when we ignore the CVE via
CVE_STATUS. So calling this is not necessary as it is already prefilled from for it, correct?
In case it would not be prefilled, then it would drop our enrichment...meta/classes/cve-check.bbclass def check_cves(d, patched_cves): · line 442
Quoted 2 lines
meta/classes/cve-check.bbclass +0
-0else: bb.note("%s-%s is vulnerable to %s" % (pn,
real_pv, cve))meta/classes/cve-check.bbclass def check_cves(d, patched_cves): · line 443
Quoted 2 lines
meta/classes/cve-check.bbclass +1
-1- cves_unpatched.append(cve) + cve_update(d, cve_data, cve, {"abbrev-status":
Quoted 8 lines
"Unpatched",
meta/classes/cve-check.bbclass def check_cves(d, patched_cves): · line 444
Quoted 1 line
meta/classes/cve-check.bbclass +0
-0"status": "version-in-range"})Could you please include the new statuses in cve-check-map.conf (e.g. version-in-range, version-not-in-range, fix-file-included) I think it's important to have all possible statuses there to be able to correct the DB manually.
Will add. There will be a bigger pass on the doc after it is all merged, because there's one additional state "Unknown".
Kind regards,
Marta
Quoted 8 lines
Quoted 4 lines
I think that there is a fundamental change in behavior here.
Previously we were taking (NVD) DB as base and only vulnerable CVEs were compared annotated withCVE_STATUSor our presence of CVE patches. Now we take theCVE_STATUSand CVE patches as base and add entries from DB only if they were not annotated yet.This was a little more complicated than that.
get_patched_cves()was taking a part ofCVE_STATUSat the beginning of the process, then applying the NVD database. The change is to import the totality and then update the status in the process. Now, the entries inCVE_STATUShad priority before, and they still have. Now it is explicit, before it was hidden in the code. I do not see changes in the end result, do you have a case in mind?
If with current master I add following to any recipe:
CVE_STATUS[2025-0001] = "not-applicable-config: test"
CVE_STATUS[2025-0002] = "fixed-version: test"
then the resulting build/tmp/log/cve/cve-summary.json which shows all CVEs for this recipe regardless of CVE status, it will NOT contain reference to these test entries. But when I apply your patch, they will be both added to the report.
So your code changes the behavior a lot (in a good direction from my point of view).
Quoted 11 lines
Quoted 5 lines
I am not arguing against it, I actually like it much more as we will be able to insert also CVEs not in DB into our reports. But I have two comments on this:
* this should be explicitly described in commit messageYes, we can add it here. The logic will be documented more in the standalone tool, because with the CVE database, the update rule is even more complex. (this is also good, it forces to write the list of priorities)
Quoted 1 line
* this makes global cve includes spill into all recipe reports, so this commit series should also get rid of cve-extra-exclusions.inc file finally (or at least add a comment into it with this sideeffect)
This is the plan. I do not want to push a series removing cve-extra-inclusions.inc with this one, but maybe, as you suggest, we add a warning to this file and then remove with a separate series?
Quoted 16 lines · Peter
Quoted 3 lines
Quoted 3 lines
I think that there is a fundamental change in behavior here. Previously we were taking (NVD) DB as base and only vulnerable CVEs
were compared annotated with
CVE_STATUSor our presence of CVE patches.Quoted 1 line
Quoted 1 line
Now we take the
CVE_STATUSand CVE patches as base and add entriesfrom DB only if they were not annotated yet.
Quoted 2 lines
This was a little more complicated than that.
get_patched_cves()wastaking a part of
CVE_STATUSat the beginning of the process, then applying the NVD database.Quoted 1 line
The change is to import the totality and then update the status in the
process. Now, the entries in
CVE_STATUShad priority before, and they still have.Quoted 1 line
Now it is explicit, before it was hidden in the code. I do not see
changes in the end result, do you have a case in mind?
Quoted 11 lines
If with current master I add following to any recipe:
CVE_STATUS[2025-0001] = "not-applicable-config: test" CVE_STATUS[2025-0002] = "fixed-version: test"
then the resultingbuild/tmp/log/cve/cve-summary.jsonwhich shows all CVEs for this recipe regardless of CVE status, it will NOT contain reference to these test entries.
But when I apply your patch, they will be both added to the report. So your code changes the behavior a lot (in a good direction from my point of view).
Hello Peter,
Got it. This is how I changed the description in the last version.
Kind regards,
Marta
Hello Marta,
This seems to break our tests:
2024-07-16 23:51:23,081 - oe-selftest - INFO - cve_check.CVECheck.test_recipe_report_json_ignored (subunit.RemotedTestCase)
2024-07-16 23:51:23,082 - oe-selftest - INFO - ... FAIL
Stderr:
2024-07-16 19:08:56,446 - oe-selftest - INFO - Adding: "include selftest.inc" in /home/pokybuild/yocto-worker/oe-selftest-fedora/build/build-st-157454/conf/local.conf
2024-07-16 19:08:56,446 - oe-selftest - INFO - Adding: "include bblayers.inc" in bblayers.conf
/usr/lib64/python3.12/unittest/case.py:580: RuntimeWarning: TestResult has no addDuration method
warnings.warn("TestResult has no addDuration method",
2024-07-16 23:51:23,086 - oe-selftest - INFO - 2: 5/45 357/595 (32.50s) (0 failed) (cve_check.CVECheck.test_recipe_report_json_ignored)
2024-07-16 23:51:23,086 - oe-selftest - INFO - testtools.testresult.real._StringException: Traceback (most recent call last):
File "/home/pokybuild/yocto-worker/oe-selftest-fedora/build/meta/lib/oeqa/selftest/cases/cve_check.py", line 240, in test_recipe_report_json_ignored
check_m4_json(summary_json)
File "/home/pokybuild/yocto-worker/oe-selftest-fedora/build/meta/lib/oeqa/selftest/cases/cve_check.py", line 222, in check_m4_json
self.assertEqual(len(found_cves["CVE-2011-1098"]["detail"]), 0)
File "/usr/lib64/python3.12/unittest/case.py", line 885, in assertEqual
assertion_func(first, second, msg=msg)
File "/usr/lib64/python3.12/unittest/case.py", line 878, in _baseAssertEqual
raise self.failureException(msg)AssertionError: 20 != 0
and
2024-07-16 23:52:46,285 - oe-selftest - INFO - cve_check.CVECheck.test_recipe_report_json_unpatched (subunit.RemotedTestCase)
2024-07-16 23:52:46,286 - oe-selftest - INFO - ... FAIL
Stderr:
2024-07-16 19:08:56,446 - oe-selftest - INFO - Adding: "include selftest.inc" in /home/pokybuild/yocto-worker/oe-selftest-fedora/build/build-st-157454/conf/local.conf
2024-07-16 19:08:56,446 - oe-selftest - INFO - Adding: "include bblayers.inc" in bblayers.conf
/usr/lib64/python3.12/unittest/case.py:580: RuntimeWarning: TestResult has no addDuration method
warnings.warn("TestResult has no addDuration method",
2024-07-16 23:52:46,286 - oe-selftest - INFO - 2: 6/45 367/595 (83.21s) (2 failed) (cve_check.CVECheck.test_recipe_report_json_unpatched)
2024-07-16 23:52:46,286 - oe-selftest - INFO - testtools.testresult.real._StringException: Traceback (most recent call last):
File "/home/pokybuild/yocto-worker/oe-selftest-fedora/build/meta/lib/oeqa/selftest/cases/cve_check.py", line 178, in test_recipe_report_json_unpatched
check_m4_json(summary_json)
File "/home/pokybuild/yocto-worker/oe-selftest-fedora/build/meta/lib/oeqa/selftest/cases/cve_check.py", line 175, in check_m4_json
self.assertEqual(package["issue"], [])
File "/usr/lib64/python3.12/unittest/case.py", line 885, in assertEqual
assertion_func(first, second, msg=msg)
File "/usr/lib64/python3.12/unittest/case.py", line 1091, in assertListEqual
self.assertSequenceEqual(list1, list2, msg, seq_type=list)
File "/usr/lib64/python3.12/unittest/case.py", line 1073, in assertSequenceEqual
self.fail(msg)
File "/usr/lib64/python3.12/unittest/case.py", line 715, in fail
raise self.failureException(msg)AssertionError: Lists differ: [{'id': 'CVE-2008-1687', 'status': 'Patche[975 chars]ge'}] != []
First list contains 2 additional elements.
First extra element 0:
{'id': 'CVE-2008-1687', 'status': 'Patched', 'link': 'https://nvd.nist.gov/vuln/detail/CVE-2008-1687', 'summary': 'The (1) maketemp and (2) mkstemp builtin functions in GNU m4 before 1.4.11 do not quote their output when a file is created, which might allow context-dependent attackers to trigger a macro expansion, leading to unspecified use of an incorrect filename.', 'scorev2': '7.5', 'scorev3': '0.0', 'vector': 'NETWORK', 'vectorString': 'AV:N/AC:L/Au:N/C:P/I:P/A:P', 'detail': 'version-not-in-range'} Diff is 1223 characters long. Set self.maxDiff to None to see it.
https://autobuilder.yoctoproject.org/typhoon/#/builders/86/builds/6985/steps/14/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/79/builds/6962/steps/15/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/80/builds/6919/steps/14/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/87/builds/6975/steps/14/logs/stdio
Quoted 451 lines · Marta Rybczynska
Previously the information passed between different function of the cve-check class included only tables of patched, unpatched, ignored vulnerabilities and the general status of the recipe.
The VEX work requires more information, and we need to pass them between different functions, so that it can be enriched as the analysis progresses. Instead of multiple tables, use a single one with annotations for each CVE encountered. For example, a patched CVE will have:
{"abbrev-status": "Patched", "status": "version-not-in-range"}
abbrev-status contains the general status (Patched, Unpatched, Ignored and Unknown that will be added in the VEX code) status contains more detailed information that can come from
CVE_STATUSand the analysis.Additional fields of the annotation include for example the name of the patch file fixing a given CVE.
Signed-off-by Marta Rybczynska <[email protected]>Signed-off-by Samantha Jalabert <[email protected]>--- meta/classes/cve-check.bbclass | 208 +++++++++++++++++---------------- meta/lib/oe/cve_check.py | 12 +- 2 files changed, 113 insertions(+), 107 deletions(-)meta/classes/cve-check.bbclass +108
-100@@ -188,10 +188,10 @@ python do_cve_check () { patched_cves = get_patched_cves(d) except FileNotFoundError: bb.fatal("Failure in searching patches") - ignored, patched, unpatched, status = check_cves(d, patched_cves) - if patched or unpatched or (d.getVar("CVE_CHECK_COVERAGE") == "1" and status): - cve_data = get_cve_info(d, patched + unpatched + ignored) - cve_write_data(d, patched, unpatched, ignored, cve_data, status) + cve_data, status = check_cves(d, patched_cves) + if len(cve_data) or (d.getVar("CVE_CHECK_COVERAGE") == "1" and status): + get_cve_info(d, cve_data) + cve_write_data(d, cve_data, status) else: bb.note("No CVE database found, skipping CVE check")@@ -294,7 +294,51 @@ ROOTFS_POSTPROCESS_COMMAND:prepend = "${@'cve_check_write_rootfs_manifest ' if d do_rootfs[recrdeptask] += "${@'do_cve_check' if d.getVar('CVE_CHECK_CREATE_MANIFEST') == '1' else ''}" do_populate_sdk[recrdeptask] += "${@'do_cve_check' if d.getVar('CVE_CHECK_CREATE_MANIFEST') == '1' else ''}" -def check_cves(d, patched_cves): +def cve_is_ignored(d, cve_data, cve): + if cve not in cve_data: + return False + if cve_data[cve]['abbrev-status'] == "Ignored": + return True + return False + +def cve_is_patched(d, cve_data, cve): + if cve not in cve_data: + return False + if cve_data[cve]['abbrev-status'] == "Patched": + return True + return False + +def cve_update(d, cve_data, cve, entry): + # If no entry, just add it + if cve not in cve_data: + cve_data[cve] = entry + return + # If we are updating, there might be change in the status + bb.debug("Trying CVE entry update for %s from %s to %s" % (cve, cve_data[cve]['abbrev-status'], entry['abbrev-status'])) + if cve_data[cve]['abbrev-status'] == "Unknown": + cve_data[cve] = entry + return + if cve_data[cve]['abbrev-status'] == entry['abbrev-status']: + return + # Update like in {'abbrev-status': 'Patched', 'status': 'version-not-in-range'} to {'abbrev-status': 'Unpatched', 'status': 'version-in-range'} + if entry['abbrev-status'] == "Unpatched" and cve_data[cve]['abbrev-status'] == "Patched": + if entry['status'] == "version-in-range" and cve_data[cve]['status'] == "version-not-in-range": + # New result from the scan, vulnerable + cve_data[cve] = entry + bb.debug("CVE entry %s update from Patched to Unpatched from the scan result" % cve) + return + if entry['abbrev-status'] == "Patched" and cve_data[cve]['abbrev-status'] == "Unpatched": + if entry['status'] == "version-not-in-range" and cve_data[cve]['status'] == "version-in-range": + # Range does not match the scan, but we already have a vulnerable match, ignore + bb.debug("CVE entry %s update from Patched to Unpatched from the scan result - not applying" % cve) + return + # If we have an "Ignored", it has a priority + if cve_data[cve]['abbrev-status'] == "Ignored": + bb.debug("CVE %s not updating because Ignored" % cve) + return + bb.warn("Unhandled CVE entry update for %s from %s to %s" % (cve, cve_data[cve], entry)) + +def check_cves(d, cve_data): """ Connect to the NVD database and find unpatched cves. """@@ -304,28 +348,19 @@ def check_cves(d, patched_cves): real_pv = d.getVar("PV") suffix = d.getVar("CVE_VERSION_SUFFIX") - cves_unpatched = [] - cves_ignored = [] cves_status = [] cves_in_recipe = False # CVE_PRODUCT can contain more than one product (eg. curl/libcurl) products = d.getVar("CVE_PRODUCT").split() # If this has been unset then we're not scanning for CVEs here (for example, image recipes) if not products: - return ([], [], [], []) + return ([], []) pv = d.getVar("CVE_VERSION").split("+git")[0] # If the recipe has been skipped/ignored we return empty lists if pn in d.getVar("CVE_CHECK_SKIP_RECIPE").split(): bb.note("Recipe has been skipped by cve-check") - return ([], [], [], []) - - # Convert CVE_STATUS into ignored CVEs and check validity - cve_ignore = [] - for cve in (d.getVarFlags("CVE_STATUS") or {}): - decoded_status, _, _ = decode_cve_status(d, cve) - if decoded_status == "Ignored": - cve_ignore.append(cve) + return ([], []) import sqlite3 db_file = d.expand("file:${CVE_CHECK_DB_FILE}?mode=ro")@@ -344,11 +379,10 @@ def check_cves(d, patched_cves): for cverow in cve_cursor: cve = cverow[0] - if cve in cve_ignore: + if cve_is_ignored(d, cve_data, cve): bb.note("%s-%s ignores %s" % (product, pv, cve)) - cves_ignored.append(cve) continue - elif cve in patched_cves: + elif cve_is_patched(d, cve_data, cve): bb.note("%s has been patched" % (cve)) continue # Write status once only for each product@@ -364,7 +398,7 @@ def check_cves(d, patched_cves): for row in product_cursor: (_, _, _, version_start, operator_start, version_end, operator_end) = row #bb.debug(2, "Evaluating row " + str(row)) - if cve in cve_ignore: + if cve_is_ignored(d, cve_data, cve): ignored = True version_start = convert_cve_version(version_start)@@ -403,16 +437,16 @@ def check_cves(d, patched_cves): if vulnerable: if ignored: bb.note("%s is ignored in %s-%s" % (cve, pn, real_pv)) - cves_ignored.append(cve) + cve_update(d, cve_data, cve, {"abbrev-status": "Ignored"}) else: bb.note("%s-%s is vulnerable to %s" % (pn, real_pv, cve)) - cves_unpatched.append(cve) + cve_update(d, cve_data, cve, {"abbrev-status": "Unpatched", "status": "version-in-range"}) break product_cursor.close() if not vulnerable: bb.note("%s-%s is not vulnerable to %s" % (pn, real_pv, cve)) - patched_cves.add(cve) + cve_update(d, cve_data, cve, {"abbrev-status": "Patched", "status": "version-not-in-range"}) cve_cursor.close() if not cves_in_product:@@ -420,48 +454,45 @@ def check_cves(d, patched_cves): cves_status.append([product, False]) conn.close() - diff_ignore = list(set(cve_ignore) - set(cves_ignored)) - if diff_ignore: - oe.qa.handle_error("cve_status_not_in_db", "Found CVE (%s) with CVE_STATUS set that are not found in database for this component" % " ".join(diff_ignore), d) if not cves_in_recipe: bb.note("No CVE records for products in recipe %s" % (pn)) - return (list(cves_ignored), list(patched_cves), cves_unpatched, cves_status) + return (cve_data, cves_status) -def get_cve_info(d, cves): +def get_cve_info(d, cve_data): """ Get CVE information from the database. """ import sqlite3 - cve_data = {} db_file = d.expand("file:${CVE_CHECK_DB_FILE}?mode=ro") conn = sqlite3.connect(db_file, uri=True) - for cve in cves: + for cve in cve_data: cursor = conn.execute("SELECT * FROM NVD WHERE ID IS ?", (cve,)) for row in cursor: - cve_data[row[0]] = {} - cve_data[row[0]]["summary"] = row[1] - cve_data[row[0]]["scorev2"] = row[2] - cve_data[row[0]]["scorev3"] = row[3] - cve_data[row[0]]["modified"] = row[4] - cve_data[row[0]]["vector"] = row[5] - cve_data[row[0]]["vectorString"] = row[6] + # The CVE itdelf has been added already + if row[0] not in cve_data: + bb.note("CVE record %s not present" % row[0]) + continue + #cve_data[row[0]] = {} + cve_data[row[0]]["NVD-summary"] = row[1] + cve_data[row[0]]["NVD-scorev2"] = row[2] + cve_data[row[0]]["NVD-scorev3"] = row[3] + cve_data[row[0]]["NVD-modified"] = row[4] + cve_data[row[0]]["NVD-vector"] = row[5] + cve_data[row[0]]["NVD-vectorString"] = row[6] cursor.close() conn.close() - return cve_data -def cve_write_data_text(d, patched, unpatched, ignored, cve_data): +def cve_write_data_text(d, cve_data): """ Write CVE information in WORKDIR; and to CVE_CHECK_DIR, and CVE manifest if enabled. """ - from oe.cve_check import decode_cve_status - cve_file = d.getVar("CVE_CHECK_LOG") fdir_name = d.getVar("FILE_DIRNAME") layer = fdir_name.split("/")[-3]@@ -478,7 +509,7 @@ def cve_write_data_text(d, patched, unpatched, ignored, cve_data): return # Early exit, the text format does not report packages without CVEs - if not patched+unpatched+ignored: + if not len(cve_data): return nvd_link = "https://nvd.nist.gov/vuln/detail/"@@ -487,36 +518,27 @@ def cve_write_data_text(d, patched, unpatched, ignored, cve_data): bb.utils.mkdirhier(os.path.dirname(cve_file)) for cve in sorted(cve_data): - is_patched = cve in patched - is_ignored = cve in ignored - - status = "Unpatched" - if (is_patched or is_ignored) and not report_all: - continue - if is_ignored: - status = "Ignored" - elif is_patched: - status = "Patched" - else: - # default value of status is Unpatched - unpatched_cves.append(cve) - write_string += "LAYER: %s\n" % layer write_string += "PACKAGE NAME: %s\n" % d.getVar("PN") write_string += "PACKAGE VERSION: %s%s\n" % (d.getVar("EXTENDPE"), d.getVar("PV")) write_string += "CVE: %s\n" % cve - write_string += "CVE STATUS: %s\n" % status - _, detail, description = decode_cve_status(d, cve) - if detail: - write_string += "CVE DETAIL: %s\n" % detail - if description: - write_string += "CVE DESCRIPTION: %s\n" % description - write_string += "CVE SUMMARY: %s\n" % cve_data[cve]["summary"] - write_string += "CVSS v2 BASE SCORE: %s\n" % cve_data[cve]["scorev2"] - write_string += "CVSS v3 BASE SCORE: %s\n" % cve_data[cve]["scorev3"] - write_string += "VECTOR: %s\n" % cve_data[cve]["vector"] - write_string += "VECTORSTRING: %s\n" % cve_data[cve]["vectorString"] + write_string += "CVE STATUS: %s\n" % cve_data[cve]["abbrev-status"] + + if 'status' in cve_data[cve]: + write_string += "CVE DETAIL: %s\n" % cve_data[cve]["status"] + if 'justification' in cve_data[cve]: + write_string += "CVE DESCRIPTION: %s\n" % cve_data[cve]["justification"] + + if "NVD-summary" in cve_data[cve]: + write_string += "CVE SUMMARY: %s\n" % cve_data[cve]["NVD-summary"] + write_string += "CVSS v2 BASE SCORE: %s\n" % cve_data[cve]["NVD-scorev2"] + write_string += "CVSS v3 BASE SCORE: %s\n" % cve_data[cve]["NVD-scorev3"] + write_string += "VECTOR: %s\n" % cve_data[cve]["NVD-vector"] + write_string += "VECTORSTRING: %s\n" % cve_data[cve]["NVD-vectorString"] + write_string += "MORE INFORMATION: %s%s\n\n" % (nvd_link, cve) + if cve_data[cve]["abbrev-status"] == "Unpatched": + unpatched_cves.append(cve) if unpatched_cves and d.getVar("CVE_CHECK_SHOW_WARNINGS") == "1": bb.warn("Found unpatched CVE (%s), for more information check %s" % (" ".join(unpatched_cves),cve_file))@@ -568,13 +590,11 @@ def cve_check_write_json_output(d, output, direct_file, deploy_file, manifest_fi with open(index_path, "a+") as f: f.write("%s\n" % fragment_path) -def cve_write_data_json(d, patched, unpatched, ignored, cve_data, cve_status): +def cve_write_data_json(d, cve_data, cve_status): """ Prepare CVE data for the JSON format, then write it. """ - from oe.cve_check import decode_cve_status - output = {"version":"1", "package": []} nvd_link = "https://nvd.nist.gov/vuln/detail/"@@ -592,8 +612,6 @@ def cve_write_data_json(d, patched, unpatched, ignored, cve_data, cve_status): if include_layers and layer not in include_layers: return - unpatched_cves = [] - product_data = [] for s in cve_status: p = {"product": s[0], "cvesInRecord": "Yes"}@@ -608,39 +626,29 @@ def cve_write_data_json(d, patched, unpatched, ignored, cve_data, cve_status): "version" : package_version, "products": product_data } + cve_list = [] for cve in sorted(cve_data): - is_patched = cve in patched - is_ignored = cve in ignored - status = "Unpatched" - if (is_patched or is_ignored) and not report_all: - continue - if is_ignored: - status = "Ignored" - elif is_patched: - status = "Patched" - else: - # default value of status is Unpatched - unpatched_cves.append(cve) - issue_link = "%s%s" % (nvd_link, cve) cve_item = { "id" : cve, - "summary" : cve_data[cve]["summary"], - "scorev2" : cve_data[cve]["scorev2"], - "scorev3" : cve_data[cve]["scorev3"], - "vector" : cve_data[cve]["vector"], - "vectorString" : cve_data[cve]["vectorString"], - "status" : status, - "link": issue_link + "status" : cve_data[cve]["abbrev-status"], + "link": issue_link, } - _, detail, description = decode_cve_status(d, cve) - if detail: - cve_item["detail"] = detail - if description: - cve_item["description"] = description + if 'NVD-summary' in cve_data[cve]: + cve_item["summary"] = cve_data[cve]["NVD-summary"] + cve_item["scorev2"] = cve_data[cve]["NVD-scorev2"] + cve_item["scorev3"] = cve_data[cve]["NVD-scorev3"] + cve_item["vector"] = cve_data[cve]["NVD-vector"] + cve_item["vectorString"] = cve_data[cve]["NVD-vectorString"] + if 'status' in cve_data[cve]: + cve_item["detail"] = cve_data[cve]["status"] + if 'justification' in cve_data[cve]: + cve_item["description"] = cve_data[cve]["justification"] + if 'resource' in cve_data[cve]: + cve_item["patch-file"] = cve_data[cve]["resource"] cve_list.append(cve_item) package_data["issue"] = cve_list@@ -652,12 +660,12 @@ def cve_write_data_json(d, patched, unpatched, ignored, cve_data, cve_status): cve_check_write_json_output(d, output, direct_file, deploy_file, manifest_file) -def cve_write_data(d, patched, unpatched, ignored, cve_data, status): +def cve_write_data(d, cve_data, status): """ Write CVE data in each enabled format. """ if d.getVar("CVE_CHECK_FORMAT_TEXT") == "1": - cve_write_data_text(d, patched, unpatched, ignored, cve_data) + cve_write_data_text(d, cve_data) if d.getVar("CVE_CHECK_FORMAT_JSON") == "1": - cve_write_data_json(d, patched, unpatched, ignored, cve_data, status) + cve_write_data_json(d, cve_data, status)meta/lib/oe/cve_check.py +5
-7@@ -88,7 +88,7 @@ def get_patched_cves(d): # (cve_match regular expression) cve_file_name_match = re.compile(r".*(CVE-\d{4}-\d+)", re.IGNORECASE) - patched_cves = set() + patched_cves = {} patches = oe.patch.src_patches(d) bb.debug(2, "Scanning %d patches for CVEs" % len(patches)) for url in patches:@@ -98,7 +98,7 @@ def get_patched_cves(d): fname_match = cve_file_name_match.search(patch_file) if fname_match: cve = fname_match.group(1).upper() - patched_cves.add(cve) + patched_cves[cve] = {"abbrev-status": "Patched", "status": "fix-file-included", "resource": patch_file} bb.debug(2, "Found %s from patch file name %s" % (cve, patch_file)) # Remote patches won't be present and compressed patches won't be@@ -124,7 +124,7 @@ def get_patched_cves(d): cves = patch_text[match.start()+5:match.end()] for cve in cves.split(): bb.debug(2, "Patch %s solves %s" % (patch_file, cve)) - patched_cves.add(cve) + patched_cves[cve] = {"abbrev-status": "Patched", "status": "fix-file-included", "resource": patch_file} text_match = True if not fname_match and not text_match:@@ -132,10 +132,8 @@ def get_patched_cves(d): # Search for additional patched CVEs for cve in (d.getVarFlags("CVE_STATUS") or {}): - decoded_status, _, _ = decode_cve_status(d, cve) - if decoded_status == "Patched": - bb.debug(2, "CVE %s is additionally patched" % cve) - patched_cves.add(cve) + decoded_status, detail, description = decode_cve_status(d, cve) + patched_cves[cve] = {"abbrev-status": decoded_status, "status": detail, "justification": description} return patched_cvesSignature · 11 lines
2.43.0 -=-=-=-=-=-=-=-=-=-=-=- Links: You receive all messages sent to this group. View/Reply Online (#201907): https://lists.openembedded.org/g/openembedded-core/message/201907 Mute This Topic: https://lists.openembedded.org/mt/107228576/3617179 Group Owner: [email protected] Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [[email protected]] -=-=-=-=-=-=-=-=-=-=-=-
Signature · 3 lines
Alexandre Belloni, co-owner and COO, Bootlin Embedded Linux and Kernel engineering https://bootlin.com
Quoted 5 lines · Alexandre Belloni
Hello Marta,
This seems to break our tests:
Hello Alexandre,
Thank you, we're on it.
Kind regards,
Marta