cve-check: add json format
PATCH 1/1 cve-check: add json format
Add an option to output the CVE check in a JSON-based format. This format is easier to parse in software than the original text-based one and allows post-processing by other tools.
Output formats are now handed by CVE_CHECK_FORMAT_TEXT and CVE_CHECK_FORMAT_JSON. Both of them are enabled by default.
The JSON output format gets generated in a similar way to the text format with the exception of the manifest: appending to JSON arrays requires parsing the file. Because of that we first write JSON fragments and then assemble them in one pass at the end.
--- meta/classes/cve-check.bbclass | 155 ++++++++++++++++++++++++++++++++- 1 file changed, 152 insertions(+), 3 deletions(-)
meta/classes/cve-check.bbclass +152 -3
@@ -34,15 +34,27 @@ CVE_CHECK_TMP_FILE ?= "${TMPDIR}/cve_check" CVE_CHECK_SUMMARY_DIR ?= "${LOG_DIR}/cve" CVE_CHECK_SUMMARY_FILE_NAME ?= "cve-summary" CVE_CHECK_SUMMARY_FILE ?= "${CVE_CHECK_SUMMARY_DIR}/${CVE_CHECK_SUMMARY_FILE_NAME}" +CVE_CHECK_SUMMARY_FILE_NAME_JSON = "cve-summary.json" +CVE_CHECK_SUMMARY_INDEX_PATH = "${CVE_CHECK_SUMMARY_DIR}/cve-summary-index.txt" + +CVE_CHECK_LOG_JSON ?= "${T}/cve.json" CVE_CHECK_DIR ??= "${DEPLOY_DIR}/cve" CVE_CHECK_RECIPE_FILE ?= "${CVE_CHECK_DIR}/${PN}" +CVE_CHECK_RECIPE_FILE_JSON ?= "${CVE_CHECK_DIR}/${PN}_cve.json" CVE_CHECK_MANIFEST ?= "${DEPLOY_DIR_IMAGE}/${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.cve" +CVE_CHECK_MANIFEST_JSON ?= "${DEPLOY_DIR_IMAGE}/${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.json" CVE_CHECK_COPY_FILES ??= "1" CVE_CHECK_CREATE_MANIFEST ??= "1" CVE_CHECK_REPORT_PATCHED ??= "1" +# Provide text output +CVE_CHECK_FORMAT_TEXT ??= "1" + +# Provide JSON output +CVE_CHECK_FORMAT_JSON ??= "1" + # Whitelist for packages (PN) CVE_CHECK_PN_WHITELIST ?= ""
@@ -120,6 +132,7 @@ python cve_check_cleanup () { Delete the file used to gather all the CVE information. """ bb.utils.remove(e.data.getVar("CVE_CHECK_TMP_FILE")) + bb.utils.remove(e.data.getVar("CVE_CHECK_SUMMARY_INDEX_PATH")) } addhandler cve_check_cleanup
@@ -136,6 +149,9 @@ python cve_check_write_rootfs_manifest () { deploy_file = d.getVar("CVE_CHECK_RECIPE_FILE") if os.path.exists(deploy_file): bb.utils.remove(deploy_file) + deploy_file_json = d.getVar("CVE_CHECK_RECIPE_FILE_JSON") + if os.path.exists(deploy_file_json): + bb.utils.remove(deploy_file_json) if os.path.exists(d.getVar("CVE_CHECK_TMP_FILE")): bb.note("Writing rootfs CVE manifest")
@@ -153,6 +169,35 @@ python cve_check_write_rootfs_manifest () { os.remove(manifest_link) os.symlink(os.path.basename(manifest_name), manifest_link) bb.plain("Image CVE report stored in: %s" % manifest_name) + + if os.path.exists(d.getVar("CVE_CHECK_SUMMARY_INDEX_PATH")): + import json + from jsonmerge import Merger + schema = { + "properties": { + "package": { + "mergeStrategy": "append" + } + } + } + merger = Merger(schema) + bb.note("Generating JSON CVE manifest") + deploy_dir = d.getVar("DEPLOY_DIR_IMAGE") + link_name = d.getVar("IMAGE_LINK_NAME") + manifest_name = d.getVar("CVE_CHECK_MANIFEST_JSON") + index_file = d.getVar("CVE_CHECK_SUMMARY_INDEX_PATH") + manifest = {} + with open(index_file) as f: + filename = f.readline() + while filename: + with open(filename.rstrip()) as j: + data = json.load(j) + manifest = merger.merge(manifest, data) + filename = f.readline() + + with open(manifest_name, "w") as f: + json.dump(manifest, f, indent=2) + bb.plain("Image CVE report stored in: %s" % manifest_name) } ROOTFS_POSTPROCESS_COMMAND:prepend = "${@'cve_check_write_rootfs_manifest; ' if d.getVar('CVE_CHECK_CREATE_MANIFEST') == '1' else ''}"
@@ -278,10 +323,10 @@ def get_cve_info(d, cves): conn.close() return cve_data -def cve_write_data(d, patched, unpatched, whitelisted, cve_data): +def cve_write_data_text(d, patched, unpatched, whitelisted, cve_data): """ - Write CVE information in WORKDIR; and to CVE_CHECK_DIR, and - CVE manifest if enabled. + Write CVE information in the text format: to WORKDIR; and to + CVE_CHECK_DIR, and CVE manifest if enabled. """
@@ -344,3 +389,107 @@ def cve_write_data(d, patched, unpatched, whitelisted, cve_data): with open(d.getVar("CVE_CHECK_TMP_FILE"), "a") as f: f.write("%s" % write_string) + +def cve_check_write_json_output(d, output, direct_file, deploy_file, manifest_file): + """ + Write CVE information in the JSON format: to WORKDIR; and to + CVE_CHECK_DIR, if CVE manifest if enabled, write fragment + files that will be assembled at the end in cve_check_write_rootfs_manifest. + """ + + import json + + write_string = json.dumps(output, indent=2) + with open(direct_file, "w") as f: + bb.note("Writing file %s with CVE information" % direct_file) + f.write(write_string) + + if d.getVar("CVE_CHECK_COPY_FILES") == "1": + bb.utils.mkdirhier(os.path.dirname(deploy_file)) + with open(deploy_file, "w") as f: + f.write(write_string) + + if d.getVar("CVE_CHECK_CREATE_MANIFEST") == "1": + cvelogpath = d.getVar("CVE_CHECK_SUMMARY_DIR") + index_path = d.getVar("CVE_CHECK_SUMMARY_INDEX_PATH") + bb.utils.mkdirhier(cvelogpath) + fragment_file = os.path.basename(deploy_file) + fragment_path = os.path.join(cvelogpath, fragment_file) + with open(fragment_path, "w") as f: + f.write(write_string) + with open(index_path, "a+") as f: + f.write("%s\n" % fragment_path) + +def cve_write_data_json(d, patched, unpatched, whitelisted, cve_data): + """ + Prepare CVE data for the JSON format, then write it. + """ + + output = {"version":"1", "package": []} + nvd_link = "https://nvd.nist.gov/vuln/detail/" + + fdir_name = d.getVar("FILE_DIRNAME") + layer = fdir_name.split("/")[-3] + + include_layers = d.getVar("CVE_CHECK_LAYER_INCLUDELIST").split() + exclude_layers = d.getVar("CVE_CHECK_LAYER_EXCLUDELIST").split() + + if exclude_layers and layer in exclude_layers: + return + + if include_layers and layer not in include_layers: + return + + unpatched_cves = [] + + package_version = "%s%s" % (d.getVar("EXTENDPE"), d.getVar("PV")) + package_data = { + "name" : d.getVar("PN"), + "layer" : layer, + "version" : package_version + } + cve_list = [] + + for cve in sorted(cve_data): + is_patched = cve in patched + status = "Unpatched" + + if cve in whitelisted: + status = "Skipped" + elif is_patched: + status = "Patched" + else: + # default value of status is Unpatched + unpatched_cves.append(cve) + + issue_link = "%s%s" % (nvd_link, cve) + + cve_item = { + "id" : cve, + "summary" : cve_data[cve]["summary"], + "scorev2" : cve_data[cve]["scorev2"], + "scorev3" : cve_data[cve]["scorev3"], + "vector" : cve_data[cve]["vector"], + "status" : status, + "link": issue_link + } + cve_list.append(cve_item) + + package_data["issue"] = cve_list + output["package"].append(package_data) + + direct_file = d.getVar("CVE_CHECK_LOG_JSON") + deploy_file = d.getVar("CVE_CHECK_RECIPE_FILE_JSON") + manifest_file = d.getVar("CVE_CHECK_SUMMARY_FILE_NAME_JSON") + + cve_check_write_json_output(d, output, direct_file, deploy_file, manifest_file) + +def cve_write_data(d, patched, unpatched, whitelisted, cve_data): + """ + Write CVE data in each enabled format. + """ + + if d.getVar("CVE_CHECK_FORMAT_TEXT") == "1": + cve_write_data_text(d, patched, unpatched, whitelisted, cve_data) + if d.getVar("CVE_CHECK_FORMAT_JSON") == "1": + cve_write_data_json(d, patched, unpatched, whitelisted, cve_data)
Signature · 1 line
2.33.0
Quoted 3 lines · Marta Rybczynska
Add an option to output the CVE check in a JSON-based format. This format is easier to parse in software than the original text-based one and allows post-processing by other tools.
Nice, thanks.
Quoted 1 line
+ from jsonmerge import Merger
This isn't part of the standard Python library, you'll have to replicate the logic.
One suggestion would be to move more of the logic into lib/oe/cve_check.py so the class doesn't get too big.
Ross
Quoted 7 lines · Ross Burton
Quoted 2 lines
+ from jsonmerge import Merger
This isn't part of the standard Python library, you'll have to replicate the logic.
Do you mean copying part of the class or reimplementing it?
Quoted 4 lines
One suggestion would be to move more of the logic into
lib/oe/cve_check.pyso the class doesn't get too big.This is possible.
Kind regards,
Marta
Quoted 4 lines · Marta Rybczynska
Quoted 2 lines
This isn't part of the standard Python library, you'll have to replicate the logic.
Do you mean copying part of the class or reimplementing it?
Reimplement, unless it's so complicated that we need to embed a copy. As this is just dictionary merging, I suspect its easier to just handcode it.
Ross