cve-update-db-native: don't hardcode the database name
PATCH 1/6 cve-update-db-native: don't hardcode the database name
Don't hardcode the database filename, there's a variable for this in cve-check.bbclass.
--- meta/recipes-core/meta/cve-update-db-native.bb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)
meta/recipes-core/meta/cve-update-db-native.bb +2 -2
@@ -28,8 +28,8 @@ python do_populate_cve_db() { BASE_URL = "https://nvd.nist.gov/feeds/json/cve/1.0/nvdcve-1.0-" YEAR_START = 2002 - db_dir = os.path.join(d.getVar("DL_DIR"), 'CVE_CHECK') - db_file = os.path.join(db_dir, 'nvdcve_1.0.db') + db_file = d.getVar("CVE_CHECK_DB_FILE") + db_dir = os.path.dirname(db_file) json_tmpfile = os.path.join(db_dir, 'nvd.json.gz') # Don't refresh the database more than once an hour
Signature · 1 line
2.20.1
PATCH 2/6 cve-update-db-native: add an index on the CVE ID column
Create an index on the PRODUCTS table which contains a row for each CPE, drastically increasing the performance of lookups for a specific CVE.
--- meta/recipes-core/meta/cve-update-db-native.bb | 3 +++ 1 file changed, 3 insertions(+)
meta/recipes-core/meta/cve-update-db-native.bb +3 -0
@@ -120,11 +120,14 @@ python do_populate_cve_db() { def initialize_db(c): c.execute("CREATE TABLE IF NOT EXISTS META (YEAR INTEGER UNIQUE, DATE TEXT)") + c.execute("CREATE TABLE IF NOT EXISTS NVD (ID TEXT UNIQUE, SUMMARY TEXT, \ SCOREV2 TEXT, SCOREV3 TEXT, MODIFIED INTEGER, VECTOR TEXT)") + c.execute("CREATE TABLE IF NOT EXISTS PRODUCTS (ID TEXT, \ VENDOR TEXT, PRODUCT TEXT, VERSION_START TEXT, OPERATOR_START TEXT, \ VERSION_END TEXT, OPERATOR_END TEXT)") + c.execute("CREATE INDEX IF NOT EXISTS PRODUCT_ID_IDX on PRODUCTS(ID);") def parse_node_and_insert(c, node, cveId): # Parse children node if needed
Signature · 1 line
2.20.1
PATCH 3/6 cve-update-db-native: clean up proxy handling
In reply to Ross Burtonurllib handles adding proxy handlers if the proxies are set in the environment, so call bb.utils.export_proxies() to do that and remove the manual setup.
--- .../recipes-core/meta/cve-update-db-native.bb | 31 +++---------------- 1 file changed, 5 insertions(+), 26 deletions(-)
meta/recipes-core/meta/cve-update-db-native.bb +5 -26
@@ -21,10 +21,12 @@ python do_populate_cve_db() { """ Update NVD database with json data feed """ - + import bb.utils import sqlite3, urllib, urllib.parse, shutil, gzip from datetime import date + bb.utils.export_proxies(d) + BASE_URL = "https://nvd.nist.gov/feeds/json/cve/1.0/nvdcve-1.0-" YEAR_START = 2002
@@ -40,16 +42,6 @@ python do_populate_cve_db() { except OSError: pass - proxy = d.getVar("https_proxy") - if proxy: - # instantiate an opener but do not install it as the global - # opener unless if we're really sure it's applicable for all - # urllib requests - proxy_handler = urllib.request.ProxyHandler({'https': proxy}) - proxy_opener = urllib.request.build_opener(proxy_handler) - else: - proxy_opener = None - cve_f = open(os.path.join(d.getVar("TMPDIR"), 'cve_check'), 'a') if not os.path.isdir(db_dir):
@@ -67,15 +59,7 @@ python do_populate_cve_db() { json_url = year_url + ".json.gz" # Retrieve meta last modified date - - response = None - - if proxy_opener: - response = proxy_opener.open(meta_url) - else: - req = urllib.request.Request(meta_url) - response = urllib.request.urlopen(req) - + response = urllib.request.urlopen(meta_url) if response: for l in response.read().decode("utf-8").splitlines(): key, value = l.split(":", 1)
@@ -95,12 +79,7 @@ python do_populate_cve_db() { # Update db with current year json file try: - if proxy_opener: - response = proxy_opener.open(json_url) - else: - req = urllib.request.Request(json_url) - response = urllib.request.urlopen(req) - + response = urllib.request.urlopen(json_url) if response: update_db(c, gzip.decompress(response.read()).decode('utf-8')) c.execute("insert or replace into META values (?, ?)", [year, last_modified])
Signature · 1 line
2.20.1
Re: [PATCH 3/6] cve-update-db-native: clean up proxy handling
Quoted 15 lines · Ross Burton
urllib handles adding proxy handlers if the proxies are set in the environment, so call
bb.utils.export_proxies()to do that and remove the manual setup.Signed-off-by Ross Burton <[email protected]>--- .../recipes-core/meta/cve-update-db-native.bb | 31 +++---------------- 1 file changed, 5 insertions(+), 26 deletions(-)meta/recipes-core/meta/cve-update-db-native.bb +0
-0@@ -21,10 +21,12 @@ python do_populate_cve_db() { """ Update NVD database with json data feed
meta/recipes-core/meta/cve-update-db-native.bb python do_populate_cve_db() { · line 31
Quoted 10 lines
meta/recipes-core/meta/cve-update-db-native.bb +3
-1""" - + import bb.utils import sqlite3, urllib, urllib.parse, shutil, gzip from datetime import date + bb.utils.export_proxies(d) + BASE_URL = "https://nvd.nist.gov/feeds/json/cve/1.0/nvdcve-1.0-" YEAR_START = 2002
Two comments, I know unrelated to this specific commit, but I noticed them while looking...
The current NVD data is now in the '1.1' format. I was lead to believe the 1.0 feeds would be stopped at some point.
Second, if we're successful with some of the SRTool components, we should be able to export the data into NVD format. So in that case, it would be nice to be able to point the cve-update components to an alternative datasource. (I do assume the data format is the same.)
--Mark
Quoted 49 lines
meta/recipes-core/meta/cve-update-db-native.bb +2
-25@@ -40,16 +42,6 @@ python do_populate_cve_db() { except OSError: pass - proxy = d.getVar("https_proxy") - if proxy: - # instantiate an opener but do not install it as the global - # opener unless if we're really sure it's applicable for all - # urllib requests - proxy_handler = urllib.request.ProxyHandler({'https': proxy}) - proxy_opener = urllib.request.build_opener(proxy_handler) - else: - proxy_opener = None - cve_f = open(os.path.join(d.getVar("TMPDIR"), 'cve_check'), 'a') if not os.path.isdir(db_dir):@@ -67,15 +59,7 @@ python do_populate_cve_db() { json_url = year_url + ".json.gz" # Retrieve meta last modified date - - response = None - - if proxy_opener: - response = proxy_opener.open(meta_url) - else: - req = urllib.request.Request(meta_url) - response = urllib.request.urlopen(req) - + response = urllib.request.urlopen(meta_url) if response: for l in response.read().decode("utf-8").splitlines(): key, value = l.split(":", 1)@@ -95,12 +79,7 @@ python do_populate_cve_db() { # Update db with current year json file try: - if proxy_opener: - response = proxy_opener.open(json_url) - else: - req = urllib.request.Request(json_url) - response = urllib.request.urlopen(req) - + response = urllib.request.urlopen(json_url) if response: update_db(c, gzip.decompress(response.read()).decode('utf-8')) c.execute("insert or replace into META values (?, ?)", [year, last_modified])
PATCH 4/6 cve-check: rewrite look to fix false negatives
In reply to Ross BurtonA previous optimisation was premature and resulted in false-negatives in the report.
Rewrite the checking algorithm to first get the list of potential CVEs by vendor:product, then iterate through every matching CPE for that CVE to determine if the bounds match or not. By doing this in two stages we can know if we've checked every CPE, instead of accidentally breaking out of the scan too early.
--- meta/classes/cve-check.bbclass | 63 ++++++++++++++++++---------------- 1 file changed, 34 insertions(+), 29 deletions(-)
meta/classes/cve-check.bbclass +34 -29
@@ -165,7 +165,6 @@ def check_cves(d, patched_cves): """ Connect to the NVD database and find unpatched cves. """ - import ast, csv, tempfile, subprocess, io from distutils.version import LooseVersion cves_unpatched = []
@@ -187,68 +186,74 @@ def check_cves(d, patched_cves): cve_whitelist = d.getVar("CVE_CHECK_WHITELIST").split() import sqlite3 - db_file = d.getVar("CVE_CHECK_DB_FILE") - conn = sqlite3.connect(db_file) + db_file = d.expand("file:${CVE_CHECK_DB_FILE}?mode=ro") + conn = sqlite3.connect(db_file, uri=True) + # For each of the known product names (e.g. curl has CPEs using curl and libcurl)... for product in products: - c = conn.cursor() if ":" in product: vendor, product = product.split(":", 1) - c.execute("SELECT * FROM PRODUCTS WHERE PRODUCT IS ? AND VENDOR IS ?", (product, vendor)) else: - c.execute("SELECT * FROM PRODUCTS WHERE PRODUCT IS ?", (product,)) + vendor = "%" - for row in c: - cve = row[0] - version_start = row[3] - operator_start = row[4] - version_end = row[5] - operator_end = row[6] + # Find all relevant CVE IDs. + for cverow in conn.execute("SELECT DISTINCT ID FROM PRODUCTS WHERE PRODUCT IS ? AND VENDOR LIKE ?", (product, vendor)): + cve = cverow[0] if cve in cve_whitelist: bb.note("%s-%s has been whitelisted for %s" % (product, pv, cve)) # TODO: this should be in the report as 'whitelisted' patched_cves.add(cve) + continue elif cve in patched_cves: bb.note("%s has been patched" % (cve)) - else: - to_append = False + continue + + vulnerable = False + for row in conn.execute("SELECT * FROM PRODUCTS WHERE ID IS ? AND PRODUCT IS ? AND VENDOR LIKE ?", (cve, product, vendor)): + (_, _, _, version_start, operator_start, version_end, operator_end) = row + #bb.debug(2, "Evaluating row " + str(row)) + if (operator_start == '=' and pv == version_start): - to_append = True + vulnerable = True else: if operator_start: try: - to_append_start = (operator_start == '>=' and LooseVersion(pv) >= LooseVersion(version_start)) - to_append_start |= (operator_start == '>' and LooseVersion(pv) > LooseVersion(version_start)) + vulnerable_start = (operator_start == '>=' and LooseVersion(pv) >= LooseVersion(version_start)) + vulnerable_start |= (operator_start == '>' and LooseVersion(pv) > LooseVersion(version_start)) except: bb.warn("%s: Failed to compare %s %s %s for %s" % (product, pv, operator_start, version_start, cve)) - to_append_start = False + vulnerable_start = False else: - to_append_start = False + vulnerable_start = False if operator_end: try: - to_append_end = (operator_end == '<=' and LooseVersion(pv) <= LooseVersion(version_end)) - to_append_end |= (operator_end == '<' and LooseVersion(pv) < LooseVersion(version_end)) + vulnerable_end = (operator_end == '<=' and LooseVersion(pv) <= LooseVersion(version_end)) + vulnerable_end |= (operator_end == '<' and LooseVersion(pv) < LooseVersion(version_end)) except: bb.warn("%s: Failed to compare %s %s %s for %s" % (product, pv, operator_end, version_end, cve)) - to_append_end = False + vulnerable_end = False else: - to_append_end = False + vulnerable_end = False if operator_start and operator_end: - to_append = to_append_start and to_append_end + vulnerable = vulnerable_start and vulnerable_end else: - to_append = to_append_start or to_append_end + vulnerable = vulnerable_start or vulnerable_end - if to_append: + if vulnerable: bb.note("%s-%s is vulnerable to %s" % (product, pv, cve)) cves_unpatched.append(cve) - else: - bb.note("%s-%s is not vulnerable to %s" % (product, pv, cve)) - patched_cves.add(cve) + break + + if not vulnerable: + bb.note("%s-%s is not vulnerable to %s" % (product, pv, cve)) + # TODO: not patched but not vulnerable + patched_cves.add(cve) + conn.close() return (list(patched_cves), cves_unpatched)
Signature · 1 line
2.20.1
PATCH 5/6 cve-check: neaten get_cve_info
In reply to Ross BurtonRemove obsolete Python 2 code, and use convenience methods for neatness.
--- meta/classes/cve-check.bbclass | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-)
meta/classes/cve-check.bbclass +5 -13
@@ -261,23 +261,15 @@ def check_cves(d, patched_cves): def get_cve_info(d, cves): """ Get CVE information from the database. - - Unfortunately the only way to get CVE info is set the output to - html (hard to parse) or query directly the database. """ - try: - import sqlite3 - except ImportError: - from pysqlite2 import dbapi2 as sqlite3 + import sqlite3 cve_data = {} - db_file = d.getVar("CVE_CHECK_DB_FILE") - placeholder = ",".join("?" * len(cves)) - query = "SELECT * FROM NVD WHERE id IN (%s)" % placeholder - conn = sqlite3.connect(db_file) - cur = conn.cursor() - for row in cur.execute(query, tuple(cves)): + conn = sqlite3.connect(d.getVar("CVE_CHECK_DB_FILE")) + placeholders = ",".join("?" * len(cves)) + query = "SELECT * FROM NVD WHERE id IN (%s)" % placeholders + for row in conn.execute(query, tuple(cves)): cve_data[row[0]] = {} cve_data[row[0]]["summary"] = row[1] cve_data[row[0]]["scorev2"] = row[2]
Signature · 1 line
2.20.1
PATCH 6/6 cve-check: fetch CVE data once at a time instead of in a single call
In reply to Ross BurtonThis code used to construct a single SQL statement that fetched the NVD data for every CVE requested. For recipes such as the kernel where there are over 2000 CVEs to report this can hit the variable count limit and the query fails with "sqlite3.OperationalError: too many SQL variables". The default limit is 999 variables, but some distributions such as Debian set the default to 250000.
As the NVD table has an index on the ID column, whilst requesting the data CVE-by-CVE is five times slower when working with 2000 CVEs the absolute time different is insignificant: 0.05s verses 0.01s on my machine.
--- meta/classes/cve-check.bbclass | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-)
meta/classes/cve-check.bbclass +10 -10
@@ -267,17 +267,17 @@ def get_cve_info(d, cves): cve_data = {} conn = sqlite3.connect(d.getVar("CVE_CHECK_DB_FILE")) - placeholders = ",".join("?" * len(cves)) - query = "SELECT * FROM NVD WHERE id IN (%s)" % placeholders - for row in conn.execute(query, tuple(cves)): - cve_data[row[0]] = {} - cve_data[row[0]]["summary"] = row[1] - cve_data[row[0]]["scorev2"] = row[2] - cve_data[row[0]]["scorev3"] = row[3] - cve_data[row[0]]["modified"] = row[4] - cve_data[row[0]]["vector"] = row[5] - conn.close() + for cve in cves: + for row in conn.execute("SELECT * FROM NVD WHERE ID IS ?", (cve,)): + cve_data[row[0]] = {} + cve_data[row[0]]["summary"] = row[1] + cve_data[row[0]]["scorev2"] = row[2] + cve_data[row[0]]["scorev3"] = row[3] + cve_data[row[0]]["modified"] = row[4] + cve_data[row[0]]["vector"] = row[5] + + conn.close() return cve_data def cve_write_data(d, patched, unpatched, cve_data):
Signature · 1 line
2.20.1
Quoted 4 lines · Ross Burton
Don't hardcode the database filename, there's a variable for this in cve-check.bbclass.
Signed-off-by Ross Burton <[email protected]>
do you recommend these being backported ?
- armin
Quoted 19 lines
--- meta/recipes-core/meta/cve-update-db-native.bb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)meta/recipes-core/meta/cve-update-db-native.bb +2
-2@@ -28,8 +28,8 @@ python do_populate_cve_db() { BASE_URL = "https://nvd.nist.gov/feeds/json/cve/1.0/nvdcve-1.0-" YEAR_START = 2002 - db_dir = os.path.join(d.getVar("DL_DIR"), 'CVE_CHECK') - db_file = os.path.join(db_dir, 'nvdcve_1.0.db') + db_file = d.getVar("CVE_CHECK_DB_FILE") + db_dir = os.path.dirname(db_file) json_tmpfile = os.path.join(db_dir, 'nvd.json.gz') # Don't refresh the database more than once an hour
Quoted 9 lines · On 18/11/2019 19:11, akuster808 wrote:
Quoted 4 lines · Ross Burton
Don't hardcode the database filename, there's a variable for this in cve-check.bbclass.
Signed-off-by Ross Burton <[email protected]>do you recommend these being backported ?
Yes, that series has some important fixes in.
Ross