support/scripts/cve.py: switch to NVD JSON version 2.0
PATCH 1/1 support/scripts/cve.py: switch to NVD JSON version 2.0
The currently used feed is deprecated and will be retired by NVD in September 2023 [0].
The new API returns up to 2000 CVEs every 5 seconds (without API key) [1]. Instead of request individual years as with the feed, one can specify two timestamps are range. Any CVE changed in this time is returned. Therefore every single CVE is stored in a seperate JSON file. All fields returned by the API are saved for future use.
This results in over 200000 files grouped by year with ~800MiB total.
[0]: https://nvd.nist.gov/General/News/change-timeline
[1]: https://nvd.nist.gov/developers/start-here
--- support/scripts/cve.py | 208 +++++++++++++++++++++++++---------------- 1 file changed, 129 insertions(+), 79 deletions(-)
support/scripts/cve.py +129 -79
@@ -17,32 +17,21 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -import datetime +from datetime import datetime, timezone import os import requests # URL checking import distutils.version import time -import gzip import sys import operator - -try: - import ijson - # backend is a module in < 2.5, a string in >= 2.5 - if 'python' in getattr(ijson.backend, '__name__', ijson.backend): - try: - import ijson.backends.yajl2_cffi as ijson - except ImportError: - sys.stderr.write('Warning: Using slow ijson python backend\n') -except ImportError: - sys.stderr.write("You need ijson to parse NVD for CVE check\n") - exit(1) +import json sys.path.append('utils/') -NVD_START_YEAR = 2002 -NVD_JSON_VERSION = "1.1" -NVD_BASE_URL = "https://nvd.nist.gov/feeds/json/cve/" + NVD_JSON_VERSION +NVD_START_YEAR = 1999 +NVD_JSON_VERSION = "2.0" +NVD_BASE_URL = "https://services.nvd.nist.gov/rest/json/cves/" + NVD_JSON_VERSION +NVD_META_FILE = "nvdcve-%s-meta.json" % (NVD_JSON_VERSION) ops = { '>=': operator.ge,
@@ -82,69 +71,126 @@ class CVE: self.nvd_cve = nvd_cve @staticmethod - def download_nvd_year(nvd_path, year): - metaf = "nvdcve-%s-%s.meta" % (NVD_JSON_VERSION, year) - path_metaf = os.path.join(nvd_path, metaf) - jsonf_gz = "nvdcve-%s-%s.json.gz" % (NVD_JSON_VERSION, year) - path_jsonf_gz = os.path.join(nvd_path, jsonf_gz) - - # If the database file is less than a day old, we assume the NVD data - # locally available is recent enough. - if os.path.exists(path_jsonf_gz) and os.stat(path_jsonf_gz).st_mtime >= time.time() - 86400: - return path_jsonf_gz - - # If not, we download the meta file - url = "%s/%s" % (NVD_BASE_URL, metaf) - print("Getting %s" % url) - page_meta = requests.get(url) - page_meta.raise_for_status() - - # If the meta file already existed, we compare the existing - # one with the data newly downloaded. If they are different, - # we need to re-download the database. - # If the database does not exist locally, we need to redownload it in - # any case. - if os.path.exists(path_metaf) and os.path.exists(path_jsonf_gz): - meta_known = open(path_metaf, "r").read() - if page_meta.text == meta_known: - return path_jsonf_gz - - # Grab the compressed JSON NVD, and write files to disk - url = "%s/%s" % (NVD_BASE_URL, jsonf_gz) - print("Getting %s" % url) - page_json = requests.get(url) - page_json.raise_for_status() - open(path_jsonf_gz, "wb").write(page_json.content) - open(path_metaf, "w").write(page_meta.text) - return path_jsonf_gz + def write_cve(nvd_dir, vulnerability): + """ + Write the given CVE to a json file in nvd_dir with the + CVE as filename. Replace if file exists. + CVE-2020-0001 is saved in nvd_dir/2020/CVE-2020-0001.json. + """ + if vulnerability['cve']['vulnStatus'] == 'Rejected': + return + + cve_id = vulnerability['cve']['id'] + year = cve_id.split('-')[1] + folder = os.path.join(nvd_dir, year) + if not os.path.exists(folder): + os.makedirs(folder) + + filename = "%s.json" % (cve_id) + path = os.path.join(folder, filename) + with open(path, 'w') as f: + json.dump(vulnerability['cve'], f) + + @staticmethod + def fetch_updates(last_update, nvd_dir): + """ + Fetch all CVEs from NVD. If last_update is set, + run a delta update. When done write the meta json file. + """ + args = {} + start_index = 0 + total_results = 0 + results_per_page = 0 + timestamp = 0 + + print("Downloading new CVEs") + + if last_update != 0: + args['lastModStartDate'] = last_update + args['lastModEndDate'] = datetime.now().isoformat() + + while True: + args['startIndex'] = start_index + + # Even with the general sleep the API returns 503 or an empty + # response every now and again. + for attempt in range(5): + try: + page = requests.get(NVD_BASE_URL, params=args) + page.raise_for_status() + content = page.json() + except Exception: + time.sleep(6) + else: + break + + results_per_page = content['resultsPerPage'] + total_results = content['totalResults'] + start_index = content['startIndex'] + timestamp = content['timestamp'] + + for vulnerability in content['vulnerabilities']: + CVE.write_cve(nvd_dir, vulnerability) + + start_index += results_per_page + print("[%06d/%06d]" % (start_index, total_results)) + + if start_index >= total_results: + break + + # recommended by NVD to not hit rate limit + time.sleep(6) + + meta = {'version': NVD_JSON_VERSION, 'timestamp': timestamp} + meta_file = os.path.join(nvd_dir, NVD_META_FILE) + with open(meta_file, 'w') as f: + json.dump(meta, f) + + @staticmethod + def check_updates(nvd_dir): + """ + Check if NVD_META_FILE exists and determine the last + update. Skip updating if last request was less than 24 + hours ago. + """ + last_update = 0 + + meta_file = os.path.join(nvd_dir, NVD_META_FILE) + if os.path.exists(meta_file): + with open(meta_file, 'r') as f: + meta = json.load(f) + if meta['version'] == NVD_JSON_VERSION: + last_update = meta['timestamp'] + last_update_date = datetime.fromisoformat(last_update).replace(tzinfo=timezone.utc) + today = datetime.now(tz=timezone.utc) + delta = today - last_update_date + # NVD is only updated once a day + if delta.total_seconds() < 86400: + return + + CVE.fetch_updates(last_update, nvd_dir) @classmethod def read_nvd_dir(cls, nvd_dir): """ - Iterate over all the CVEs contained in NIST Vulnerability Database - feeds since NVD_START_YEAR. If the files are missing or outdated in - nvd_dir, a fresh copy will be downloaded, and kept in .json.gz + Check if NIST Vulnerability Database needs to be updated. + Afterwards load all JSON files containing CVEs since + NVD_START_YEAR. """ - for year in range(NVD_START_YEAR, datetime.datetime.now().year + 1): - filename = CVE.download_nvd_year(nvd_dir, year) - try: - content = ijson.items(gzip.GzipFile(filename), 'CVE_Items.item') - except: # noqa: E722 - print("ERROR: cannot read %s. Please remove the file then rerun this script" % filename) - raise - for cve in content: - yield cls(cve) - - def each_product(self): - """Iterate over each product section of this cve""" - for vendor in self.nvd_cve['cve']['affects']['vendor']['vendor_data']: - for product in vendor['product']['product_data']: - yield product + CVE.check_updates(nvd_dir) + + for year in range(NVD_START_YEAR, datetime.now().year + 1): + year_folder = os.path.join(nvd_dir, str(year)) + # sort files by CVE number within a year + for cve_file in sorted(os.listdir(year_folder), key=lambda f: int(f.split('.')[0].split('-')[-1])): + path = os.path.join(year_folder, cve_file) + with open(path, 'r') as f: + yield cls(json.load(f)) def parse_node(self, node): """ Parse the node inside the configurations section to extract the - cpe information usefull to know if a product is affected by + cpe information useful to know if a product is affected by the CVE. Actually only the product name and the version descriptor are needed, but we also provide the vendor name. """
@@ -155,11 +201,11 @@ class CVE: for parsed_node in self.parse_node(child): yield parsed_node - for cpe in node.get('cpe_match', ()): + for cpe in node.get('cpeMatch', ()): if not cpe['vulnerable']: return - product = cpe_product(cpe['cpe23Uri']) - version = cpe_version(cpe['cpe23Uri']) + product = cpe_product(cpe['criteria']) + version = cpe_version(cpe['criteria']) # ignore when product is '-', which means N/A if product == '-': return
@@ -191,7 +237,7 @@ class CVE: v_end = cpe['versionEndExcluding'] yield { - 'id': cpe['cpe23Uri'], + 'id': cpe['criteria'], 'v_start': v_start, 'op_start': op_start, 'v_end': v_end,
@@ -199,14 +245,18 @@ class CVE: } def each_cpe(self): - for node in self.nvd_cve['configurations']['nodes']: - for cpe in self.parse_node(node): - yield cpe + if 'configurations' not in self.nvd_cve: + return [] + + for config in self.nvd_cve['configurations']: + for node in config['nodes']: + for cpe in self.parse_node(node): + yield cpe @property def identifier(self): """The CVE unique identifier""" - return self.nvd_cve['cve']['CVE_data_meta']['ID'] + return self.nvd_cve['id'] @property def affected_products(self):
Signature · 6 lines
2.41.0 _______________________________________________ buildroot mailing list [email protected] https://lists.buildroot.org/mailman/listinfo/buildroot
Hello Daniel,
Quoted 9 lines · On Mon, 31 Jul 2023 22:14:20 +0200 Daniel Lang <[email protected]> wrote:
The currently used feed is deprecated and will be retired by NVD in September 2023 [0].
The new API returns up to 2000 CVEs every 5 seconds (without API key) [1]. Instead of request individual years as with the feed, one can specify two timestamps are range. Any CVE changed in this time is returned. Therefore every single CVE is stored in a seperate JSON file. All fields returned by the API are saved for future use.
This results in over 200000 files grouped by year with ~800MiB total.
Quoted 4 lines
[0]: https://nvd.nist.gov/General/News/change-timeline
[1]: https://nvd.nist.gov/developers/start-hereSigned-off-by Daniel Lang <[email protected]>
Wow, thanks for working on this! Is the storing of 200k files workable, or do we need to consider some other option like a local sqlite database or something?
Another question: did you do a run of "make pkg-stats" before and after your patch to compare the results in terms of CVEs reported for each Buildroot package?
Thomas
Signature · 7 lines
Thomas Petazzoni, co-owner and CEO, Bootlin Embedded Linux and Kernel engineering and training https://bootlin.com _______________________________________________ buildroot mailing list [email protected] https://lists.buildroot.org/mailman/listinfo/buildroot
Hello Thomas,
Quoted 14 lines · Thomas Petazzoni via buildroot
Hello Daniel,
Quoted 9 lines · On Mon, 31 Jul 2023 22:14:20 +0200 Daniel Lang <[email protected]> wrote:
The currently used feed is deprecated and will be retired by NVD in September 2023 [0].
The new API returns up to 2000 CVEs every 5 seconds (without API key) [1]. Instead of request individual years as with the feed, one can specify two timestamps are range. Any CVE changed in this time is returned. Therefore every single CVE is stored in a seperate JSON file. All fields returned by the API are saved for future use.
This results in over 200000 files grouped by year with ~800MiB total.
Quoted 8 lines
Quoted 4 lines
[0]: https://nvd.nist.gov/General/News/change-timeline
[1]: https://nvd.nist.gov/developers/start-hereSigned-off-by Daniel Lang <[email protected]>Wow, thanks for working on this! Is the storing of 200k files workable, or do we need to consider some other option like a local sqlite database or something?
From testing on my system I can say that it seems to be workable. Generating pkg-stats for all packages takes roughly the same time
old: ./support/scripts/pkg-stats --html old.html --nvd-path dl/buildroot-nvd/ --disable url,upstream,cpe 252,39s user 45,10s system 100% cpu 4:54,85 total
new: ./support/scripts/pkg-stats --html new.html --nvd-path dl/buildroot-nvd/ --disable url,upstream,cpe 250,04s user 46,24s system 100% cpu 4:53,72 total
I did consider a sqlite database given that that's the approach yocto uses. In the end I decided against it as I wasn't sure how future proof it would be. The current approach means that additional information (score, description,...) could be added or used for other purposes without having to download again. Whereas I thought I had to make a selection for the database.
In hindsight I could have just added a column for every information available.
If there is concern I can see with I have the time to also implement a database approach for comparison.
Not sure if updating would be faster with a database. It takes ~1.5 seconds on my system to save the batch of 2k CVEs to file. But I guess the main bottleneck is the API given that the initial download took upwards of 30 minutes during my test runs and only ~2.5 minutes are spend creating files.
Quoted 4 lines
Another question: did you do a run of "make pkg-stats" before and after your patch to compare the results in terms of CVEs reported for each Buildroot package?
I did. For a 1:1 comparison the sorting on line 185 has to be changed to for cve_file in sorted(os.listdir(year_folder)):
Otherwise CVEs within a package are sorted differently making a comparison very hard.
Running pkg-stats with this change generates identical reports:
diff old.html new.html57505c57505
< <p><i>Updated on 2023-08-01 07:34:14.594976, git commit 22e476d7886163484d233803b42a2a4c2b588a5b</i></p>
---
Quoted 1 line
<p><i>Updated on 2023-08-01 08:40:33.290711, git commit 22e476d7886163484d233803b42a2a4c2b588a5b</i></p>
Additional information about sorting:
Currently the sorting done by NVD when creating the tar.gz is used. NVDs sorts CVE-2023-10000 before CVE-2023-1000 which might not be ideal. The new implementation adds custom sorting as we would otherwise rely on the filesystem sorting.
Quoted 2 lines
Thomas
One final note: I'm in no way a python expert, so any optimization or general input is welcome.
Regards,
Daniel
_______________________________________________
buildroot mailing list
[email protected]
https://lists.buildroot.org/mailman/listinfo/buildroot
Hello Daniel,
Quoted 9 lines · On Tue, 1 Aug 2023 16:13:03 +0200 Daniel Lang <[email protected]> wrote:
Quoted 3 lines
Wow, thanks for working on this! Is the storing of 200k files workable, or do we need to consider some other option like a local sqlite database or something?
From testing on my system I can say that it seems to be workable. Generating pkg-stats for all packages takes roughly the same time
old: ./support/scripts/pkg-stats --html old.html --nvd-path dl/buildroot-nvd/ --disable url,upstream,cpe 252,39s user 45,10s system 100% cpu 4:54,85 total
new: ./support/scripts/pkg-stats --html new.html --nvd-path dl/buildroot-nvd/ --disable url,upstream,cpe 250,04s user 46,24s system 100% cpu 4:53,72 total
Nice!
I see you have --disable cpe. Is the CPE database unchanged on the NVD side?
Quoted 6 lines
I did consider a sqlite database given that that's the approach yocto uses. In the end I decided against it as I wasn't sure how future proof it would be. The current approach means that additional information (score, description,...) could be added or used for other purposes without having to download again. Whereas I thought I had to make a selection for the database.
In hindsight I could have just added a column for every information available.
I'm not sure if trying to map all fields of the JSON into sqlite fields would be relevant. In fact, we would only need some kind of key/value store, where the key is the CVE identifier, and the value is the JSON blob.
Quoted 2 lines
If there is concern I can see with I have the time to also implement a database approach for comparison.
Not sure it's needed for now. The filesystem is also a good database :-)
Quoted 4 lines
Not sure if updating would be faster with a database. It takes ~1.5 seconds on my system to save the batch of 2k CVEs to file. But I guess the main bottleneck is the API given that the initial download took upwards of 30 minutes during my test runs and only ~2.5 minutes are spend creating files.
OK.
Quoted 11 lines
I did. For a 1:1 comparison the sorting on line 185 has to be changed to for
cve_filein sorted(os.listdir(year_folder)):
Otherwise CVEs within a package are sorted differently making a comparison very hard.
Running pkg-stats with this change generates identical reports:diff old.html new.html
57505c57505
< <p><i>Updated on 2023-08-01 07:34:14.594976, git commit 22e476d7886163484d233803b42a2a4c2b588a5b</i></p>
---Quoted 1 line
<p><i>Updated on 2023-08-01 08:40:33.290711, git commit 22e476d7886163484d233803b42a2a4c2b588a5b</i></p>
Excellent.
Quoted 2 lines
One final note: I'm in no way a python expert, so any optimization or general input is welcome.
No problem, I'm also no a python expert at all :-)
Pending some feedback from you on the CPE question above, I think I'm going to do some quick testing of your proposal and push it.
Thanks!
Thomas
Signature · 7 lines
Thomas Petazzoni, co-owner and CEO, Bootlin Embedded Linux and Kernel engineering and training https://bootlin.com _______________________________________________ buildroot mailing list [email protected] https://lists.buildroot.org/mailman/listinfo/buildroot
Hello Thomas,
Quoted 12 lines · Thomas Petazzoni via buildroot
Hello Daniel,
Quoted 7 lines · On Tue, 1 Aug 2023 16:13:03 +0200 Daniel Lang <[email protected]> wrote:
Quoted 3 lines
Wow, thanks for working on this! Is the storing of 200k files workable, or do we need to consider some other option like a local sqlite database or something?
From testing on my system I can say that it seems to be workable. Generating pkg-stats for all packages takes roughly the same time
Quoted 7 lines
Quoted 2 lines
old: ./support/scripts/pkg-stats --html old.html --nvd-path dl/buildroot-nvd/ --disable url,upstream,cpe 252,39s user 45,10s system 100% cpu 4:54,85 total
new: ./support/scripts/pkg-stats --html new.html --nvd-path dl/buildroot-nvd/ --disable url,upstream,cpe 250,04s user 46,24s system 100% cpu 4:53,72 totalNice!
I see you have --disable cpe. Is the CPE database unchanged on the NVD side?
I disabled all parts that would request information online during execution to get a more predictable timing information.
To be honest I didn't check if the CPE feed is deprecated as well. But looking at the feed's website (https://nvd.nist.gov/products/cpe) it seems like it will be retired as well in favor of running delta updates via an API (https://nvd.nist.gov/developers/products).
The "problem" is that NVD currently tracks 1M CPEs and the API returns 10k at a time.
Saving them one by one into a file won't make much sense. Another idea is to put them in an array and save it as a big json file. When updating one has to make sure that entries are unique.
I will send a separate patch once I have that part updated as well.
Quoted 42 lines
Quoted 7 lines
I did consider a sqlite database given that that's the approach yocto uses. In the end I decided against it as I wasn't sure how future proof it would be. The current approach means that additional information (score, description,...) could be added or used for other purposes without having to download again. Whereas I thought I had to make a selection for the database.
In hindsight I could have just added a column for every information available.I'm not sure if trying to map all fields of the JSON into sqlite fields would be relevant. In fact, we would only need some kind of key/value store, where the key is the CVE identifier, and the value is the JSON blob.
Quoted 2 lines
If there is concern I can see with I have the time to also implement a database approach for comparison.
Not sure it's needed for now. The filesystem is also a good database :-)
Quoted 4 lines
Not sure if updating would be faster with a database. It takes ~1.5 seconds on my system to save the batch of 2k CVEs to file. But I guess the main bottleneck is the API given that the initial download took upwards of 30 minutes during my test runs and only ~2.5 minutes are spend creating files.
OK.
Quoted 11 lines
I did. For a 1:1 comparison the sorting on line 185 has to be changed to for
cve_filein sorted(os.listdir(year_folder)):
Otherwise CVEs within a package are sorted differently making a comparison very hard.
Running pkg-stats with this change generates identical reports:diff old.html new.html
57505c57505
< <p><i>Updated on 2023-08-01 07:34:14.594976, git commit 22e476d7886163484d233803b42a2a4c2b588a5b</i></p>
---Quoted 1 line
<p><i>Updated on 2023-08-01 08:40:33.290711, git commit 22e476d7886163484d233803b42a2a4c2b588a5b</i></p>
Excellent.
Quoted 2 lines
One final note: I'm in no way a python expert, so any optimization or general input is welcome.
Quoted 8 lines
No problem, I'm also no a python expert at all :-)
Pending some feedback from you on the CPE question above, I think I'm going to do some quick testing of your proposal and push it.
Thanks!
Thomas
_______________________________________________
buildroot mailing list
[email protected]
https://lists.buildroot.org/mailman/listinfo/buildroot
Hello Daniel,
Quoted 2 lines · On Tue, 1 Aug 2023 16:44:12 +0200 Daniel Lang <[email protected]> wrote:
I disabled all parts that would request information online during execution to get a more predictable timing information.
Sure.
Quoted 7 lines
To be honest I didn't check if the CPE feed is deprecated as well. But looking at the feed's website (https://nvd.nist.gov/products/cpe) it seems like it will be retired as well in favor of running delta updates via an API (https://nvd.nist.gov/developers/products).
The "problem" is that NVD currently tracks 1M CPEs and the API returns 10k at a time.
Gah. I'm not sure this API thing is really a win compared to the good old way of providing a database dump (JSON blob or anything similar).
Quoted 3 lines
Saving them one by one into a file won't make much sense. Another idea is to put them in an array and save it as a big json file. When updating one has to make sure that entries are unique.
How do you know which ones to retrieve? You can request them by timestamp or something, to ensure your local copy has all entries, without having to redownload them all?
Quoted 1 line
I will send a separate patch once I have that part updated as well.
Awesome, thanks a lot for working on this!
Thomas
Signature · 7 lines
Thomas Petazzoni, co-owner and CEO, Bootlin Embedded Linux and Kernel engineering and training https://bootlin.com _______________________________________________ buildroot mailing list [email protected] https://lists.buildroot.org/mailman/listinfo/buildroot
Hello Thomas,
Quoted 15 lines · Thomas Petazzoni via buildroot
Hello Daniel,
Quoted 2 lines · On Tue, 1 Aug 2023 16:44:12 +0200 Daniel Lang <[email protected]> wrote:
I disabled all parts that would request information online during execution to get a more predictable timing information.
Sure.
Quoted 5 lines
To be honest I didn't check if the CPE feed is deprecated as well. But looking at the feed's website (https://nvd.nist.gov/products/cpe) it seems like it will be retired as well in favor of running delta updates via an API (https://nvd.nist.gov/developers/products).
Quoted 5 lines
Quoted 2 lines
The "problem" is that NVD currently tracks 1M CPEs and the API returns 10k at a time.
Gah. I'm not sure this API thing is really a win compared to the good old way of providing a database dump (JSON blob or anything similar).
For our use case it would be a lot easier, I guess on their end the API makes thing a little more scalable.
Quoted 8 lines
Quoted 4 lines
Saving them one by one into a file won't make much sense. Another idea is to put them in an array and save it as a big json file. When updating one has to make sure that entries are unique.
How do you know which ones to retrieve? You can request them by timestamp or something, to ensure your local copy has all entries, without having to redownload them all?
Both APIs (CVE and CPE) are every similar.
NVD allow you to either download everything or query for entries that have been modified within a time window that you define. This way only a subset is downloaded on subsequent runs.
Quoted 4 lines
Quoted 2 lines
I will send a separate patch once I have that part updated as well.
Awesome, thanks a lot for working on this!
As the core logic/timing is the same for both, I'm thinking of moving that part into a reusable class and send a v2 for this patch together with the reworked CVE logic.
Therefore I will mark this patch as superseded even though the new version is still in the works.
Quoted 2 lines
Thomas
Regards,
Daniel
_______________________________________________
buildroot mailing list
[email protected]
https://lists.buildroot.org/mailman/listinfo/buildroot
Quoted 12 lines · Thomas Petazzoni via buildroot
Hello Daniel,
Quoted 7 lines · On Tue, 1 Aug 2023 16:13:03 +0200 Daniel Lang <[email protected]> wrote:
Quoted 3 lines
Wow, thanks for working on this! Is the storing of 200k files workable, or do we need to consider some other option like a local sqlite database or something?
From testing on my system I can say that it seems to be workable.
Generating pkg-stats for all packages takes roughly the same time
Quoted 7 lines
Quoted 2 lines
old: ./support/scripts/pkg-stats --html old.html --nvd-path dl/buildroot-nvd/ --disable url,upstream,cpe 252,39s user 45,10s system 100% cpu 4:54,85 total
new: ./support/scripts/pkg-stats --html new.html --nvd-path dl/buildroot-nvd/ --disable url,upstream,cpe 250,04s user 46,24s system 100% cpu 4:53,72 totalNice!
I see you have --disable cpe. Is the CPE database unchanged on the NVD side?
Using the CPE database is actually useless. I think we should drop it.
It actually doesn't matter at all if a CPE entry (including the version) is
found in the CPE database. If there's a CVE for it, then the entry will exist. But usually, the CVE will have a version range. In that case, we anyway match the version range without caring at all if the specific version exists in the CPE database or not.
So, I think we should just construct a CPE string and match it against the
CVEs, without consulting the CPE database at all.
It _does_ make sense to do a lookup in the CPE database for the CPE string,
but with * as the version part. This allows us to validate if the vendor/project/etc. are set correctly. But that's something we can do in individual API calls for each package, like we do for release-monitoring.
Regards,
Arnout
_______________________________________________
buildroot mailing list
[email protected]
https://lists.buildroot.org/mailman/listinfo/buildroot
Quoted 1 line · On Wed, 9 Aug 2023 22:31:11 +0200 Arnout Vandecappelle <[email protected]> wrote:
Using the CPE database is actually useless. I think we should drop it.
When I read this, I disagreed...
Quoted 6 lines
It actually doesn't matter at all if a CPE entry (including the version) is
found in the CPE database. If there's a CVE for it, then the entry will exist. But usually, the CVE will have a version range. In that case, we anyway match the version range without caring at all if the specific version exists in the CPE database or not.
Quoted 7 lines
So, I think we should just construct a CPE string and match it against the
CVEs, without consulting the CPE database at all.It _does_ make sense to do a lookup in the CPE database for the CPE string,
but with * as the version part. This allows us to validate if the vendor/project/etc. are set correctly. But that's something we can do in individual API calls for each package, like we do for release-monitoring.
... but then you say we should still use the CPE database, and I agree on the why we should use it: to have some reasonable certainty that the CPE ID we create in Buildroot for each package has a chance of matching the CPEs that will be associated to the CVEs that will perhaps one day be reported against this software package. So yes, perhaps we should just match in the CPE database with version set to '*', so that we don't care if the CPE database isn't aware of the latest releases of software packages, which it rarely is.
Thomas
Signature · 7 lines
Thomas Petazzoni, co-owner and CEO, Bootlin Embedded Linux and Kernel engineering and training https://bootlin.com _______________________________________________ buildroot mailing list [email protected] https://lists.buildroot.org/mailman/listinfo/buildroot
Quoted 15 lines · Thomas Petazzoni via buildroot
Quoted 1 line · On Wed, 9 Aug 2023 22:31:11 +0200 Arnout Vandecappelle <[email protected]> wrote:
Using the CPE database is actually useless. I think we should drop it.
When I read this, I disagreed...
Quoted 8 lines
It actually doesn't matter at all if a CPE entry (including the version) is
found in the CPE database. If there's a CVE for it, then the entry will exist. But usually, the CVE will have a version range. In that case, we anyway match the version range without caring at all if the specific version exists in the CPE database or not.So, I think we should just construct a CPE string and match it against the
CVEs, without consulting the CPE database at all.
That's what pkg-stats does anyway.
CPEs are checked separately, not for each CVE.
Quoted 5 lines
Quoted 5 lines
It _does_ make sense to do a lookup in the CPE database for the CPE string,
but with * as the version part. This allows us to validate if the vendor/project/etc. are set correctly. But that's something we can do in individual API calls for each package, like we do for release-monitoring.
The problem here is, that the new API (even the one for CPEs) constrains us to a 6 second timeout between requests [0]. We currently have ~700 packages with CPEs. This would come out to 4200 seconds or about 70 minutes, each time we run pkg-stats for all packages.
The only way around this is requesting an API key [1] which allows "50 requests in a rolling 30 seconds window". NVD still recommends to sleep in between requests...
Quoted 9 lines
... but then you say we should still use the CPE database, and I agree on the why we should use it: to have some reasonable certainty that the CPE ID we create in Buildroot for each package has a chance of matching the CPEs that will be associated to the CVEs that will perhaps one day be reported against this software package. So yes, perhaps we should just match in the CPE database with version set to '*', so that we don't care if the CPE database isn't aware of the latest releases of software packages, which it rarely is.
On that "latest release" note, we have a second, probably rarely used, use case for CPEs which is support/scripts/gen-missing-cpe. This script tries to generate a XML structure for each version that isn't registered in the database. For this script a lot of information about the CPE needs to be stored.
Side note here: I'm not sure that script will works correctly. When I tested it I got a file for polkit 122 that contained 0.120 as version.
Quoted 2 lines
Thomas
Regards
Daniel
[0]: https://nvd.nist.gov/developers/start-here
[1]: https://nvd.nist.gov/developers/request-an-api-key
_______________________________________________
buildroot mailing list
[email protected]
https://lists.buildroot.org/mailman/listinfo/buildroot
Hello Daniel,
Quoted 7 lines · On Thu, 10 Aug 2023 07:50:34 +0200 Daniel Lang <[email protected]> wrote:
The problem here is, that the new API (even the one for CPEs) constrains us to a 6 second timeout between requests [0]. We currently have ~700 packages with CPEs. This would come out to 4200 seconds or about 70 minutes, each time we run pkg-stats for all packages.
The only way around this is requesting an API key [1] which allows "50 requests in a rolling 30 seconds window". NVD still recommends to sleep in between requests...
Agreed, but what you do in the patch series you posted is just fine
IMO: you download the full CPE database, and then we locally check against it. Your last patch implements exactly what Arnout suggested:
to not check the full CPE including version number, but only the vendor/product.
Quoted 2 lines
On that "latest release" note, we have a second, probably rarely used, use case for CPEs which is support/scripts/gen-missing-cpe.
I'm not sure why you call that "second use-case".
Quoted 3 lines
This script tries to generate a XML structure for each version that isn't registered in the database. For this script a lot of information about the CPE needs to be stored.
The idea of this script was to be able to contribute new entries to the official CPE database, by generating the XML file that they require as input to contribute such new entries. I've never used it myself, and we would need to submit gazillions of new entries all the time to keep their CPE database up-to-date.
It could still be useful to have something to contribute new entries, for those packages that have no entry at all (regardless of their version number) in the CPE database.
Thomas
Signature · 7 lines
Thomas Petazzoni, co-owner and CEO, Bootlin Embedded Linux and Kernel engineering and training https://bootlin.com _______________________________________________ buildroot mailing list [email protected] https://lists.buildroot.org/mailman/listinfo/buildroot
Quoted 28 lines · Thomas Petazzoni
Hello Daniel,
Quoted 7 lines · On Thu, 10 Aug 2023 07:50:34 +0200 Daniel Lang <[email protected]> wrote:
The problem here is, that the new API (even the one for CPEs) constrains us to a 6 second timeout between requests [0]. We currently have ~700 packages with CPEs. This would come out to 4200 seconds or about 70 minutes, each time we run pkg-stats for all packages.
The only way around this is requesting an API key [1] which allows "50 requests in a rolling 30 seconds window". NVD still recommends to sleep in between requests...Agreed, but what you do in the patch series you posted is just fine
IMO: you download the full CPE database, and then we locally check against it. Your last patch implements exactly what Arnout suggested:
to not check the full CPE including version number, but only the vendor/product.Quoted 2 lines
On that "latest release" note, we have a second, probably rarely used, use case for CPEs which is support/scripts/gen-missing-cpe.
I'm not sure why you call that "second use-case".
Quoted 3 lines
This script tries to generate a XML structure for each version that isn't registered in the database. For this script a lot of information about the CPE needs to be stored.
Quoted 5 lines
The idea of this script was to be able to contribute new entries to the official CPE database, by generating the XML file that they require as input to contribute such new entries. I've never used it myself, and we would need to submit gazillions of new entries all the time to keep their CPE database up-to-date.
Indeed, for the same reasons I mentioned, it doesn't make any sense for us to
submit new CPE version entries.
Quoted 3 lines
It could still be useful to have something to contribute new entries, for those packages that have no entry at all (regardless of their version number) in the CPE database.
This makes no sense at all. The only reason to have a CPE database entry is in
order to link it to a CVE. If there is already a CVE, then it should already have a CPE entry. If there's no CVE yet, then will the first person to ever submit a CVE for it use the same ID?
So I would remove that script entirely.
Regards,
Arnout
_______________________________________________
buildroot mailing list
[email protected]
https://lists.buildroot.org/mailman/listinfo/buildroot
Quoted 8 lines · On Thu, 10 Aug 2023 15:18:42 +0200 Arnout Vandecappelle <[email protected]> wrote:
Quoted 3 lines
It could still be useful to have something to contribute new entries, for those packages that have no entry at all (regardless of their version number) in the CPE database.
This makes no sense at all. The only reason to have a CPE database entry is in
order to link it to a CVE. If there is already a CVE, then it should already have a CPE entry. If there's no CVE yet, then will the first person to ever submit a CVE for it use the same ID?
Well, that would be my expectation indeed. A package in Buildroot has no CPE in the database, no CVE. We submit a CPE to the NVD database. My hope (but perhaps I'm dreaming too much) is that the day there is a CVE on this software component that CPE identifier that was submitted will be used, and therefore our CVE tracking will work.
Maybe I'm dreaming here, but if it doesn't work like this, it basically means that for any package in Buildroot that never had any CVE, we have absolutely no guarantee that we will properly notice when the first CVE gets reported. Maybe that's life and we have to live with it, but it kinda sucks.
Thomas
Signature · 7 lines
Thomas Petazzoni, co-owner and CEO, Bootlin Embedded Linux and Kernel engineering and training https://bootlin.com _______________________________________________ buildroot mailing list [email protected] https://lists.buildroot.org/mailman/listinfo/buildroot
Quoted 12 lines · Thomas Petazzoni
Quoted 8 lines · On Thu, 10 Aug 2023 15:18:42 +0200 Arnout Vandecappelle <[email protected]> wrote:
Quoted 3 lines
It could still be useful to have something to contribute new entries, for those packages that have no entry at all (regardless of their version number) in the CPE database.
This makes no sense at all. The only reason to have a CPE database entry is in
order to link it to a CVE. If there is already a CVE, then it should already have a CPE entry. If there's no CVE yet, then will the first person to ever submit a CVE for it use the same ID?
Quoted 5 lines
Well, that would be my expectation indeed. A package in Buildroot has no CPE in the database, no CVE. We submit a CPE to the NVD database. My hope (but perhaps I'm dreaming too much) is that the day there is a CVE on this software component that CPE identifier that was submitted will be used, and therefore our CVE tracking will work.
There is a decent change that they would, I guess. On the other hand, I guess
it's more likely that the CVE's CPE entry would be created by the maintainer of the project, and I guess they would just assume that there is no CPE entry yet and create something, without searching for a pre-existing name. They may even search for it, see that it exists already, and decide that it's a conflict and choose a different name...
Quoted 5 lines
Maybe I'm dreaming here, but if it doesn't work like this, it basically means that for any package in Buildroot that never had any CVE, we have absolutely no guarantee that we will properly notice when the first CVE gets reported. Maybe that's life and we have to live with it, but it kinda sucks.
Yup. That's why I claimed in my EOSS-ELC talk that the CPE approach is broken :-)
Regards,
Arnout
_______________________________________________
buildroot mailing list
[email protected]
https://lists.buildroot.org/mailman/listinfo/buildroot
Quoted 11 lines · Arnout Vandecappelle
Quoted 8 lines · Thomas Petazzoni
On Thu, 10 Aug 2023 15:18:42 +0200
Arnout Vandecappelle <[email protected]> wrote:Maybe I'm dreaming here, but if it doesn't work like this, it basically means that for any package in Buildroot that never had any CVE, we have absolutely no guarantee that we will properly notice when the first CVE gets reported. Maybe that's life and we have to live with it, but it kinda sucks.
We still match against *:<pkg name> if the CPE is unknown. That's why CVE-2021-45464 is listed for kvmtool even though no CPE ID is known.
So there is hope, that a CVE get listed even if the package has no CPE ID listed.
Quoted 2 lines
Yup. That's why I claimed in my EOSS-ELC talk that the CPE approach is broken :-)
Working with the data, I have to the agree. There is a lot of inconsistency and weirdness.
The thing that's relevant for me in the current patchset [0]:
Do we want to keep gen-missing-cpe or drop it?
I'm asking because a lot of additional data needs to be stored to the database for gen-missing-cpe to work (title, reference urls...). Dropping that script would also mean simplifying the database, which is easier now than once transition happened.
Quoted 3 lines
Regards,
Arnout
Daniel
[0]: https://patchwork.ozlabs.org/project/buildroot/list/?series=368172
_______________________________________________
buildroot mailing list
[email protected]
https://lists.buildroot.org/mailman/listinfo/buildroot
Quoted 6 lines · On Thu, 10 Aug 2023 22:12:36 +0200 Daniel Lang <[email protected]> wrote:
The thing that's relevant for me in the current patchset [0]:
Do we want to keep gen-missing-cpe or drop it?
I'm asking because a lot of additional data needs to be stored to the database for gen-missing-cpe to work (title, reference urls...). Dropping that script would also mean simplifying the database, which is easier now than once transition happened.
On my side, I've never used it. In order to have all the entries in the CPE database kept up-to-date with the zillions of version bumps getting into Buildroot every day/week, there would be a massive work, I doubt anyone is ever going to do this. So to me, gen-missing-cpe can probably go away.
Thomas
Signature · 7 lines
Thomas Petazzoni, co-owner and CEO, Bootlin Embedded Linux and Kernel engineering and training https://bootlin.com _______________________________________________ buildroot mailing list [email protected] https://lists.buildroot.org/mailman/listinfo/buildroot
Quoted 10 lines · Thomas Petazzoni
Quoted 6 lines · On Thu, 10 Aug 2023 22:12:36 +0200 Daniel Lang <[email protected]> wrote:
The thing that's relevant for me in the current patchset [0]:
Do we want to keep gen-missing-cpe or drop it?
I'm asking because a lot of additional data needs to be stored to the database for gen-missing-cpe to work (title, reference urls...). Dropping that script would also mean simplifying the database, which is easier now than once transition happened.
Quoted 5 lines
On my side, I've never used it. In order to have all the entries in the CPE database kept up-to-date with the zillions of version bumps getting into Buildroot every day/week, there would be a massive work, I doubt anyone is ever going to do this. So to me, gen-missing-cpe can probably go away.
Just to be sure, I'm putting Matt in Cc who originally contributed this, but I
know that he is no longer working with Buildroot at all so I doubt he cares.
When it is removed, its support in Makefile needs to be removed as well (cfr.
commits fd7312940aeffe015592396a9b24e014839e0eb3 and 1f187371d002fb321a175233afddb50b2b6c3607).
Regards,
Arnout
_______________________________________________
buildroot mailing list
[email protected]
https://lists.buildroot.org/mailman/listinfo/buildroot
All,
Quoted 12 lines
From: Arnout Vandecappelle <[email protected]>
Sent: Friday, August 11, 2023 1:51 AM
To: Thomas Petazzoni <[email protected]>; Daniel Lang <[email protected]>
Cc: [email protected] <[email protected]>; [email protected] <[email protected]>; Weber, Matthew L > Collins <[email protected]>
Subject: [External] Re: [Buildroot] [PATCH]support/scripts/cve.py: switch to NVD JSON version 2.0On my side, I've never used it. In order to have all the entries in the CPE database kept up-to-date with the zillions of version bumps getting into Buildroot every day/week, there would be a massive work, I doubt anyone is ever going to do this. So to me, gen-missing-cpe can probably go away.
Quoted 6 lines
Just to be sure, I'm putting Matt in Cc who originally contributed this, but I
know that he is no longer working with Buildroot at all so I doubt he cares.When it is removed, its support in
Makefileneeds to be removed as well (cfr.
commitsfd7312940aeffe015592396a9b24e014839e0eb3and1f187371d002fb321a175233afddb50b2b6c3607).
Makes sense to drop it. There never got to be enough momentum in the overall software community to make CVE or even the new identifier really accurate. Thanks for CC'n me Arnout.
Regards,
Matt
_______________________________________________
buildroot mailing list
[email protected]
https://lists.buildroot.org/mailman/listinfo/buildroot
Hello Thomas,
Quoted 7 lines · Thomas Petazzoni
Hello Daniel,
On Thu, 10 Aug 2023 07:50:34 +0200
Daniel Lang <[email protected]> wrote:>> On that "latest release" note, we have a second, probably rarely used,Quoted 1 line
use case for CPEs which is support/scripts/gen-missing-cpe.
I'm not sure why you call that "second use-case".
Just wanted to point out, that we need the CPE database for more than pkg-stats.
However gen-missing-cpe needs more fields per CPE (title, references,...) whereas pkg-stats would do with just the CPE ID.
Quoted 10 lines
Quoted 4 lines
This script tries to generate a XML structure for each version that isn't registered in the database. For this script a lot of information about the CPE needs to be stored.
The idea of this script was to be able to contribute new entries to the official CPE database, by generating the XML file that they require as input to contribute such new entries. I've never used it myself, and we would need to submit gazillions of new entries all the time to keep their CPE database up-to-date.
I think I understand its purpose as that what I implemented in the updated version. However I'm not sure that's what the script is currently doing. It seems to me, that files with existing entries are generated and the replace logic doesn't work.
Quoted 4 lines
It could still be useful to have something to contribute new entries, for those packages that have no entry at all (regardless of their version number) in the CPE database.
This use case is not supported by gen-missing-cpe. For now it only generated xml files for packages that have older entries in the database.
Quoted 2 lines
Thomas
_______________________________________________
buildroot mailing list
[email protected]
https://lists.buildroot.org/mailman/listinfo/buildroot