Pull request
cover Pull request
Fixes to improve the local CVE database reliability. Even if the branch
is EOL, the CVE metrics is the only part we maintain.
Should fix 15660 – [kirkstone, scarthgap] AB-INT: cve-update-nvd2-native-1.0-r0 do_fetch: CVE database update failed
https://bugzilla.yoctoproject.org/show_bug.cgi?id=15660
The following changes since commit c4fb7d1f0f157ffafd9f907f49ea74b93b2c1d40:
build-appliance-image: Update to dunfell head revision (2024-04-15 19:14:34 -0700)
are available in the Git repository at:
https://git.openembedded.org/openembedded-core-contrib stable/dunfell-next
http://cgit.openembedded.org/openembedded-core-contrib/log/?h=stable/dunfell-next
Paul Barker (1):
cve-update: Avoid NFS caching issues
Richard Purdie (1):
cve_check: Use a local copy of the database during builds
meta/classes/cve-check.bbclass | 7 ++--- .../recipes-core/meta/cve-update-db-native.bb | 9 +++++-- .../meta/cve-update-nvd2-native.bb | 27 ++++++++++++++----- 3 files changed, 31 insertions(+), 12 deletions(-)
PATCH 1/2 cve_check: Use a local copy of the database during builds
From: Richard Purdie <[email protected]>
Rtaher than trying to use a sqlite database over NFS from DL_DIR, work from a local copy in STAGING DIR after fetching.
--- meta/classes/cve-check.bbclass | 7 ++++--- .../meta/cve-update-nvd2-native.bb | 18 +++++++++++++----- 2 files changed, 17 insertions(+), 8 deletions(-)
meta/classes/cve-check.bbclass +4 -3
@@ -25,8 +25,9 @@ CVE_PRODUCT ??= "${BPN}" CVE_VERSION ??= "${PV}" -CVE_CHECK_DB_DIR ?= "${DL_DIR}/CVE_CHECK" -CVE_CHECK_DB_FILE ?= "${CVE_CHECK_DB_DIR}/nvdcve_2.db" +CVE_CHECK_DB_FILENAME ?= "nvdcve_2.db" +CVE_CHECK_DB_DIR ?= "${STAGING_DIR}/CVE_CHECK" +CVE_CHECK_DB_FILE ?= "${CVE_CHECK_DB_DIR}/${CVE_CHECK_DB_FILENAME}" CVE_CHECK_DB_FILE_LOCK ?= "${CVE_CHECK_DB_FILE}.lock" CVE_CHECK_LOG ?= "${T}/cve.log"
@@ -156,7 +157,7 @@ python do_cve_check () { } addtask cve_check before do_build -do_cve_check[depends] = "cve-update-nvd2-native:do_fetch" +do_cve_check[depends] = "cve-update-nvd2-native:do_unpack" do_cve_check[nostamp] = "1" python cve_check_cleanup () {
meta/recipes-core/meta/cve-update-nvd2-native.bb +13 -5
@@ -8,7 +8,6 @@ INHIBIT_DEFAULT_DEPS = "1" inherit native -deltask do_unpack deltask do_patch deltask do_configure deltask do_compile
@@ -35,7 +34,9 @@ CVE_DB_INCR_UPDATE_AGE_THRES ?= "10368000" # Number of attempts for each http query to nvd server before giving up CVE_DB_UPDATE_ATTEMPTS ?= "5" -CVE_DB_TEMP_FILE ?= "${CVE_CHECK_DB_DIR}/temp_nvdcve_2.db" +CVE_CHECK_DB_DLDIR_FILE ?= "${DL_DIR}/CVE_CHECK/${CVE_CHECK_DB_FILENAME}" +CVE_CHECK_DB_DLDIR_LOCK ?= "${CVE_CHECK_DB_DLDIR_FILE}.lock" +CVE_CHECK_DB_TEMP_FILE ?= "${CVE_CHECK_DB_FILE}.tmp" python () { if not bb.data.inherits_class("cve-check", d):
@@ -52,9 +53,9 @@ python do_fetch() { bb.utils.export_proxies(d) - db_file = d.getVar("CVE_CHECK_DB_FILE") + db_file = d.getVar("CVE_CHECK_DB_DLDIR_FILE") db_dir = os.path.dirname(db_file) - db_tmp_file = d.getVar("CVE_DB_TEMP_FILE") + db_tmp_file = d.getVar("CVE_CHECK_DB_TEMP_FILE") cleanup_db_download(db_file, db_tmp_file) # By default let's update the whole database (since time 0)
@@ -77,6 +78,7 @@ python do_fetch() { pass bb.utils.mkdirhier(db_dir) + bb.utils.mkdirhier(os.path.dirname(db_tmp_file)) if os.path.exists(db_file): shutil.copy2(db_file, db_tmp_file)
@@ -89,10 +91,16 @@ python do_fetch() { os.remove(db_tmp_file) } -do_fetch[lockfiles] += "${CVE_CHECK_DB_FILE_LOCK}" +do_fetch[lockfiles] += "${CVE_CHECK_DB_DLDIR_LOCK}" do_fetch[file-checksums] = "" do_fetch[vardeps] = "" +python do_unpack() { + import shutil + shutil.copyfile(d.getVar("CVE_CHECK_DB_DLDIR_FILE"), d.getVar("CVE_CHECK_DB_FILE")) +} +do_unpack[lockfiles] += "${CVE_CHECK_DB_DLDIR_LOCK} ${CVE_CHECK_DB_FILE_LOCK}" + def cleanup_db_download(db_file, db_tmp_file): """ Cleanup the download space from possible failed downloads
PATCH 2/2 cve-update: Avoid NFS caching issues
In reply to Yoann CongalFrom: Paul Barker <[email protected]>
When moving the updated CVE database file to the downloads directory, ensure that it has a different inode number to the previous version of this file.
We have seen "sqlite3.DatabaseError: database disk image is malformed" exceptions on our autobuilder when trying to read the CVE database in do_cve_check tasks. The context here is that the downloads directory (where the updated database file is copied to) is shared between workers as an NFS mount. Different autobuilder workers were seeing different checksums for the database file, which indicates that a mix of both new and stale data was being read. Forcing each new version of the database file to have a different inode number will prevent stale data from being read from local caches.
This should fix [YOCTO #16086].
--- meta/recipes-core/meta/cve-update-db-native.bb | 9 +++++++-- meta/recipes-core/meta/cve-update-nvd2-native.bb | 9 +++++++-- 2 files changed, 14 insertions(+), 4 deletions(-)
meta/recipes-core/meta/cve-update-db-native.bb +7 -2
@@ -63,8 +63,13 @@ python do_fetch() { shutil.copy2(db_file, db_tmp_file) if update_db_file(db_tmp_file, d) == True: - # Update downloaded correctly, can swap files - shutil.move(db_tmp_file, db_file) + # Update downloaded correctly, we can swap files. To avoid potential + # NFS caching issues, ensure that the destination file has a new inode + # number. We do this in two steps as the downloads directory may be on + # a different filesystem to tmpdir we're working in. + new_file = "%s.new" % (db_file) + shutil.move(db_tmp_file, new_file) + os.rename(new_file, db_file) else: # Update failed, do not modify the database bb.note("CVE database update failed")
meta/recipes-core/meta/cve-update-nvd2-native.bb +7 -2
@@ -83,8 +83,13 @@ python do_fetch() { shutil.copy2(db_file, db_tmp_file) if update_db_file(db_tmp_file, d, database_time) == True: - # Update downloaded correctly, can swap files - shutil.move(db_tmp_file, db_file) + # Update downloaded correctly, we can swap files. To avoid potential + # NFS caching issues, ensure that the destination file has a new inode + # number. We do this in two steps as the downloads directory may be on + # a different filesystem to tmpdir we're working in. + new_file = "%s.new" % (db_file) + shutil.move(db_tmp_file, new_file) + os.rename(new_file, db_file) else: # Update failed, do not modify the database bb.warn("CVE database update failed")