cve-update-db-native: move download cleanup to a function
PATCH 1/2 cve-update-db-native: move download cleanup to a function
Move cleanup of the database after a failed download to a separate function. This will be useful when we have more actions to do in that situation.
--- meta/recipes-core/meta/cve-update-db-native.bb | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-)
meta/recipes-core/meta/cve-update-db-native.bb +10 -7
@@ -44,13 +44,7 @@ python do_fetch() { cve_socket_timeout = int(d.getVar("CVE_SOCKET_TIMEOUT")) - if os.path.exists("{0}-journal".format(db_file)): - # If a journal is present the last update might have been interrupted. In that case, - # just wipe any leftovers and force the DB to be recreated. - os.remove("{0}-journal".format(db_file)) - - if os.path.exists(db_file): - os.remove(db_file) + cleanup_db_download(db_file) # The NVD database changes once a day, so no need to update more frequently # Allow the user to force-update
@@ -134,6 +128,15 @@ do_fetch[lockfiles] += "${CVE_CHECK_DB_FILE_LOCK}" do_fetch[file-checksums] = "" do_fetch[vardeps] = "" +def cleanup_db_download(db_file): + if os.path.exists("{0}-journal".format(db_file)): + # If a journal is present the last update might have been interrupted. In that case, + # just wipe any leftovers and force the DB to be recreated. + os.remove("{0}-journal".format(db_file)) + + if os.path.exists(db_file): + os.remove(db_file) + def initialize_db(conn): with conn: c = conn.cursor()
Signature · 1 line
2.35.1
PATCH 2/2 cve-update-db-native: avoid partial updates
The database update has been done on the original file. In case of network connection issues, temporary outage of the NVD server or a similar situation, the function could exit with incomplete data in the database. This patch solves the issue by performing the update on a copy of the database. It replaces the main one only if the whole update was successful.
See https://bugzilla.yoctoproject.org/show_bug.cgi?id=14929
--- .../recipes-core/meta/cve-update-db-native.bb | 81 +++++++++++++------ 1 file changed, 56 insertions(+), 25 deletions(-)
meta/recipes-core/meta/cve-update-db-native.bb +56 -25
@@ -21,6 +21,8 @@ CVE_DB_UPDATE_INTERVAL ?= "86400" # Timeout for blocking socket operations, such as the connection attempt. CVE_SOCKET_TIMEOUT ?= "60" +CVE_DB_TEMP_FILE ?= "${CVE_CHECK_DB_DIR}/temp_nvdcve_1.1.db" + python () { if not bb.data.inherits_class("cve-check", d): raise bb.parse.SkipRecipe("Skip recipe when cve-check class is not loaded.")
@@ -32,19 +34,15 @@ python do_fetch() { """ import bb.utils import bb.progress - import sqlite3, urllib, urllib.parse, gzip - from datetime import date + import shutil bb.utils.export_proxies(d) - YEAR_START = 2002 - db_file = d.getVar("CVE_CHECK_DB_FILE") db_dir = os.path.dirname(db_file) + db_tmp_file = d.getVar("CVE_DB_TEMP_FILE") - cve_socket_timeout = int(d.getVar("CVE_SOCKET_TIMEOUT")) - - cleanup_db_download(db_file) + cleanup_db_download(db_file, db_tmp_file) # The NVD database changes once a day, so no need to update more frequently # Allow the user to force-update
@@ -62,9 +60,55 @@ python do_fetch() { pass bb.utils.mkdirhier(db_dir) + if os.path.exists(db_file): + shutil.copy2(db_file, db_tmp_file) + + if update_db_file(db_tmp_file, d) == True: + # Update downloaded correctly, can swap files + shutil.move(db_tmp_file, db_file) + else: + # Update failed, do not modify the database + bb.note("CVE database update failed") + os.remove(db_tmp_file) +} + +do_fetch[lockfiles] += "${CVE_CHECK_DB_FILE_LOCK}" +do_fetch[file-checksums] = "" +do_fetch[vardeps] = "" + +def cleanup_db_download(db_file, db_tmp_file): + """ + Cleanup the download space from possible failed downloads + """ + if os.path.exists("{0}-journal".format(db_file)): + # If a journal is present the last update might have been interrupted. In that case, + # just wipe any leftovers and force the DB to be recreated. + os.remove("{0}-journal".format(db_file)) + + if os.path.exists(db_file): + os.remove(db_file) + + if os.path.exists("{0}-journal".format(db_tmp_file)): + # If a journal is present the last update might have been interrupted. In that case, + # just wipe any leftovers and force the DB to be recreated. + os.remove("{0}-journal".format(db_tmp_file)) + + if os.path.exists(db_tmp_file): + os.remove(db_tmp_file) + +def update_db_file(db_tmp_file, d): + """ + Update the given database file + """ + import bb.utils, bb.progress + from datetime import date + import urllib, gzip, sqlite3 + + YEAR_START = 2002 + cve_socket_timeout = int(d.getVar("CVE_SOCKET_TIMEOUT")) # Connect to database - conn = sqlite3.connect(db_file) + conn = sqlite3.connect(db_tmp_file) initialize_db(conn) with bb.progress.ProgressHandler(d) as ph, open(os.path.join(d.getVar("TMPDIR"), 'cve_check'), 'a') as cve_f:
@@ -82,7 +126,7 @@ python do_fetch() { except urllib.error.URLError as e: cve_f.write('Warning: CVE db update error, Unable to fetch CVE data.\n\n') bb.warn("Failed to fetch CVE data (%s)" % e.reason) - return + return False if response: for l in response.read().decode("utf-8").splitlines():
@@ -92,7 +136,7 @@ python do_fetch() { break else: bb.warn("Cannot parse CVE metadata, update failed") - return + return False # Compare with current db last modified date cursor = conn.execute("select DATE from META where YEAR = ?", (year,))
@@ -113,7 +157,7 @@ python do_fetch() { except urllib.error.URLError as e: cve_f.write('Warning: CVE db update error, CVE data is outdated.\n\n') bb.warn("Cannot parse CVE data (%s), update failed" % e.reason) - return + return False else: bb.debug(2, "Already up to date (last modified %s)" % last_modified) # Update success, set the date to cve_check file.
@@ -122,20 +166,7 @@ python do_fetch() { conn.commit() conn.close() -} - -do_fetch[lockfiles] += "${CVE_CHECK_DB_FILE_LOCK}" -do_fetch[file-checksums] = "" -do_fetch[vardeps] = "" - -def cleanup_db_download(db_file): - if os.path.exists("{0}-journal".format(db_file)): - # If a journal is present the last update might have been interrupted. In that case, - # just wipe any leftovers and force the DB to be recreated. - os.remove("{0}-journal".format(db_file)) - - if os.path.exists(db_file): - os.remove(db_file) + return True def initialize_db(conn): with conn:
Signature · 1 line
2.35.1
Re: [OE-core] [PATCH 2/2] cve-update-db-native: avoid partial updates
Hi Marta,
Marta Rybczynska <[email protected]> escreveu no dia segunda, 2/01/2023 à(s) 07:03:
Quoted 87 lines
The database update has been done on the original file. In case of network connection issues, temporary outage of the NVD server or a similar situation, the function could exit with incomplete data in the database. This patch solves the issue by performing the update on a copy of the database. It replaces the main one only if the whole update was successful.
See https://bugzilla.yoctoproject.org/show_bug.cgi?id=14929
Reported-by Alberto Pianon <[email protected]>Signed-off-by Marta Rybczynska <[email protected]>--- .../recipes-core/meta/cve-update-db-native.bb | 81 +++++++++++++------ 1 file changed, 56 insertions(+), 25 deletions(-)meta/recipes-core/meta/cve-update-db-native.bb +2
-0@@ -21,6 +21,8 @@ CVE_DB_UPDATE_INTERVAL ?= "86400" # Timeout for blocking socket operations, such as the connection attempt. CVE_SOCKET_TIMEOUT ?= "60" +CVE_DB_TEMP_FILE ?= "${CVE_CHECK_DB_DIR}/temp_nvdcve_1.1.db" + python () { if not bb.data.inherits_class("cve-check", d): raise bb.parse.SkipRecipe("Skip recipe when cve-check class isnot loaded.")
meta/recipes-core/meta/cve-update-db-native.bb +3
-7@@ -32,19 +34,15 @@ python do_fetch() { """ import bb.utils import bb.progress - import sqlite3, urllib, urllib.parse, gzip - from datetime import date + import shutil bb.utils.export_proxies(d) - YEAR_START = 2002 - db_file = d.getVar("CVE_CHECK_DB_FILE") db_dir = os.path.dirname(db_file) + db_tmp_file = d.getVar("CVE_DB_TEMP_FILE") - cve_socket_timeout = int(d.getVar("CVE_SOCKET_TIMEOUT")) - - cleanup_db_download(db_file) + cleanup_db_download(db_file, db_tmp_file) # The NVD database changes once a day, so no need to update morefrequently
# Allow the user to force-updatemeta/recipes-core/meta/cve-update-db-native.bb +28
-0@@ -62,9 +60,55 @@ python do_fetch() { pass bb.utils.mkdirhier(db_dir) + if os.path.exists(db_file): + shutil.copy2(db_file, db_tmp_file) + + if update_db_file(db_tmp_file, d) == True: + # Update downloaded correctly, can swap files + shutil.move(db_tmp_file, db_file) + else: + # Update failed, do not modify the database + bb.note("CVE database update failed") + os.remove(db_tmp_file) +} + +do_fetch[lockfiles] += "${CVE_CHECK_DB_FILE_LOCK}" +do_fetch[file-checksums] = "" +do_fetch[vardeps] = "" + +def cleanup_db_download(db_file, db_tmp_file): + """ + Cleanup the download space from possible failed downloads + """ + if os.path.exists("{0}-journal".format(db_file)): + # If a journal is present the last update might have been interrupted. In that case, + # just wipe any leftovers and force the DB to be recreated. + os.remove("{0}-journal".format(db_file)) + + if os.path.exists(db_file): + os.remove(db_file) +
meta/recipes-core/meta/cve-update-db-native.bb python do_fetch() { · line 101
Quoted 10 lines
meta/recipes-core/meta/cve-update-db-native.bb +8
-0+ if os.path.exists("{0}-journal".format(db_tmp_file)): + # If a journal is present the last update might have been interrupted. In that case, + # just wipe any leftovers and force the DB to be recreated. + os.remove("{0}-journal".format(db_tmp_file)) + + if os.path.exists(db_tmp_file): + os.remove(db_tmp_file) +
It seems to me that this function is a duplication of the old version with an extra argument.
So I think that using the old function version and call it with the proper argument does the same:
cleanup_db_download(db_file)cleanup_db_download(db_tmp_file)
Jose
Quoted 88 lines
meta/recipes-core/meta/cve-update-db-native.bb +11
-1+def update_db_file(db_tmp_file, d): + """ + Update the given database file + """ + import bb.utils, bb.progress + from datetime import date + import urllib, gzip, sqlite3 + + YEAR_START = 2002 + cve_socket_timeout = int(d.getVar("CVE_SOCKET_TIMEOUT")) # Connect to database - conn = sqlite3.connect(db_file) + conn = sqlite3.connect(db_tmp_file)
initialize_db(conn)with
bb.progress.ProgressHandler(d)as ph,open(os.path.join(d.getVar("TMPDIR"), 'cve_check'), 'a')ascve_f:meta/recipes-core/meta/cve-update-db-native.bb +1
-1@@ -82,7 +126,7 @@ python do_fetch() { except urllib.error.URLError as e: cve_f.write('Warning: CVE db update error, Unable to fetch CVE data.\n\n') bb.warn("Failed to fetch CVE data (%s)" % e.reason) - return + return False if response:for l in response.read().decode("utf-8").splitlines():meta/recipes-core/meta/cve-update-db-native.bb +1
-1@@ -92,7 +136,7 @@ python do_fetch() { break else: bb.warn("Cannot parse CVE metadata, update failed") - return + return False # Compare with current db last modified date cursor = conn.execute("select DATE from META where YEAR = ?",(year,))
meta/recipes-core/meta/cve-update-db-native.bb +1
-1@@ -113,7 +157,7 @@ python do_fetch() { except urllib.error.URLError as e: cve_f.write('Warning: CVE db update error, CVE data is outdated.\n\n') bb.warn("Cannot parse CVE data (%s), update failed" % e.reason) - return + return False else:bb.debug(2, "Already up to date (last modified %s)" %
last_modified)
# Update success, set the date tocve_checkfile.meta/recipes-core/meta/cve-update-db-native.bb +1
-14@@ -122,20 +166,7 @@ python do_fetch() { conn.commit() conn.close() -} - -do_fetch[lockfiles] += "${CVE_CHECK_DB_FILE_LOCK}" -do_fetch[file-checksums] = "" -do_fetch[vardeps] = "" - -def cleanup_db_download(db_file): - if os.path.exists("{0}-journal".format(db_file)): - # If a journal is present the last update might have been interrupted. In that case, - # just wipe any leftovers and force the DB to be recreated. - os.remove("{0}-journal".format(db_file)) - - if os.path.exists(db_file): - os.remove(db_file) + return True def initialize_db(conn):with conn:
--
2.35.1-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group. View/Reply Online (#175314):
https://lists.openembedded.org/g/openembedded-core/message/175314
Mute This Topic: https://lists.openembedded.org/mt/96002809/5052612
Group Owner: [email protected]
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [ [email protected]]
-=-=-=-=-=-=-=-=-=-=-=-
Signature · 3 lines
Best regards, José Quaresma
Re: [OE-core] [PATCH 2/2] cve-update-db-native: avoid partial updates
Quoted 108 lines · Jose Quaresma
Hi Marta,
Marta Rybczynska <[email protected]> escreveu no dia segunda, 2/01/2023 à(s) 07:03:
Quoted 87 lines
The database update has been done on the original file. In case of network connection issues, temporary outage of the NVD server or a similar situation, the function could exit with incomplete data in the database. This patch solves the issue by performing the update on a copy of the database. It replaces the main one only if the whole update was successful.
See https://bugzilla.yoctoproject.org/show_bug.cgi?id=14929
Reported-by Alberto Pianon <[email protected]>Signed-off-by Marta Rybczynska <[email protected]>--- .../recipes-core/meta/cve-update-db-native.bb | 81 +++++++++++++------ 1 file changed, 56 insertions(+), 25 deletions(-)meta/recipes-core/meta/cve-update-db-native.bb +2
-0@@ -21,6 +21,8 @@ CVE_DB_UPDATE_INTERVAL ?= "86400" # Timeout for blocking socket operations, such as the connection attempt. CVE_SOCKET_TIMEOUT ?= "60" +CVE_DB_TEMP_FILE ?= "${CVE_CHECK_DB_DIR}/temp_nvdcve_1.1.db" + python () { if not bb.data.inherits_class("cve-check", d): raise bb.parse.SkipRecipe("Skip recipe when cve-check class isnot loaded.")
meta/recipes-core/meta/cve-update-db-native.bb +3
-7@@ -32,19 +34,15 @@ python do_fetch() { """ import bb.utils import bb.progress - import sqlite3, urllib, urllib.parse, gzip - from datetime import date + import shutil bb.utils.export_proxies(d) - YEAR_START = 2002 - db_file = d.getVar("CVE_CHECK_DB_FILE") db_dir = os.path.dirname(db_file) + db_tmp_file = d.getVar("CVE_DB_TEMP_FILE") - cve_socket_timeout = int(d.getVar("CVE_SOCKET_TIMEOUT")) - - cleanup_db_download(db_file) + cleanup_db_download(db_file, db_tmp_file) # The NVD database changes once a day, so no need to update morefrequently
# Allow the user to force-updatemeta/recipes-core/meta/cve-update-db-native.bb +28
-0@@ -62,9 +60,55 @@ python do_fetch() { pass bb.utils.mkdirhier(db_dir) + if os.path.exists(db_file): + shutil.copy2(db_file, db_tmp_file) + + if update_db_file(db_tmp_file, d) == True: + # Update downloaded correctly, can swap files + shutil.move(db_tmp_file, db_file) + else: + # Update failed, do not modify the database + bb.note("CVE database update failed") + os.remove(db_tmp_file) +} + +do_fetch[lockfiles] += "${CVE_CHECK_DB_FILE_LOCK}" +do_fetch[file-checksums] = "" +do_fetch[vardeps] = "" + +def cleanup_db_download(db_file, db_tmp_file): + """ + Cleanup the download space from possible failed downloads + """ + if os.path.exists("{0}-journal".format(db_file)): + # If a journal is present the last update might have been interrupted. In that case, + # just wipe any leftovers and force the DB to be recreated. + os.remove("{0}-journal".format(db_file)) + + if os.path.exists(db_file): + os.remove(db_file) +meta/recipes-core/meta/cve-update-db-native.bb python do_fetch() { · line 101
Quoted 10 lines
meta/recipes-core/meta/cve-update-db-native.bb +8
-0+ if os.path.exists("{0}-journal".format(db_tmp_file)): + # If a journal is present the last update might have been interrupted. In that case, + # just wipe any leftovers and force the DB to be recreated. + os.remove("{0}-journal".format(db_tmp_file)) + + if os.path.exists(db_tmp_file): + os.remove(db_tmp_file) +It seems to me that this function is a duplication of the old version with an extra argument.
So I think that using the old function version and call it with the proper argument does the same:
Quoted 4 lines
cleanup_db_download(db_file)cleanup_db_download(db_tmp_file)
Hi Jose,
Thanks for looking into that. The function is not a total duplicate: the difference is that
the it always removes the db_tmp_file, not only if the journal file exists (Python code
formatting!).
I was hesitating on this part a bit, because with the old path could be taken only in some
specific situations: at the code update and if you share the DL_DIR and some of the builds
use the old code, some the new version. I think we should keep both for now for safety.
Kind regards,
Marta
Re: [OE-core] [PATCH 2/2] cve-update-db-native: avoid partial updates
Marta Rybczynska <[email protected]> escreveu no dia segunda, 2/01/2023 à(s) 16:38:
Quoted 118 lines
Quoted 109 lines · Jose Quaresma
Hi Marta,
Marta Rybczynska <[email protected]> escreveu no dia segunda, 2/01/2023 à(s) 07:03:
Quoted 88 lines
The database update has been done on the original file. In case of network connection issues, temporary outage of the NVD server or a similar situation, the function could exit with incomplete data in the database. This patch solves the issue by performing the update on a copy of the database. It replaces the main one only if the whole update was successful.
See https://bugzilla.yoctoproject.org/show_bug.cgi?id=14929
Reported-by Alberto Pianon <[email protected]>Signed-off-by Marta Rybczynska <[email protected]>--- .../recipes-core/meta/cve-update-db-native.bb | 81 +++++++++++++------ 1 file changed, 56 insertions(+), 25 deletions(-)meta/recipes-core/meta/cve-update-db-native.bb +0
-0@@ -21,6 +21,8 @@ CVE_DB_UPDATE_INTERVAL ?= "86400" # Timeout for blocking socket operations, such as the connectionattempt.
CVE_SOCKET_TIMEOUT?= "60"meta/recipes-core/meta/cve-update-db-native.bb +2
-0+CVE_DB_TEMP_FILE ?= "${CVE_CHECK_DB_DIR}/temp_nvdcve_1.1.db" + python () { if not bb.data.inherits_class("cve-check", d): raise bb.parse.SkipRecipe("Skip recipe when cve-check class isnot loaded.")
meta/recipes-core/meta/cve-update-db-native.bb +3
-7@@ -32,19 +34,15 @@ python do_fetch() { """ import bb.utils import bb.progress - import sqlite3, urllib, urllib.parse, gzip - from datetime import date + import shutil bb.utils.export_proxies(d) - YEAR_START = 2002 - db_file = d.getVar("CVE_CHECK_DB_FILE") db_dir = os.path.dirname(db_file) + db_tmp_file = d.getVar("CVE_DB_TEMP_FILE") - cve_socket_timeout = int(d.getVar("CVE_SOCKET_TIMEOUT")) - - cleanup_db_download(db_file) + cleanup_db_download(db_file, db_tmp_file) # The NVD database changes once a day, so no need to update morefrequently
# Allow the user to force-updatemeta/recipes-core/meta/cve-update-db-native.bb +28
-0@@ -62,9 +60,55 @@ python do_fetch() { pass bb.utils.mkdirhier(db_dir) + if os.path.exists(db_file): + shutil.copy2(db_file, db_tmp_file) + + if update_db_file(db_tmp_file, d) == True: + # Update downloaded correctly, can swap files + shutil.move(db_tmp_file, db_file) + else: + # Update failed, do not modify the database + bb.note("CVE database update failed") + os.remove(db_tmp_file) +} + +do_fetch[lockfiles] += "${CVE_CHECK_DB_FILE_LOCK}" +do_fetch[file-checksums] = "" +do_fetch[vardeps] = "" + +def cleanup_db_download(db_file, db_tmp_file): + """ + Cleanup the download space from possible failed downloads + """ + if os.path.exists("{0}-journal".format(db_file)): + # If a journal is present the last update might have been interrupted. In that case, + # just wipe any leftovers and force the DB to be recreated. + os.remove("{0}-journal".format(db_file)) + + if os.path.exists(db_file): + os.remove(db_file) +meta/recipes-core/meta/cve-update-db-native.bb python do_fetch() { · line 101
Quoted 10 lines
meta/recipes-core/meta/cve-update-db-native.bb +8
-0+ if os.path.exists("{0}-journal".format(db_tmp_file)): + # If a journal is present the last update might have been interrupted. In that case, + # just wipe any leftovers and force the DB to be recreated. + os.remove("{0}-journal".format(db_tmp_file)) + + if os.path.exists(db_tmp_file): + os.remove(db_tmp_file) +It seems to me that this function is a duplication of the old version with an extra argument.
So I think that using the old function version and call it with the proper argument does the same:Quoted 4 lines
cleanup_db_download(db_file)cleanup_db_download(db_tmp_file)
Quoted 7 lines
Hi Jose,
Thanks for looking into that. The function is not a total duplicate: the difference is that
the it always removes thedb_tmp_file, not only if the journal file exists (Python code
formatting!).
Don't see on the first time that the db_tmp_file is always removed, I need new glasses :)
Quoted 8 lines
I was hesitating on this part a bit, because with the old path could be taken only in some
specific situations: at the code update and if you share theDL_DIRand some of the builds
use the old code, some the new version. I think we should keep both for now for safety.
makes sense, thanks for your explanation.
Jose
Quoted 4 lines
Kind regards,
Marta
Signature · 3 lines
Best regards, José Quaresma
Re: [OE-core] [PATCH 2/2] cve-update-db-native: avoid partial updates
Quoted 144 lines · Jose Quaresma
Marta Rybczynska <[email protected]> escreveu no dia segunda, 2/01/2023 à(s) 16:38:
Quoted 118 lines
Quoted 109 lines · Jose Quaresma
Hi Marta,
Marta Rybczynska <[email protected]> escreveu no dia segunda, 2/01/2023 à(s) 07:03:
Quoted 88 lines
The database update has been done on the original file. In case of network connection issues, temporary outage of the NVD server or a similar situation, the function could exit with incomplete data in the database. This patch solves the issue by performing the update on a copy of the database. It replaces the main one only if the whole update was successful.
See https://bugzilla.yoctoproject.org/show_bug.cgi?id=14929
Reported-by Alberto Pianon <[email protected]>Signed-off-by Marta Rybczynska <[email protected]>--- .../recipes-core/meta/cve-update-db-native.bb | 81 +++++++++++++------ 1 file changed, 56 insertions(+), 25 deletions(-)meta/recipes-core/meta/cve-update-db-native.bb +0
-0@@ -21,6 +21,8 @@ CVE_DB_UPDATE_INTERVAL ?= "86400" # Timeout for blocking socket operations, such as the connectionattempt.
CVE_SOCKET_TIMEOUT?= "60"meta/recipes-core/meta/cve-update-db-native.bb +2
-0+CVE_DB_TEMP_FILE ?= "${CVE_CHECK_DB_DIR}/temp_nvdcve_1.1.db" + python () { if not bb.data.inherits_class("cve-check", d): raise bb.parse.SkipRecipe("Skip recipe when cve-check class isnot loaded.")
meta/recipes-core/meta/cve-update-db-native.bb +3
-7@@ -32,19 +34,15 @@ python do_fetch() { """ import bb.utils import bb.progress - import sqlite3, urllib, urllib.parse, gzip - from datetime import date + import shutil bb.utils.export_proxies(d) - YEAR_START = 2002 - db_file = d.getVar("CVE_CHECK_DB_FILE") db_dir = os.path.dirname(db_file) + db_tmp_file = d.getVar("CVE_DB_TEMP_FILE") - cve_socket_timeout = int(d.getVar("CVE_SOCKET_TIMEOUT")) - - cleanup_db_download(db_file) + cleanup_db_download(db_file, db_tmp_file) # The NVD database changes once a day, so no need to update morefrequently
# Allow the user to force-updatemeta/recipes-core/meta/cve-update-db-native.bb +28
-0@@ -62,9 +60,55 @@ python do_fetch() { pass bb.utils.mkdirhier(db_dir) + if os.path.exists(db_file): + shutil.copy2(db_file, db_tmp_file) + + if update_db_file(db_tmp_file, d) == True: + # Update downloaded correctly, can swap files + shutil.move(db_tmp_file, db_file) + else: + # Update failed, do not modify the database + bb.note("CVE database update failed") + os.remove(db_tmp_file) +} + +do_fetch[lockfiles] += "${CVE_CHECK_DB_FILE_LOCK}" +do_fetch[file-checksums] = "" +do_fetch[vardeps] = "" + +def cleanup_db_download(db_file, db_tmp_file): + """ + Cleanup the download space from possible failed downloads + """ + if os.path.exists("{0}-journal".format(db_file)): + # If a journal is present the last update might have been interrupted. In that case, + # just wipe any leftovers and force the DB to be recreated. + os.remove("{0}-journal".format(db_file)) + + if os.path.exists(db_file): + os.remove(db_file) +meta/recipes-core/meta/cve-update-db-native.bb python do_fetch() { · line 101
Quoted 10 lines
meta/recipes-core/meta/cve-update-db-native.bb +8
-0+ if os.path.exists("{0}-journal".format(db_tmp_file)): + # If a journal is present the last update might have been interrupted. In that case, + # just wipe any leftovers and force the DB to be recreated. + os.remove("{0}-journal".format(db_tmp_file)) + + if os.path.exists(db_tmp_file): + os.remove(db_tmp_file) +It seems to me that this function is a duplication of the old version with an extra argument.
So I think that using the old function version and call it with the proper argument does the same:Quoted 4 lines
cleanup_db_download(db_file)cleanup_db_download(db_tmp_file)Quoted 7 lines
Hi Jose,
Thanks for looking into that. The function is not a total duplicate: the difference is that
the it always removes thedb_tmp_file, not only if the journal file exists (Python code
formatting!).Don't see on the first time that the
db_tmp_fileis always removed, I need new glasses :)Quoted 8 lines
I was hesitating on this part a bit, because with the old path could be taken only in some
specific situations: at the code update and if you share theDL_DIRand some of the builds
use the old code, some the new version. I think we should keep both for now for safety.
Quoted 3 lines
makes sense, thanks for your explanation.
Well... I'll submit a v2 with comments to make sure nobody tries to optimize that piece :)
Regards,
Marta
A message that is not in the archive (sent off-list or not yet received): [email protected]
Re: [OE-core] [PATCH 2/2] cve-update-db-native: avoid partial updates
Quoted 24 lines
Quoted 24 lines
Quoted 10 lines
Hi Jose,
Thanks for looking into that. The function is not a total duplicate: the difference is that
the it always removes thedb_tmp_file, not only if the journal file exists (Python code
formatting!).Don't see on the first time that the
db_tmp_fileis always removed, I need new glasses :)Quoted 8 lines
I was hesitating on this part a bit, because with the old path could be taken only in some
specific situations: at the code update and if you share theDL_DIRand some of the builds
use the old code, some the new version. I think we should keep both for now for safety.
Quoted 6 lines
Quoted 3 lines
makes sense, thanks for your explanation.
Well... I'll submit a v2 with comments to make sure nobody tries to optimize that piece :)
And here it is (with a minor title change):
https://lists.openembedded.org/g/openembedded-core/message/175355
Regards,
Marta
Re: [OE-core] [PATCH 2/2] cve-update-db-native: avoid partial updates
Marta Rybczynska <[email protected]> escreveu no dia quarta, 4/01/2023 à(s) 15:28:
Quoted 26 lines
Quoted 26 lines
Quoted 26 lines
Quoted 9 lines
Hi Jose,
Thanks for looking into that. The function is not a total duplicate:
the difference is that
the it always removes thedb_tmp_file, not only if the journal file exists (Python code
formatting!).Don't see on the first time that the
db_tmp_fileis always removed, I need new glasses :)Quoted 8 lines
I was hesitating on this part a bit, because with the old path could be taken only in some
specific situations: at the code update and if you share theDL_DIRand some of the builds
use the old code, some the new version. I think we should keep both for now for safety.makes sense, thanks for your explanation.
Quoted 7 lines
Quoted 3 lines
Well... I'll submit a v2 with comments to make sure nobody tries to optimize that piece :)
And here it is (with a minor title change):
https://lists.openembedded.org/g/openembedded-core/message/175355
Thanks,
Jose
Quoted 4 lines
Regards,
Marta
Signature · 3 lines
Best regards, José Quaresma
This patch is replaced by
https://lists.openembedded.org/g/openembedded-core/message/175355 (with a minor patch title change)
Quoted 52 lines · Marta Rybczynska
Move cleanup of the database after a failed download to a separate function. This will be useful when we have more actions to do in that situation.
Signed-off-by Marta Rybczynska <[email protected]>--- meta/recipes-core/meta/cve-update-db-native.bb | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-)meta/recipes-core/meta/cve-update-db-native.bb +1
-7@@ -44,13 +44,7 @@ python do_fetch() { cve_socket_timeout = int(d.getVar("CVE_SOCKET_TIMEOUT")) - if os.path.exists("{0}-journal".format(db_file)): - # If a journal is present the last update might have been interrupted. In that case, - # just wipe any leftovers and force the DB to be recreated. - os.remove("{0}-journal".format(db_file)) - - if os.path.exists(db_file): - os.remove(db_file) + cleanup_db_download(db_file) # The NVD database changes once a day, so no need to update morefrequently
# Allow the user to force-updatemeta/recipes-core/meta/cve-update-db-native.bb +9
-0@@ -134,6 +128,15 @@ do_fetch[lockfiles] += "${CVE_CHECK_DB_FILE_LOCK}" do_fetch[file-checksums] = "" do_fetch[vardeps] = "" +def cleanup_db_download(db_file): + if os.path.exists("{0}-journal".format(db_file)): + # If a journal is present the last update might have been interrupted. In that case, + # just wipe any leftovers and force the DB to be recreated. + os.remove("{0}-journal".format(db_file)) + + if os.path.exists(db_file): + os.remove(db_file) + def initialize_db(conn): with conn:c =
conn.cursor()
--
2.35.1