Skip to contentSkip to conversation

cve-update-db-native: move download cleanup to a function

PATCH In review Marta Rybczynska 2 participants 9 messages Jan 2, 2023 – Jan 4, 2023 1 message not in the archive

lore
Share & tools
lore
#0 lore raw

PATCH 1/2 cve-update-db-native: move download cleanup to a function

Move cleanup of the database after a failed download to a separate function. This will be useful when we have more actions to do in that situation.

---
 meta/recipes-core/meta/cve-update-db-native.bb | 17 ++++++++++-------
 1 file changed, 10 insertions(+), 7 deletions(-)
meta/recipes-core/meta/cve-update-db-native.bb +10 -7
index 9b9dbbd75f..642fda5395 100644
@@ -44,13 +44,7 @@ python do_fetch() {
 
     cve_socket_timeout = int(d.getVar("CVE_SOCKET_TIMEOUT"))
 
-    if os.path.exists("{0}-journal".format(db_file)):
-        # If a journal is present the last update might have been interrupted. In that case,
-        # just wipe any leftovers and force the DB to be recreated.
-        os.remove("{0}-journal".format(db_file))
-
-        if os.path.exists(db_file):
-            os.remove(db_file)
+    cleanup_db_download(db_file)
 
     # The NVD database changes once a day, so no need to update more frequently
     # Allow the user to force-update
@@ -134,6 +128,15 @@ do_fetch[lockfiles] += "${CVE_CHECK_DB_FILE_LOCK}"
 do_fetch[file-checksums] = ""
 do_fetch[vardeps] = ""
 
+def cleanup_db_download(db_file):
+    if os.path.exists("{0}-journal".format(db_file)):
+        # If a journal is present the last update might have been interrupted. In that case,
+        # just wipe any leftovers and force the DB to be recreated.
+        os.remove("{0}-journal".format(db_file))
+
+        if os.path.exists(db_file):
+            os.remove(db_file)
+
 def initialize_db(conn):
     with conn:
         c = conn.cursor()
Signature · 1 line
2.35.1
#1 lore raw

PATCH 2/2 cve-update-db-native: avoid partial updates

The database update has been done on the original file. In case of network connection issues, temporary outage of the NVD server or a similar situation, the function could exit with incomplete data in the database. This patch solves the issue by performing the update on a copy of the database. It replaces the main one only if the whole update was successful.

See https://bugzilla.yoctoproject.org/show_bug.cgi?id=14929

---
 .../recipes-core/meta/cve-update-db-native.bb | 81 +++++++++++++------
 1 file changed, 56 insertions(+), 25 deletions(-)
meta/recipes-core/meta/cve-update-db-native.bb +56 -25
index 642fda5395..89804b9e5c 100644
@@ -21,6 +21,8 @@ CVE_DB_UPDATE_INTERVAL ?= "86400"
 # Timeout for blocking socket operations, such as the connection attempt.
 CVE_SOCKET_TIMEOUT ?= "60"
 
+CVE_DB_TEMP_FILE ?= "${CVE_CHECK_DB_DIR}/temp_nvdcve_1.1.db"
+
 python () {
     if not bb.data.inherits_class("cve-check", d):
         raise bb.parse.SkipRecipe("Skip recipe when cve-check class is not loaded.")
@@ -32,19 +34,15 @@ python do_fetch() {
     """
     import bb.utils
     import bb.progress
-    import sqlite3, urllib, urllib.parse, gzip
-    from datetime import date
+    import shutil
 
     bb.utils.export_proxies(d)
 
-    YEAR_START = 2002
-
     db_file = d.getVar("CVE_CHECK_DB_FILE")
     db_dir = os.path.dirname(db_file)
+    db_tmp_file = d.getVar("CVE_DB_TEMP_FILE")
 
-    cve_socket_timeout = int(d.getVar("CVE_SOCKET_TIMEOUT"))
-
-    cleanup_db_download(db_file)
+    cleanup_db_download(db_file, db_tmp_file)
 
     # The NVD database changes once a day, so no need to update more frequently
     # Allow the user to force-update
@@ -62,9 +60,55 @@ python do_fetch() {
         pass
 
     bb.utils.mkdirhier(db_dir)
+    if os.path.exists(db_file):
+        shutil.copy2(db_file, db_tmp_file)
+
+    if update_db_file(db_tmp_file, d) == True:
+        # Update downloaded correctly, can swap files
+        shutil.move(db_tmp_file, db_file)
+    else:
+        # Update failed, do not modify the database
+        bb.note("CVE database update failed")
+        os.remove(db_tmp_file)
+}
+
+do_fetch[lockfiles] += "${CVE_CHECK_DB_FILE_LOCK}"
+do_fetch[file-checksums] = ""
+do_fetch[vardeps] = ""
+
+def cleanup_db_download(db_file, db_tmp_file):
+    """
+    Cleanup the download space from possible failed downloads
+    """
+    if os.path.exists("{0}-journal".format(db_file)):
+        # If a journal is present the last update might have been interrupted. In that case,
+        # just wipe any leftovers and force the DB to be recreated.
+        os.remove("{0}-journal".format(db_file))
+
+        if os.path.exists(db_file):
+            os.remove(db_file)
+
+    if os.path.exists("{0}-journal".format(db_tmp_file)):
+        # If a journal is present the last update might have been interrupted. In that case,
+        # just wipe any leftovers and force the DB to be recreated.
+        os.remove("{0}-journal".format(db_tmp_file))
+
+    if os.path.exists(db_tmp_file):
+        os.remove(db_tmp_file)
+
+def update_db_file(db_tmp_file, d):
+    """
+    Update the given database file
+    """
+    import bb.utils, bb.progress
+    from datetime import date
+    import urllib, gzip, sqlite3
+
+    YEAR_START = 2002
+    cve_socket_timeout = int(d.getVar("CVE_SOCKET_TIMEOUT"))
 
     # Connect to database
-    conn = sqlite3.connect(db_file)
+    conn = sqlite3.connect(db_tmp_file)
     initialize_db(conn)
 
     with bb.progress.ProgressHandler(d) as ph, open(os.path.join(d.getVar("TMPDIR"), 'cve_check'), 'a') as cve_f:
@@ -82,7 +126,7 @@ python do_fetch() {
             except urllib.error.URLError as e:
                 cve_f.write('Warning: CVE db update error, Unable to fetch CVE data.\n\n')
                 bb.warn("Failed to fetch CVE data (%s)" % e.reason)
-                return
+                return False
 
             if response:
                 for l in response.read().decode("utf-8").splitlines():
@@ -92,7 +136,7 @@ python do_fetch() {
                         break
                 else:
                     bb.warn("Cannot parse CVE metadata, update failed")
-                    return
+                    return False
 
             # Compare with current db last modified date
             cursor = conn.execute("select DATE from META where YEAR = ?", (year,))
@@ -113,7 +157,7 @@ python do_fetch() {
                 except urllib.error.URLError as e:
                     cve_f.write('Warning: CVE db update error, CVE data is outdated.\n\n')
                     bb.warn("Cannot parse CVE data (%s), update failed" % e.reason)
-                    return
+                    return False
             else:
                 bb.debug(2, "Already up to date (last modified %s)" % last_modified)
             # Update success, set the date to cve_check file.
@@ -122,20 +166,7 @@ python do_fetch() {
 
         conn.commit()
         conn.close()
-}
-
-do_fetch[lockfiles] += "${CVE_CHECK_DB_FILE_LOCK}"
-do_fetch[file-checksums] = ""
-do_fetch[vardeps] = ""
-
-def cleanup_db_download(db_file):
-    if os.path.exists("{0}-journal".format(db_file)):
-        # If a journal is present the last update might have been interrupted. In that case,
-        # just wipe any leftovers and force the DB to be recreated.
-        os.remove("{0}-journal".format(db_file))
-
-        if os.path.exists(db_file):
-            os.remove(db_file)
+        return True
 
 def initialize_db(conn):
     with conn:
Signature · 1 line
2.35.1
#2 lore raw

Re: [OE-core] [PATCH 2/2] cve-update-db-native: avoid partial updates

Hi Marta,

Marta Rybczynska <[email protected]> escreveu no dia segunda, 2/01/2023 à(s) 07:03:

Quoted 87 lines

The database update has been done on the original file. In case of network connection issues, temporary outage of the NVD server or a similar situation, the function could exit with incomplete data in the database. This patch solves the issue by performing the update on a copy of the database. It replaces the main one only if the whole update was successful.

See https://bugzilla.yoctoproject.org/show_bug.cgi?id=14929

---
 .../recipes-core/meta/cve-update-db-native.bb | 81 +++++++++++++------
 1 file changed, 56 insertions(+), 25 deletions(-)
meta/recipes-core/meta/cve-update-db-native.bb +2 -0
index 642fda5395..89804b9e5c 100644
@@ -21,6 +21,8 @@ CVE_DB_UPDATE_INTERVAL ?= "86400"
 # Timeout for blocking socket operations, such as the connection attempt.
 CVE_SOCKET_TIMEOUT ?= "60"
 
+CVE_DB_TEMP_FILE ?= "${CVE_CHECK_DB_DIR}/temp_nvdcve_1.1.db"
+
 python () {
     if not bb.data.inherits_class("cve-check", d):
         raise bb.parse.SkipRecipe("Skip recipe when cve-check class is

not loaded.")

meta/recipes-core/meta/cve-update-db-native.bb +3 -7
@@ -32,19 +34,15 @@ python do_fetch() {
     """
     import bb.utils
     import bb.progress
-    import sqlite3, urllib, urllib.parse, gzip
-    from datetime import date
+    import shutil
 
     bb.utils.export_proxies(d)
 
-    YEAR_START = 2002
-
     db_file = d.getVar("CVE_CHECK_DB_FILE")
     db_dir = os.path.dirname(db_file)
+    db_tmp_file = d.getVar("CVE_DB_TEMP_FILE")
 
-    cve_socket_timeout = int(d.getVar("CVE_SOCKET_TIMEOUT"))
-
-    cleanup_db_download(db_file)
+    cleanup_db_download(db_file, db_tmp_file)
 
     # The NVD database changes once a day, so no need to update more

frequently
# Allow the user to force-update

meta/recipes-core/meta/cve-update-db-native.bb +28 -0
@@ -62,9 +60,55 @@ python do_fetch() {
         pass
 
     bb.utils.mkdirhier(db_dir)
+    if os.path.exists(db_file):
+        shutil.copy2(db_file, db_tmp_file)
+
+    if update_db_file(db_tmp_file, d) == True:
+        # Update downloaded correctly, can swap files
+        shutil.move(db_tmp_file, db_file)
+    else:
+        # Update failed, do not modify the database
+        bb.note("CVE database update failed")
+        os.remove(db_tmp_file)
+}
+
+do_fetch[lockfiles] += "${CVE_CHECK_DB_FILE_LOCK}"
+do_fetch[file-checksums] = ""
+do_fetch[vardeps] = ""
+
+def cleanup_db_download(db_file, db_tmp_file):
+    """
+    Cleanup the download space from possible failed downloads
+    """
+    if os.path.exists("{0}-journal".format(db_file)):
+        # If a journal is present the last update might have been
 interrupted. In that case,
+        # just wipe any leftovers and force the DB to be recreated.
+        os.remove("{0}-journal".format(db_file))
+
+        if os.path.exists(db_file):
+            os.remove(db_file)
+

meta/recipes-core/meta/cve-update-db-native.bb python do_fetch() { · line 101

Quoted 10 lines
meta/recipes-core/meta/cve-update-db-native.bb +8 -0
+    if os.path.exists("{0}-journal".format(db_tmp_file)):
+        # If a journal is present the last update might have been
 interrupted. In that case,
+        # just wipe any leftovers and force the DB to be recreated.
+        os.remove("{0}-journal".format(db_tmp_file))
+
+    if os.path.exists(db_tmp_file):
+        os.remove(db_tmp_file)
+
 

It seems to me that this function is a duplication of the old version with an extra argument.
So I think that using the old function version and call it with the proper argument does the same:

cleanup_db_download(db_file)
cleanup_db_download(db_tmp_file)

Jose

Quoted 88 lines
meta/recipes-core/meta/cve-update-db-native.bb +11 -1
+def update_db_file(db_tmp_file, d):
+    """
+    Update the given database file
+    """
+    import bb.utils, bb.progress
+    from datetime import date
+    import urllib, gzip, sqlite3
+
+    YEAR_START = 2002
+    cve_socket_timeout = int(d.getVar("CVE_SOCKET_TIMEOUT"))
 
     # Connect to database
-    conn = sqlite3.connect(db_file)
+    conn = sqlite3.connect(db_tmp_file)

initialize_db(conn)

with bb.progress.ProgressHandler(d) as ph,
open(os.path.join(d.getVar("TMPDIR"), 'cve_check'), 'a') as cve_f:

meta/recipes-core/meta/cve-update-db-native.bb +1 -1
@@ -82,7 +126,7 @@ python do_fetch() {
             except urllib.error.URLError as e:
                 cve_f.write('Warning: CVE db update error, Unable to
 fetch CVE data.\n\n')
                 bb.warn("Failed to fetch CVE data (%s)" % e.reason)
-                return
+                return False
 
             if response:
                 for l in response.read().decode("utf-8").splitlines():
meta/recipes-core/meta/cve-update-db-native.bb +1 -1
@@ -92,7 +136,7 @@ python do_fetch() {
                         break
                 else:
                     bb.warn("Cannot parse CVE metadata, update failed")
-                    return
+                    return False
 
             # Compare with current db last modified date
             cursor = conn.execute("select DATE from META where YEAR = ?",

(year,))

meta/recipes-core/meta/cve-update-db-native.bb +1 -1
@@ -113,7 +157,7 @@ python do_fetch() {
                 except urllib.error.URLError as e:
                     cve_f.write('Warning: CVE db update error, CVE data
 is outdated.\n\n')
                     bb.warn("Cannot parse CVE data (%s), update failed" %
 e.reason)
-                    return
+                    return False
             else:

bb.debug(2, "Already up to date (last modified %s)" %
last_modified)
# Update success, set the date to cve_check file.

meta/recipes-core/meta/cve-update-db-native.bb +1 -14
@@ -122,20 +166,7 @@ python do_fetch() {
 
         conn.commit()
         conn.close()
-}
-
-do_fetch[lockfiles] += "${CVE_CHECK_DB_FILE_LOCK}"
-do_fetch[file-checksums] = ""
-do_fetch[vardeps] = ""
-
-def cleanup_db_download(db_file):
-    if os.path.exists("{0}-journal".format(db_file)):
-        # If a journal is present the last update might have been
 interrupted. In that case,
-        # just wipe any leftovers and force the DB to be recreated.
-        os.remove("{0}-journal".format(db_file))
-
-        if os.path.exists(db_file):
-            os.remove(db_file)
+        return True
 
 def initialize_db(conn):

with conn:
--
2.35.1

-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group. View/Reply Online (#175314):
https://lists.openembedded.org/g/openembedded-core/message/175314
Mute This Topic: https://lists.openembedded.org/mt/96002809/5052612
Group Owner: [email protected]
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [ [email protected]]
-=-=-=-=-=-=-=-=-=-=-=-

Signature · 3 lines
Best regards,

José Quaresma
#3 lore raw

Re: [OE-core] [PATCH 2/2] cve-update-db-native: avoid partial updates

Quoted 108 lines · Jose Quaresma

Hi Marta,

Marta Rybczynska <[email protected]> escreveu no dia segunda, 2/01/2023 à(s) 07:03:

Quoted 87 lines

The database update has been done on the original file. In case of network connection issues, temporary outage of the NVD server or a similar situation, the function could exit with incomplete data in the database. This patch solves the issue by performing the update on a copy of the database. It replaces the main one only if the whole update was successful.

See https://bugzilla.yoctoproject.org/show_bug.cgi?id=14929

---
 .../recipes-core/meta/cve-update-db-native.bb | 81 +++++++++++++------
 1 file changed, 56 insertions(+), 25 deletions(-)
meta/recipes-core/meta/cve-update-db-native.bb +2 -0
index 642fda5395..89804b9e5c 100644
@@ -21,6 +21,8 @@ CVE_DB_UPDATE_INTERVAL ?= "86400"
 # Timeout for blocking socket operations, such as the connection attempt.
 CVE_SOCKET_TIMEOUT ?= "60"
 
+CVE_DB_TEMP_FILE ?= "${CVE_CHECK_DB_DIR}/temp_nvdcve_1.1.db"
+
 python () {
     if not bb.data.inherits_class("cve-check", d):
         raise bb.parse.SkipRecipe("Skip recipe when cve-check class is

not loaded.")

meta/recipes-core/meta/cve-update-db-native.bb +3 -7
@@ -32,19 +34,15 @@ python do_fetch() {
     """
     import bb.utils
     import bb.progress
-    import sqlite3, urllib, urllib.parse, gzip
-    from datetime import date
+    import shutil
 
     bb.utils.export_proxies(d)
 
-    YEAR_START = 2002
-
     db_file = d.getVar("CVE_CHECK_DB_FILE")
     db_dir = os.path.dirname(db_file)
+    db_tmp_file = d.getVar("CVE_DB_TEMP_FILE")
 
-    cve_socket_timeout = int(d.getVar("CVE_SOCKET_TIMEOUT"))
-
-    cleanup_db_download(db_file)
+    cleanup_db_download(db_file, db_tmp_file)
 
     # The NVD database changes once a day, so no need to update more

frequently
# Allow the user to force-update

meta/recipes-core/meta/cve-update-db-native.bb +28 -0
@@ -62,9 +60,55 @@ python do_fetch() {
         pass
 
     bb.utils.mkdirhier(db_dir)
+    if os.path.exists(db_file):
+        shutil.copy2(db_file, db_tmp_file)
+
+    if update_db_file(db_tmp_file, d) == True:
+        # Update downloaded correctly, can swap files
+        shutil.move(db_tmp_file, db_file)
+    else:
+        # Update failed, do not modify the database
+        bb.note("CVE database update failed")
+        os.remove(db_tmp_file)
+}
+
+do_fetch[lockfiles] += "${CVE_CHECK_DB_FILE_LOCK}"
+do_fetch[file-checksums] = ""
+do_fetch[vardeps] = ""
+
+def cleanup_db_download(db_file, db_tmp_file):
+    """
+    Cleanup the download space from possible failed downloads
+    """
+    if os.path.exists("{0}-journal".format(db_file)):
+        # If a journal is present the last update might have been
 interrupted. In that case,
+        # just wipe any leftovers and force the DB to be recreated.
+        os.remove("{0}-journal".format(db_file))
+
+        if os.path.exists(db_file):
+            os.remove(db_file)
+

meta/recipes-core/meta/cve-update-db-native.bb python do_fetch() { · line 101

Quoted 10 lines
meta/recipes-core/meta/cve-update-db-native.bb +8 -0
+    if os.path.exists("{0}-journal".format(db_tmp_file)):
+        # If a journal is present the last update might have been
 interrupted. In that case,
+        # just wipe any leftovers and force the DB to be recreated.
+        os.remove("{0}-journal".format(db_tmp_file))
+
+    if os.path.exists(db_tmp_file):
+        os.remove(db_tmp_file)
+
 

It seems to me that this function is a duplication of the old version with an extra argument.
So I think that using the old function version and call it with the proper argument does the same:

Quoted 4 lines

cleanup_db_download(db_file)
cleanup_db_download(db_tmp_file)

Hi Jose,
Thanks for looking into that. The function is not a total duplicate: the difference is that
the it always removes the db_tmp_file, not only if the journal file exists (Python code
formatting!).

I was hesitating on this part a bit, because with the old path could be taken only in some
specific situations: at the code update and if you share the DL_DIR and some of the builds
use the old code, some the new version. I think we should keep both for now for safety.

Kind regards,
Marta

#4 lore raw

Re: [OE-core] [PATCH 2/2] cve-update-db-native: avoid partial updates

Marta Rybczynska <[email protected]> escreveu no dia segunda, 2/01/2023 à(s) 16:38:

Quoted 118 lines
Quoted 109 lines · Jose Quaresma

Hi Marta,

Marta Rybczynska <[email protected]> escreveu no dia segunda, 2/01/2023 à(s) 07:03:

Quoted 88 lines

The database update has been done on the original file. In case of network connection issues, temporary outage of the NVD server or a similar situation, the function could exit with incomplete data in the database. This patch solves the issue by performing the update on a copy of the database. It replaces the main one only if the whole update was successful.

See https://bugzilla.yoctoproject.org/show_bug.cgi?id=14929

---
 .../recipes-core/meta/cve-update-db-native.bb | 81 +++++++++++++------
 1 file changed, 56 insertions(+), 25 deletions(-)
meta/recipes-core/meta/cve-update-db-native.bb +0 -0
index 642fda5395..89804b9e5c 100644
@@ -21,6 +21,8 @@ CVE_DB_UPDATE_INTERVAL ?= "86400"
 # Timeout for blocking socket operations, such as the connection

attempt.
CVE_SOCKET_TIMEOUT ?= "60"

meta/recipes-core/meta/cve-update-db-native.bb +2 -0
+CVE_DB_TEMP_FILE ?= "${CVE_CHECK_DB_DIR}/temp_nvdcve_1.1.db"
+
 python () {
     if not bb.data.inherits_class("cve-check", d):
         raise bb.parse.SkipRecipe("Skip recipe when cve-check class is

not loaded.")

meta/recipes-core/meta/cve-update-db-native.bb +3 -7
@@ -32,19 +34,15 @@ python do_fetch() {
     """
     import bb.utils
     import bb.progress
-    import sqlite3, urllib, urllib.parse, gzip
-    from datetime import date
+    import shutil
 
     bb.utils.export_proxies(d)
 
-    YEAR_START = 2002
-
     db_file = d.getVar("CVE_CHECK_DB_FILE")
     db_dir = os.path.dirname(db_file)
+    db_tmp_file = d.getVar("CVE_DB_TEMP_FILE")
 
-    cve_socket_timeout = int(d.getVar("CVE_SOCKET_TIMEOUT"))
-
-    cleanup_db_download(db_file)
+    cleanup_db_download(db_file, db_tmp_file)
 
     # The NVD database changes once a day, so no need to update more

frequently
# Allow the user to force-update

meta/recipes-core/meta/cve-update-db-native.bb +28 -0
@@ -62,9 +60,55 @@ python do_fetch() {
         pass
 
     bb.utils.mkdirhier(db_dir)
+    if os.path.exists(db_file):
+        shutil.copy2(db_file, db_tmp_file)
+
+    if update_db_file(db_tmp_file, d) == True:
+        # Update downloaded correctly, can swap files
+        shutil.move(db_tmp_file, db_file)
+    else:
+        # Update failed, do not modify the database
+        bb.note("CVE database update failed")
+        os.remove(db_tmp_file)
+}
+
+do_fetch[lockfiles] += "${CVE_CHECK_DB_FILE_LOCK}"
+do_fetch[file-checksums] = ""
+do_fetch[vardeps] = ""
+
+def cleanup_db_download(db_file, db_tmp_file):
+    """
+    Cleanup the download space from possible failed downloads
+    """
+    if os.path.exists("{0}-journal".format(db_file)):
+        # If a journal is present the last update might have been
 interrupted. In that case,
+        # just wipe any leftovers and force the DB to be recreated.
+        os.remove("{0}-journal".format(db_file))
+
+        if os.path.exists(db_file):
+            os.remove(db_file)
+

meta/recipes-core/meta/cve-update-db-native.bb python do_fetch() { · line 101

Quoted 10 lines
meta/recipes-core/meta/cve-update-db-native.bb +8 -0
+    if os.path.exists("{0}-journal".format(db_tmp_file)):
+        # If a journal is present the last update might have been
 interrupted. In that case,
+        # just wipe any leftovers and force the DB to be recreated.
+        os.remove("{0}-journal".format(db_tmp_file))
+
+    if os.path.exists(db_tmp_file):
+        os.remove(db_tmp_file)
+
 

It seems to me that this function is a duplication of the old version with an extra argument.
So I think that using the old function version and call it with the proper argument does the same:

Quoted 4 lines

cleanup_db_download(db_file)
cleanup_db_download(db_tmp_file)

Quoted 7 lines

Hi Jose,
Thanks for looking into that. The function is not a total duplicate: the difference is that
the it always removes the db_tmp_file, not only if the journal file exists (Python code
formatting!).

Don't see on the first time that the db_tmp_file is always removed, I need new glasses :)

Quoted 8 lines

I was hesitating on this part a bit, because with the old path could be taken only in some
specific situations: at the code update and if you share the DL_DIR and some of the builds
use the old code, some the new version. I think we should keep both for now for safety.

makes sense, thanks for your explanation.

Jose

Quoted 4 lines

Kind regards,
Marta

Signature · 3 lines
Best regards,

José Quaresma
#5 lore raw

Re: [OE-core] [PATCH 2/2] cve-update-db-native: avoid partial updates

Quoted 144 lines · Jose Quaresma

Marta Rybczynska <[email protected]> escreveu no dia segunda, 2/01/2023 à(s) 16:38:

Quoted 118 lines
Quoted 109 lines · Jose Quaresma

Hi Marta,

Marta Rybczynska <[email protected]> escreveu no dia segunda, 2/01/2023 à(s) 07:03:

Quoted 88 lines

The database update has been done on the original file. In case of network connection issues, temporary outage of the NVD server or a similar situation, the function could exit with incomplete data in the database. This patch solves the issue by performing the update on a copy of the database. It replaces the main one only if the whole update was successful.

See https://bugzilla.yoctoproject.org/show_bug.cgi?id=14929

---
 .../recipes-core/meta/cve-update-db-native.bb | 81 +++++++++++++------
 1 file changed, 56 insertions(+), 25 deletions(-)
meta/recipes-core/meta/cve-update-db-native.bb +0 -0
index 642fda5395..89804b9e5c 100644
@@ -21,6 +21,8 @@ CVE_DB_UPDATE_INTERVAL ?= "86400"
 # Timeout for blocking socket operations, such as the connection

attempt.
CVE_SOCKET_TIMEOUT ?= "60"

meta/recipes-core/meta/cve-update-db-native.bb +2 -0
+CVE_DB_TEMP_FILE ?= "${CVE_CHECK_DB_DIR}/temp_nvdcve_1.1.db"
+
 python () {
     if not bb.data.inherits_class("cve-check", d):
         raise bb.parse.SkipRecipe("Skip recipe when cve-check class is

not loaded.")

meta/recipes-core/meta/cve-update-db-native.bb +3 -7
@@ -32,19 +34,15 @@ python do_fetch() {
     """
     import bb.utils
     import bb.progress
-    import sqlite3, urllib, urllib.parse, gzip
-    from datetime import date
+    import shutil
 
     bb.utils.export_proxies(d)
 
-    YEAR_START = 2002
-
     db_file = d.getVar("CVE_CHECK_DB_FILE")
     db_dir = os.path.dirname(db_file)
+    db_tmp_file = d.getVar("CVE_DB_TEMP_FILE")
 
-    cve_socket_timeout = int(d.getVar("CVE_SOCKET_TIMEOUT"))
-
-    cleanup_db_download(db_file)
+    cleanup_db_download(db_file, db_tmp_file)
 
     # The NVD database changes once a day, so no need to update more

frequently
# Allow the user to force-update

meta/recipes-core/meta/cve-update-db-native.bb +28 -0
@@ -62,9 +60,55 @@ python do_fetch() {
         pass
 
     bb.utils.mkdirhier(db_dir)
+    if os.path.exists(db_file):
+        shutil.copy2(db_file, db_tmp_file)
+
+    if update_db_file(db_tmp_file, d) == True:
+        # Update downloaded correctly, can swap files
+        shutil.move(db_tmp_file, db_file)
+    else:
+        # Update failed, do not modify the database
+        bb.note("CVE database update failed")
+        os.remove(db_tmp_file)
+}
+
+do_fetch[lockfiles] += "${CVE_CHECK_DB_FILE_LOCK}"
+do_fetch[file-checksums] = ""
+do_fetch[vardeps] = ""
+
+def cleanup_db_download(db_file, db_tmp_file):
+    """
+    Cleanup the download space from possible failed downloads
+    """
+    if os.path.exists("{0}-journal".format(db_file)):
+        # If a journal is present the last update might have been
 interrupted. In that case,
+        # just wipe any leftovers and force the DB to be recreated.
+        os.remove("{0}-journal".format(db_file))
+
+        if os.path.exists(db_file):
+            os.remove(db_file)
+

meta/recipes-core/meta/cve-update-db-native.bb python do_fetch() { · line 101

Quoted 10 lines
meta/recipes-core/meta/cve-update-db-native.bb +8 -0
+    if os.path.exists("{0}-journal".format(db_tmp_file)):
+        # If a journal is present the last update might have been
 interrupted. In that case,
+        # just wipe any leftovers and force the DB to be recreated.
+        os.remove("{0}-journal".format(db_tmp_file))
+
+    if os.path.exists(db_tmp_file):
+        os.remove(db_tmp_file)
+
 

It seems to me that this function is a duplication of the old version with an extra argument.
So I think that using the old function version and call it with the proper argument does the same:

Quoted 4 lines

cleanup_db_download(db_file)
cleanup_db_download(db_tmp_file)

Quoted 7 lines

Hi Jose,
Thanks for looking into that. The function is not a total duplicate: the difference is that
the it always removes the db_tmp_file, not only if the journal file exists (Python code
formatting!).

Don't see on the first time that the db_tmp_file is always removed, I need new glasses :)

Quoted 8 lines

I was hesitating on this part a bit, because with the old path could be taken only in some
specific situations: at the code update and if you share the DL_DIR and some of the builds
use the old code, some the new version. I think we should keep both for now for safety.

Quoted 3 lines

makes sense, thanks for your explanation.

Well... I'll submit a v2 with comments to make sure nobody tries to optimize that piece :)

Regards,
Marta

A message that is not in the archive (sent off-list or not yet received): [email protected]

#7 lore raw

Re: [OE-core] [PATCH 2/2] cve-update-db-native: avoid partial updates

Quoted 24 lines
Quoted 24 lines
Quoted 10 lines

Hi Jose,
Thanks for looking into that. The function is not a total duplicate: the difference is that
the it always removes the db_tmp_file, not only if the journal file exists (Python code
formatting!).

Don't see on the first time that the db_tmp_file is always removed, I need new glasses :)

Quoted 8 lines

I was hesitating on this part a bit, because with the old path could be taken only in some
specific situations: at the code update and if you share the DL_DIR and some of the builds
use the old code, some the new version. I think we should keep both for now for safety.

Quoted 6 lines
Quoted 3 lines

makes sense, thanks for your explanation.

Well... I'll submit a v2 with comments to make sure nobody tries to optimize that piece :)

And here it is (with a minor title change):
https://lists.openembedded.org/g/openembedded-core/message/175355

Regards,
Marta

#8 lore raw

Re: [OE-core] [PATCH 2/2] cve-update-db-native: avoid partial updates

Marta Rybczynska <[email protected]> escreveu no dia quarta, 4/01/2023 à(s) 15:28:

Quoted 26 lines
Quoted 26 lines
Quoted 26 lines
Quoted 9 lines

Hi Jose,
Thanks for looking into that. The function is not a total duplicate:
the difference is that
the it always removes the db_tmp_file, not only if the journal file exists (Python code
formatting!).

Don't see on the first time that the db_tmp_file is always removed, I need new glasses :)

Quoted 8 lines

I was hesitating on this part a bit, because with the old path could be taken only in some
specific situations: at the code update and if you share the DL_DIR and some of the builds
use the old code, some the new version. I think we should keep both for now for safety.

makes sense, thanks for your explanation.

Quoted 7 lines
Quoted 3 lines

Well... I'll submit a v2 with comments to make sure nobody tries to optimize that piece :)

And here it is (with a minor title change):
https://lists.openembedded.org/g/openembedded-core/message/175355

Thanks,
Jose

Quoted 4 lines

Regards,
Marta

Signature · 3 lines
Best regards,

José Quaresma
#9 lore raw
In reply to Marta Rybczynska

This patch is replaced by
https://lists.openembedded.org/g/openembedded-core/message/175355 (with a minor patch title change)

Quoted 52 lines · Marta Rybczynska

Move cleanup of the database after a failed download to a separate function. This will be useful when we have more actions to do in that situation.

---
 meta/recipes-core/meta/cve-update-db-native.bb | 17 ++++++++++-------
 1 file changed, 10 insertions(+), 7 deletions(-)
meta/recipes-core/meta/cve-update-db-native.bb +1 -7
index 9b9dbbd75f..642fda5395 100644
@@ -44,13 +44,7 @@ python do_fetch() {
 
     cve_socket_timeout = int(d.getVar("CVE_SOCKET_TIMEOUT"))
 
-    if os.path.exists("{0}-journal".format(db_file)):
-        # If a journal is present the last update might have been
 interrupted. In that case,
-        # just wipe any leftovers and force the DB to be recreated.
-        os.remove("{0}-journal".format(db_file))
-
-        if os.path.exists(db_file):
-            os.remove(db_file)
+    cleanup_db_download(db_file)
 
     # The NVD database changes once a day, so no need to update more

frequently
# Allow the user to force-update

meta/recipes-core/meta/cve-update-db-native.bb +9 -0
@@ -134,6 +128,15 @@ do_fetch[lockfiles] += "${CVE_CHECK_DB_FILE_LOCK}"
 do_fetch[file-checksums] = ""
 do_fetch[vardeps] = ""
 
+def cleanup_db_download(db_file):
+    if os.path.exists("{0}-journal".format(db_file)):
+        # If a journal is present the last update might have been
 interrupted. In that case,
+        # just wipe any leftovers and force the DB to be recreated.
+        os.remove("{0}-journal".format(db_file))
+
+        if os.path.exists(db_file):
+            os.remove(db_file)
+
 def initialize_db(conn):
     with conn:

c = conn.cursor()
--
2.35.1