cve-update-db: do_populate_cve_db depends on do_fetch
PATCH v5 1/1 cve-update-db: do_populate_cve_db depends on do_fetch
From: Pierre Le Magourou <[email protected]>
To be able to populate NVD database on a fetchall
(bitbake <image> --run-all=fetch), set the do_populate_cve_db task to be executed before do_fetch.
Do not get CVE_CHECK_DB_DIR, CVE_CHECK_DB_FILE and CVE_CHECK_TMP_FILE variable because do_populate_cve_db can be called in a context where cve-check class is not loaded.
--- meta/recipes-core/meta/cve-update-db.bb | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-)
meta/recipes-core/meta/cve-update-db.bb +13 -8
@@ -6,7 +6,6 @@ PACKAGES = "" inherit nopackages -deltask do_fetch deltask do_unpack deltask do_patch deltask do_configure
@@ -24,11 +23,16 @@ python do_populate_cve_db() { BASE_URL = "https://nvd.nist.gov/feeds/json/cve/1.0/nvdcve-1.0-" YEAR_START = 2002 - JSON_TMPFILE = d.getVar("CVE_CHECK_DB_DIR") + '/nvd.json.gz' + + db_dir = d.getVar("DL_DIR") + '/CVE_CHECK' + db_file = db_dir + '/nvd-json.db' + json_tmpfile = db_dir + '/nvd.json.gz' proxy = d.getVar("https_proxy") + if not os.path.isdir(db_dir): + os.mkdir(db_dir) + # Connect to database - db_file = d.getVar("CVE_CHECK_DB_FILE") conn = sqlite3.connect(db_file) c = conn.cursor()
@@ -55,9 +59,9 @@ python do_populate_cve_db() { req = urllib.request.Request(json_url) if proxy: req.set_proxy(proxy, 'https') - with urllib.request.urlopen(req) as r, open(JSON_TMPFILE, 'wb') as tmpfile: + with urllib.request.urlopen(req) as r, open(json_tmpfile, 'wb') as tmpfile: shutil.copyfileobj(r, tmpfile) - with gzip.open(JSON_TMPFILE, 'rt') as jsonfile: + with gzip.open(json_tmpfile, 'rt') as jsonfile: update_db(c, jsonfile) c.execute("insert or replace into META values (?, ?)", [year, last_modified])
@@ -65,8 +69,9 @@ python do_populate_cve_db() { conn.commit() conn.close() - with open(d.getVar("CVE_CHECK_TMP_FILE"), 'a'): - os.utime(d.getVar("CVE_CHECK_TMP_FILE"), None) + cve_check_tmp_file = d.getVar("TMPDIR") + '/cve_check' + with open(cve_check_tmp_file, 'a'): + os.utime(cve_check_tmp_file, None) } # DJB2 hash algorithm
@@ -120,7 +125,7 @@ def update_db(c, json_filename): -addtask do_populate_cve_db before do_cve_check +addtask do_populate_cve_db before do_fetch do_populate_cve_db[nostamp] = "1" EXCLUDE_FROM_WORLD = "1"
Signature · 1 line
2.11.0
Hi Pierre,
This patch broke "bitbake universe --runall=fetch" totally:
ERROR: Running idle function
Traceback (most recent call last):
File "/buildarea1/lyang1/poky/bitbake/lib/bb/runqueue.py", line 1500, inRunQueue.execute_runqueue():
try:
> return self._execute_runqueue()
except bb.runqueue.TaskFailure:
File "/buildarea1/lyang1/poky/bitbake/lib/bb/runqueue.py", line 1420, inRunQueue._execute_runqueue():
[43, 967, 4,
3, 1, 5, 3, 7, 13, 1, 2, 1, 1, 246, 35, 1, 38, 1, 35, 2, 338, 204, 142, 3, 3, 37, 244])
Quoted 1 line
if self.rqdata.prepare() == 0:
self.state = runQueueComplete
File "/buildarea1/lyang1/poky/bitbake/lib/bb/runqueue.py", line 906, inRunQueueData.prepare():
for tid in list(runall_tids):
> mark_active(tid,1)
File "/buildarea1/lyang1/poky/bitbake/lib/bb/runqueue.py", line 837, inmark_active(tid='/buildarea1/lyang1/poky/meta/recipes-core/meta/cve-update-db.bb:do_fetch', depth=1):
for depend in depends:
> mark_active(depend, depth+1)
File "/buildarea1/lyang1/poky/bitbake/lib/bb/runqueue.py", line 835, inmark_active(tid='/buildarea1/lyang1/poky/meta/recipes-core/meta/cve-update-db.bb:do_populate_cve_db', depth=2):
Quoted 1 line
depends = self.runtaskentries[tid].depends
for depend in depends:
KeyError: '/buildarea1/lyang1/poky/meta/recipes-core/meta/cve-update-db.bb:do_populate_cve_db'
The line which caused the problem is:
addtask do_populate_cve_db before do_fetch
Would you please fix it? Or I can help if needed.
// Robert
Quoted 80 lines · Pierre Le Magourou
From: Pierre Le Magourou <[email protected]>
To be able to populate NVD database on a fetchall
(bitbake <image> --run-all=fetch), set thedo_populate_cve_dbtask to be executed beforedo_fetch.Do not get
CVE_CHECK_DB_DIR,CVE_CHECK_DB_FILEandCVE_CHECK_TMP_FILEvariable becausedo_populate_cve_dbcan be called in a context where cve-check class is not loaded.Signed-off-by Pierre Le Magourou <[email protected]>--- meta/recipes-core/meta/cve-update-db.bb | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-)meta/recipes-core/meta/cve-update-db.bb +13
-8@@ -6,7 +6,6 @@ PACKAGES = "" inherit nopackages -deltask do_fetch deltask do_unpack deltask do_patch deltask do_configure@@ -24,11 +23,16 @@ python do_populate_cve_db() { BASE_URL = "https://nvd.nist.gov/feeds/json/cve/1.0/nvdcve-1.0-" YEAR_START = 2002 - JSON_TMPFILE = d.getVar("CVE_CHECK_DB_DIR") + '/nvd.json.gz' + + db_dir = d.getVar("DL_DIR") + '/CVE_CHECK' + db_file = db_dir + '/nvd-json.db' + json_tmpfile = db_dir + '/nvd.json.gz' proxy = d.getVar("https_proxy") + if not os.path.isdir(db_dir): + os.mkdir(db_dir) + # Connect to database - db_file = d.getVar("CVE_CHECK_DB_FILE") conn = sqlite3.connect(db_file) c = conn.cursor()@@ -55,9 +59,9 @@ python do_populate_cve_db() { req = urllib.request.Request(json_url) if proxy: req.set_proxy(proxy, 'https') - with urllib.request.urlopen(req) as r, open(JSON_TMPFILE, 'wb') as tmpfile: + with urllib.request.urlopen(req) as r, open(json_tmpfile, 'wb') as tmpfile: shutil.copyfileobj(r, tmpfile) - with gzip.open(JSON_TMPFILE, 'rt') as jsonfile: + with gzip.open(json_tmpfile, 'rt') as jsonfile: update_db(c, jsonfile) c.execute("insert or replace into META values (?, ?)", [year, last_modified])@@ -65,8 +69,9 @@ python do_populate_cve_db() { conn.commit() conn.close() - with open(d.getVar("CVE_CHECK_TMP_FILE"), 'a'): - os.utime(d.getVar("CVE_CHECK_TMP_FILE"), None) + cve_check_tmp_file = d.getVar("TMPDIR") + '/cve_check' + with open(cve_check_tmp_file, 'a'): + os.utime(cve_check_tmp_file, None) } # DJB2 hash algorithm@@ -120,7 +125,7 @@ def update_db(c, json_filename): -addtask do_populate_cve_db before do_cve_check +addtask do_populate_cve_db before do_fetch do_populate_cve_db[nostamp] = "1" EXCLUDE_FROM_WORLD = "1"
Hello,
Quoted 7 lines
This patch broke "bitbake universe --runall=fetch" totally:
The line which caused the problem is:
addtaskdo_populate_cve_dbbeforedo_fetchWould you please fix it? Or I can help if needed.
This seems related to the NVD website down problem. I sent a patch to manage inaccessible URLs. cve-update-db will not fail anymore if the NVD data feeds cannot be downloaded.
Pierre
Hi Pierre,
Did you try
$ bitbake universe --runall=fetch
Please? It still doesn't work on latest master branch.
ERROR: Running idle function
Traceback (most recent call last):
File "/buildarea1/lyang1/poky/bitbake/lib/bb/runqueue.py", line 1500, inRunQueue.execute_runqueue():
try:
> return self._execute_runqueue()
except bb.runqueue.TaskFailure:
File "/buildarea1/lyang1/poky/bitbake/lib/bb/runqueue.py", line 1420, inRunQueue._execute_runqueue():
[43, 967, 4,
3, 1, 5, 3, 7, 13, 1, 2, 1, 1, 246, 35, 1, 38, 1, 35, 2, 338, 204, 142, 3, 3, 37, 244])
Quoted 1 line
if self.rqdata.prepare() == 0:
self.state = runQueueComplete
File "/buildarea1/lyang1/poky/bitbake/lib/bb/runqueue.py", line 906, inRunQueueData.prepare():
for tid in list(runall_tids):
> mark_active(tid,1)
File "/buildarea1/lyang1/poky/bitbake/lib/bb/runqueue.py", line 837, inmark_active(tid='/buildarea1/lyang1/poky/meta/recipes-core/meta/cve-update-db.bb:do_fetch', depth=1):
for depend in depends:
> mark_active(depend, depth+1)
File "/buildarea1/lyang1/poky/bitbake/lib/bb/runqueue.py", line 835, inmark_active(tid='/buildarea1/lyang1/poky/meta/recipes-core/meta/cve-update-db.bb:do_populate_cve_db', depth=2):
Quoted 1 line
depends = self.runtaskentries[tid].depends
for depend in depends:
KeyError: '/buildarea1/lyang1/poky/meta/recipes-core/meta/cve-update-db.bb:do_populate_cve_db'
// Robert
Quoted 16 lines · Pierre Le Magourou
Hello,
Quoted 7 lines
This patch broke "bitbake universe --runall=fetch" totally:
The line which caused the problem is:
addtaskdo_populate_cve_dbbeforedo_fetchWould you please fix it? Or I can help if needed.
This seems related to the NVD website down problem. I sent a patch to manage inaccessible URLs. cve-update-db will not fail anymore if the NVD data feeds cannot be downloaded.
Pierre
Hi Pierre,
Did you try
$ bitbake universe --runall=fetch
Please? It still doesn't work on latest master branch.
ERROR: Running idle function
Traceback (most recent call last):
File "/buildarea1/lyang1/poky/bitbake/lib/bb/runqueue.py", line 1500, inRunQueue.execute_runqueue():
try:
> return self._execute_runqueue()
except bb.runqueue.TaskFailure:
File "/buildarea1/lyang1/poky/bitbake/lib/bb/runqueue.py", line 1420, inRunQueue._execute_runqueue():
[43, 967, 4,
3, 1, 5, 3, 7, 13, 1, 2, 1, 1, 246, 35, 1, 38, 1, 35, 2, 338, 204, 142, 3, 3, 37, 244])
Quoted 1 line
if self.rqdata.prepare() == 0:
self.state = runQueueComplete
File "/buildarea1/lyang1/poky/bitbake/lib/bb/runqueue.py", line 906, inRunQueueData.prepare():
for tid in list(runall_tids):
> mark_active(tid,1)
File "/buildarea1/lyang1/poky/bitbake/lib/bb/runqueue.py", line 837, inmark_active(tid='/buildarea1/lyang1/poky/meta/recipes-core/meta/cve-update-db.bb:do_fetch', depth=1):
for depend in depends:
> mark_active(depend, depth+1)
File "/buildarea1/lyang1/poky/bitbake/lib/bb/runqueue.py", line 835, inmark_active(tid='/buildarea1/lyang1/poky/meta/recipes-core/meta/cve-update-db.bb:do_populate_cve_db', depth=2):
Quoted 1 line
depends = self.runtaskentries[tid].depends
for depend in depends:
KeyError: '/buildarea1/lyang1/poky/meta/recipes-core/meta/cve-update-db.bb:do_populate_cve_db'
// Robert
Quoted 16 lines · Pierre Le Magourou
Hello,
Quoted 7 lines
This patch broke "bitbake universe --runall=fetch" totally:
The line which caused the problem is:
addtaskdo_populate_cve_dbbeforedo_fetchWould you please fix it? Or I can help if needed.
This seems related to the NVD website down problem. I sent a patch to manage inaccessible URLs. cve-update-db will not fail anymore if the NVD data feeds cannot be downloaded.
Pierre
Hello,
Quoted 6 lines
Did you try
$ bitbake universe --runall=fetchPlease? It still doesn't work on latest master branch.
I sent a patch to skip cve-update-db recipe if the cve-check class is not loaded. This should fix the problem.
Pierre