cve-check-tool progress reporting

PATCH Superseded André Draszik 2 participants 8 messages Sep 26, 2016 – Sep 28, 2016

lore

Share & tools

lore

André Draszik

author

2016-09-26 15:42 UTC #0 lore raw

cover cve-check-tool progress reporting

These patches implement progress reporting for cve-check-tool when it is downloading the CVE database.

André Draszik

author

2016-09-26 15:42 UTC #1 lore raw

PATCH 1/2 cve-check-tool: convert do_populate_cve_db() from python to sh

This will allow us to easily incorporate progress support via bb.process.run()

Signed-off-by André Draszik <[email protected]>

---
 .../cve-check-tool/cve-check-tool_5.6.4.bb         | 45 +++++++++-------------
 1 file changed, 19 insertions(+), 26 deletions(-)

meta/recipes-devtools/cve-check-tool/cve-check-tool_5.6.4.bb +19 ~~-26~~

index 1165559..5bb22d1 100644
@@ -22,35 +22,28 @@ inherit pkgconfig autotools
 EXTRA_OECONF = "--disable-coverage"
 CFLAGS_append = " -Wno-error=pedantic"
 
-python do_populate_cve_db () {
-    import subprocess
-    import time
-
-    if d.getVar("BB_NO_NETWORK", True) == "1":
-        bb.error("BB_NO_NETWORK is set; Can't update cve-check-tool database, "
-                  "CVEs won't be checked")
+do_populate_cve_db() {
+    if [ "${BB_NO_NETWORK}" = "1" ] ; then
+        bberror "BB_NO_NETWORK is set; Can't update cve-check-tool database, CVEs won't be checked"
         return
+    fi
 
-    bb.utils.export_proxies(d)
     # In case we don't inherit cve-check class, use default values defined in the class.
-    cve_dir = d.getVar("CVE_CHECK_DB_DIR", True) or d.expand("${DL_DIR}/CVE_CHECK")
-    cve_file = d.getVar("CVE_CHECK_TMP_FILE", True) or d.expand("${TMPDIR}/cve_check")
-    cve_cmd = "cve-check-update"
-    cmd = [cve_cmd, "-d", cve_dir]
-    bb.debug(1, "Updating cve-check-tool database located in %s" % cve_dir)
-    try:
-        output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
-        bb.debug(2, "Command '%s' returned:\n%s" % ("\n".join(cmd), output))
-        time_utc = time.gmtime(time.time())
-        time_format = "%Y-%m-%d %H:%M:%S"
-        with open(cve_file, "w") as f:
-            f.write("CVE database was updated on %s UTC\n\n"
-                    % time.strftime(time_format, time_utc))
-
-    except subprocess.CalledProcessError as e:
-        bb.warn("Error in executing cve-check-update: %s (output %s)" % (e, e.output))
-        if bb.data.inherits_class('cve-check', d):
-            bb.warn("Failed to update cve-check-tool database, CVEs won't be checked")
+    cve_dir="${CVE_CHECK_DB_DIR}"
+    cve_file="${CVE_CHECK_TMP_FILE}"
+
+    [ -z "${cve_dir}" ] && cve_dir="${DL_DIR}/CVE_CHECK"
+    [ -z "${cve_file}" ] && cve_file="${TMPDIR}/cve_check"
+
+    bbdebug 2 "Updating cve-check-tool database located in $cve_dir"
+    if cve-check-update -d "$cve_dir" ; then
+        printf "CVE database was updated on %s UTC\n\n" "$(LANG=C date --utc +'%F %T')" > "$cve_file"
+    else
+        bbwarn "Error in executing cve-check-update"
+        if [ "${@'1' if bb.data.inherits_class('cve-check', d) else '0'}" -ne 0 ] ; then
+            bbwarn "Failed to update cve-check-tool database, CVEs won't be checked"
+        fi
+    fi
 }
 
 addtask populate_cve_db after do_populate_sysroot

Signature · 1 line

2.9.3

André Draszik

author

2016-09-26 15:42 UTC #2 lore raw

PATCH 2/2 cve-check-tool: report progress when downloading CVE database

In reply to André Draszik

We add a patch to report the progress, and at the same time inform bitbake that progress can be extracted via the simple 'percent' progress handler.

Signed-off-by André Draszik <[email protected]>

---
 .../cve-check-tool/cve-check-tool_5.6.4.bb         |   2 +
 ...ogress-in-percent-when-downloading-CVE-db.patch | 132 +++++++++++++++++++++
 2 files changed, 134 insertions(+)
 create mode 100644 meta/recipes-devtools/cve-check-tool/files/0001-print-progress-in-percent-when-downloading-CVE-db.patch

meta/recipes-devtools/cve-check-tool/cve-check-tool_5.6.4.bb +2 -0

index 5bb22d1..1baadea 100644
@@ -8,6 +8,7 @@ LIC_FILES_CHKSUM = "file://LICENSE;md5=e8c1458438ead3c34974bc0be3a03ed6"
 
 SRC_URI = "https://github.com/ikeydoherty/${BPN}/releases/download/v${PV}/${BP}.tar.xz \
            file://check-for-malloc_trim-before-using-it.patch \
+           file://0001-print-progress-in-percent-when-downloading-CVE-db.patch \
           "
 
 SRC_URI[md5sum] = "c5f4247140fc9be3bf41491d31a34155"

@@ -48,5 +49,6 @@ do_populate_cve_db() {
 
 addtask populate_cve_db after do_populate_sysroot
 do_populate_cve_db[nostamp] = "1"
+do_populate_cve_db[progress] = "percent"
 
 BBCLASSEXTEND = "native nativesdk"

meta/recipes-devtools/cve-check-tool/files/0001-print-progress-in-percent-when-downloading-CVE-db.patch new file +132 -0

132 lines not on this page · raw message

Signature · 1 line

2.9.3

Burton, Ross

2016-09-28 10:55 UTC #3 lore raw

Re: [PATCH 2/2] cve-check-tool: report progress when downloading CVE database

Quoted 10 lines · André Draszik

++static int progress_callback_new(void *ptr, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow) ++{
++ (void) ultotal;
++ (void) ulnow;
++
++ struct percent_t *percent = (struct percent_t *) ptr;
++
++ if (dltotal && percent) {
++ long diff = percent->end - percent->start;

Quoted 9 lines

++ if (diff) {
++ fprintf(stderr,"completed: %lu%%\r",
percent->start + (diff * dlnow / dltotal));
++ }
++ }
++
++ return 0;
++}

This fails on the autobuilder cluster:

| ../../cve-check-tool-5.6.4/src/library/fetch.c: In function 'progress_callback_new':
| ../../cve-check-tool-5.6.4/src/library/fetch.c:55:54: error: format '%lu' expects argument of type 'long unsigned int', but argument 3 has type 'curl_off_t {aka long long int}' [-Werror=format=]
| fprintf(stderr,"completed: %lu%%\r",
percent->start + (diff * dlnow / dltotal));
| ^

Ross

André Draszik

author

2016-09-28 11:56 UTC #4 lore raw

Re: [PATCH 2/2] cve-check-tool: report progress when downloading CVE database

Ups, sorry for that... :-(

Should I send a patch against master-next, or against master?

Quoted 40 lines · Ross

Quoted 11 lines · André Draszik
++static int progress_callback_new(void *ptr, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow) ++{
++        (void) ultotal;
++        (void) ulnow;
++
++        struct percent_t *percent = (struct percent_t *) ptr; ++
++        if (dltotal && percent) {
++                long diff = percent->end - percent->start;
Quoted 9 lines
++                if (diff) {
++                        fprintf(stderr,"completed: %lu%%\r", percent->start + (diff * dlnow / dltotal));
++                }
++        }
++
++        return 0;
++}
This fails on the autobuilder cluster:
Quoted 2 lines
../../cve-check-tool-5.6.4/src/library/fetch.c: In function
'progress_callback_new':
Quoted 3 lines
../../cve-check-tool-5.6.4/src/library/fetch.c:55:54: error: format '%lu'
expects argument of type 'long unsigned int', but argument 3 has type 'curl_off_t {aka long long int}' [-Werror=format=]
Quoted 2 lines
                         fprintf(stderr,"completed: %lu%%\r",
percent->start + (diff * dlnow / dltotal));
Quoted 2 lines
                                                      ^
Ross

Burton, Ross

2016-09-28 11:58 UTC #5 lore raw

Re: [PATCH 2/2] cve-check-tool: report progress when downloading CVE database

Quoted 4 lines · André Draszik

Ups, sorry for that... :-(
Should I send a patch against master-next, or against master?

Happens to the best of us :) Edit your patch and resend it against master.

Ross

André Draszik

author

2016-09-28 12:05 UTC #6 lore raw

PATCH v2 1/2 cve-check-tool: convert do_populate_cve_db() from python to sh

In reply to André Draszik

This will allow us to easily incorporate progress support via bb.process.run()

Signed-off-by André Draszik <[email protected]>

---
 .../cve-check-tool/cve-check-tool_5.6.4.bb         | 45 +++++++++-------------
 1 file changed, 19 insertions(+), 26 deletions(-)

meta/recipes-devtools/cve-check-tool/cve-check-tool_5.6.4.bb +19 ~~-26~~

index 1165559..5bb22d1 100644
@@ -22,35 +22,28 @@ inherit pkgconfig autotools
 EXTRA_OECONF = "--disable-coverage"
 CFLAGS_append = " -Wno-error=pedantic"
 
-python do_populate_cve_db () {
-    import subprocess
-    import time
-
-    if d.getVar("BB_NO_NETWORK", True) == "1":
-        bb.error("BB_NO_NETWORK is set; Can't update cve-check-tool database, "
-                  "CVEs won't be checked")
+do_populate_cve_db() {
+    if [ "${BB_NO_NETWORK}" = "1" ] ; then
+        bberror "BB_NO_NETWORK is set; Can't update cve-check-tool database, CVEs won't be checked"
         return
+    fi
 
-    bb.utils.export_proxies(d)
     # In case we don't inherit cve-check class, use default values defined in the class.
-    cve_dir = d.getVar("CVE_CHECK_DB_DIR", True) or d.expand("${DL_DIR}/CVE_CHECK")
-    cve_file = d.getVar("CVE_CHECK_TMP_FILE", True) or d.expand("${TMPDIR}/cve_check")
-    cve_cmd = "cve-check-update"
-    cmd = [cve_cmd, "-d", cve_dir]
-    bb.debug(1, "Updating cve-check-tool database located in %s" % cve_dir)
-    try:
-        output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
-        bb.debug(2, "Command '%s' returned:\n%s" % ("\n".join(cmd), output))
-        time_utc = time.gmtime(time.time())
-        time_format = "%Y-%m-%d %H:%M:%S"
-        with open(cve_file, "w") as f:
-            f.write("CVE database was updated on %s UTC\n\n"
-                    % time.strftime(time_format, time_utc))
-
-    except subprocess.CalledProcessError as e:
-        bb.warn("Error in executing cve-check-update: %s (output %s)" % (e, e.output))
-        if bb.data.inherits_class('cve-check', d):
-            bb.warn("Failed to update cve-check-tool database, CVEs won't be checked")
+    cve_dir="${CVE_CHECK_DB_DIR}"
+    cve_file="${CVE_CHECK_TMP_FILE}"
+
+    [ -z "${cve_dir}" ] && cve_dir="${DL_DIR}/CVE_CHECK"
+    [ -z "${cve_file}" ] && cve_file="${TMPDIR}/cve_check"
+
+    bbdebug 2 "Updating cve-check-tool database located in $cve_dir"
+    if cve-check-update -d "$cve_dir" ; then
+        printf "CVE database was updated on %s UTC\n\n" "$(LANG=C date --utc +'%F %T')" > "$cve_file"
+    else
+        bbwarn "Error in executing cve-check-update"
+        if [ "${@'1' if bb.data.inherits_class('cve-check', d) else '0'}" -ne 0 ] ; then
+            bbwarn "Failed to update cve-check-tool database, CVEs won't be checked"
+        fi
+    fi
 }
 
 addtask populate_cve_db after do_populate_sysroot

Signature · 1 line

2.9.3

André Draszik

author

2016-09-28 12:05 UTC #7 lore raw

PATCH v2 2/2 cve-check-tool: report progress when downloading CVE database

We add a patch to report the progress, and at the same time inform bitbake that progress can be extracted via the simple 'percent' progress handler.

Signed-off-by André Draszik <[email protected]>

---
 .../cve-check-tool/cve-check-tool_5.6.4.bb         |   2 +
 ...ogress-in-percent-when-downloading-CVE-db.patch | 135 +++++++++++++++++++++
 2 files changed, 137 insertions(+)
 create mode 100644 meta/recipes-devtools/cve-check-tool/files/0001-print-progress-in-percent-when-downloading-CVE-db.patch

meta/recipes-devtools/cve-check-tool/cve-check-tool_5.6.4.bb +2 -0

index 5bb22d1..1baadea 100644
@@ -8,6 +8,7 @@ LIC_FILES_CHKSUM = "file://LICENSE;md5=e8c1458438ead3c34974bc0be3a03ed6"
 
 SRC_URI = "https://github.com/ikeydoherty/${BPN}/releases/download/v${PV}/${BP}.tar.xz \
            file://check-for-malloc_trim-before-using-it.patch \
+           file://0001-print-progress-in-percent-when-downloading-CVE-db.patch \
           "
 
 SRC_URI[md5sum] = "c5f4247140fc9be3bf41491d31a34155"

@@ -48,5 +49,6 @@ do_populate_cve_db() {
 
 addtask populate_cve_db after do_populate_sysroot
 do_populate_cve_db[nostamp] = "1"
+do_populate_cve_db[progress] = "percent"
 
 BBCLASSEXTEND = "native nativesdk"

meta/recipes-devtools/cve-check-tool/files/0001-print-progress-in-percent-when-downloading-CVE-db.patch new file +135 -0

135 lines not on this page · raw message

Signature · 1 line

2.9.3