Skip to contentSkip to conversation

Improvements to the vuls test

PATCH New Daniel Sangorrin 1 participant 12 messages Dec 27, 2017

lore
Share & tools
lore
#0 lore raw

cover Improvements to the vuls test

Hi Tim,

I send you a few patches for vuls that I wrote on top of the ones that I sent last time. They are on my git repo as well.

Daniel Sangorrin (11):
vuls: make it a benchmark test
vuls: deep scans require root permissions
vuls: rename all_version to versions
vuls: remove gitrepo definition
vuls: add specs for all distros supported by vuls
vuls: fix path during test_build
vuls: remove one-line report
vuls: use full reports
vuls: fix deep argument and do not configtest
vuls: move config.toml to the logdir
vuls: rewrite all parsing related files

 engine/tests/Benchmark.vuls/chart_config.json  |  4 ++
 engine/tests/Benchmark.vuls/criteria.json      | 33 +++++++++
 engine/tests/Benchmark.vuls/fuego_test.sh      | 98 ++++++++++++++++++++++++++
 engine/tests/Benchmark.vuls/parser.py          | 20 ++++++
 engine/tests/Benchmark.vuls/reference.json     | 45 ++++++++++++
 engine/tests/Benchmark.vuls/spec.json          | 35 +++++++++
 engine/tests/Functional.vuls/chart_config.json |  3 -
 engine/tests/Functional.vuls/criteria.json     | 21 ------
 engine/tests/Functional.vuls/fuego_test.sh     | 93 ------------------------
 engine/tests/Functional.vuls/parser.py         | 17 -----
 engine/tests/Functional.vuls/spec.json         | 17 -----
 11 files changed, 235 insertions(+), 151 deletions(-)
 create mode 100644 engine/tests/Benchmark.vuls/chart_config.json
 create mode 100644 engine/tests/Benchmark.vuls/criteria.json
 create mode 100644 engine/tests/Benchmark.vuls/fuego_test.sh
 create mode 100644 engine/tests/Benchmark.vuls/parser.py
 create mode 100644 engine/tests/Benchmark.vuls/reference.json
 create mode 100644 engine/tests/Benchmark.vuls/spec.json
 delete mode 100644 engine/tests/Functional.vuls/chart_config.json
 delete mode 100644 engine/tests/Functional.vuls/criteria.json
 delete mode 100644 engine/tests/Functional.vuls/fuego_test.sh
 delete mode 100644 engine/tests/Functional.vuls/parser.py
 delete mode 100644 engine/tests/Functional.vuls/spec.json
Signature · 1 line
2.7.4
#1 lore raw

PATCH 1/11 vuls: make it a benchmark test

The reason is that we are going to use thresholds against the number of vulnerabilities (a number) so it is easier to represent as a benchmark.

In fact the variables inside fuego_test were already assuming to be a benchmark by mistake.

---
 engine/tests/Benchmark.vuls/chart_config.json  |  3 +
 engine/tests/Benchmark.vuls/criteria.json      | 21 ++++++
 engine/tests/Benchmark.vuls/fuego_test.sh      | 93 ++++++++++++++++++++++++++
 engine/tests/Benchmark.vuls/parser.py          | 17 +++++
 engine/tests/Benchmark.vuls/spec.json          | 17 +++++
 engine/tests/Functional.vuls/chart_config.json |  3 -
 engine/tests/Functional.vuls/criteria.json     | 21 ------
 engine/tests/Functional.vuls/fuego_test.sh     | 93 --------------------------
 engine/tests/Functional.vuls/parser.py         | 17 -----
 engine/tests/Functional.vuls/spec.json         | 17 -----
 10 files changed, 151 insertions(+), 151 deletions(-)
 create mode 100644 engine/tests/Benchmark.vuls/chart_config.json
 create mode 100644 engine/tests/Benchmark.vuls/criteria.json
 create mode 100644 engine/tests/Benchmark.vuls/fuego_test.sh
 create mode 100644 engine/tests/Benchmark.vuls/parser.py
 create mode 100644 engine/tests/Benchmark.vuls/spec.json
 delete mode 100644 engine/tests/Functional.vuls/chart_config.json
 delete mode 100644 engine/tests/Functional.vuls/criteria.json
 delete mode 100644 engine/tests/Functional.vuls/fuego_test.sh
 delete mode 100644 engine/tests/Functional.vuls/parser.py
 delete mode 100644 engine/tests/Functional.vuls/spec.json
engine/tests/Benchmark.vuls/chart_config.json new file +3 -0
new file mode 100644
index 0000000..1849992
@@ -0,0 +1,3 @@
+{
+    "chart_type": "testset_summary_table"
+}
engine/tests/Benchmark.vuls/criteria.json new file +21 -0
new file mode 100644
index 0000000..91c0f8a
@@ -0,0 +1,21 @@
+{
+    "schema_version":"1.0",
+    "criteria":[
+        {
+            "tguid":"HIGH",
+            "max_fail": 0
+        },
+        {
+            "tguid":"MEDIUM",
+            "max_fail": 100
+        },
+        {
+            "tguid":"LOW",
+            "max_fail": 100
+        },
+        {
+            "tguid":"(unknown)",
+            "max_fail": 200
+        }
+    ]
+}
engine/tests/Benchmark.vuls/fuego_test.sh new file +93 -0

93 lines not on this page · raw message

engine/tests/Benchmark.vuls/parser.py new file +17 -0
new file mode 100644
index 0000000..3bfa196
@@ -0,0 +1,17 @@
+#!/bin/python
+
+import os, re, sys
+
+sys.path.insert(0, os.environ['FUEGO_CORE'] + '/engine/scripts/parser')
+import common as plib
+
+results = {}
+with open(plib.TEST_LOG) as f:
+    for line in f:
+        if line.startswith("CVE-"):
+            fields = line.split()
+            test_set = fields[2]
+            test_case = fields[0]
+            results[test_set+'.'+test_case] = "FAIL"
+
+sys.exit(plib.process(results))
engine/tests/Benchmark.vuls/spec.json new file +17 -0
new file mode 100644
index 0000000..60e0d8a
@@ -0,0 +1,17 @@
+{
+    "testName": "Benchmark.vuls",
+    "specs": {
+        "default": {
+            "gitrepo": "https://github.com/future-architect/vuls.git",
+            "distro": "debian",
+            "all_version": "7 8 9 10",
+            "isdeep": "false"
+        },
+        "ubuntu": {
+            "gitrepo": "https://github.com/future-architect/vuls.git",
+            "distro": "ubuntu",
+            "all_version": "12 14 16",
+            "isdeep": "false"
+        }
+    }
+}
engine/tests/Functional.vuls/chart_config.json deleted +0 -3
deleted file mode 100644
index 1849992..0000000
@@ -1,3 +0,0 @@
-{
-    "chart_type": "testset_summary_table"
-}
engine/tests/Functional.vuls/criteria.json deleted +0 -21
deleted file mode 100644
index 91c0f8a..0000000
@@ -1,21 +0,0 @@
-{
-    "schema_version":"1.0",
-    "criteria":[
-        {
-            "tguid":"HIGH",
-            "max_fail": 0
-        },
-        {
-            "tguid":"MEDIUM",
-            "max_fail": 100
-        },
-        {
-            "tguid":"LOW",
-            "max_fail": 100
-        },
-        {
-            "tguid":"(unknown)",
-            "max_fail": 200
-        }
-    ]
-}
engine/tests/Functional.vuls/fuego_test.sh deleted +0 -93

93 lines not on this page · raw message

engine/tests/Functional.vuls/parser.py deleted +0 -17
deleted file mode 100644
index 3bfa196..0000000
@@ -1,17 +0,0 @@
-#!/bin/python
-
-import os, re, sys
-
-sys.path.insert(0, os.environ['FUEGO_CORE'] + '/engine/scripts/parser')
-import common as plib
-
-results = {}
-with open(plib.TEST_LOG) as f:
-    for line in f:
-        if line.startswith("CVE-"):
-            fields = line.split()
-            test_set = fields[2]
-            test_case = fields[0]
-            results[test_set+'.'+test_case] = "FAIL"
-
-sys.exit(plib.process(results))
engine/tests/Functional.vuls/spec.json deleted +0 -17
deleted file mode 100644
index 60e0d8a..0000000
@@ -1,17 +0,0 @@
-{
-    "testName": "Benchmark.vuls",
-    "specs": {
-        "default": {
-            "gitrepo": "https://github.com/future-architect/vuls.git",
-            "distro": "debian",
-            "all_version": "7 8 9 10",
-            "isdeep": "false"
-        },
-        "ubuntu": {
-            "gitrepo": "https://github.com/future-architect/vuls.git",
-            "distro": "ubuntu",
-            "all_version": "12 14 16",
-            "isdeep": "false"
-        }
-    }
-}
Signature · 1 line
2.7.4
#2 lore raw

PATCH 2/11 vuls: deep scans require root permissions

In reply to Daniel Sangorrin
---
 engine/tests/Benchmark.vuls/fuego_test.sh | 6 ++++++
 1 file changed, 6 insertions(+)
engine/tests/Benchmark.vuls/fuego_test.sh +6 -0
index 068acc4..d7d298e 100644
@@ -27,6 +27,12 @@ function test_pre_check {
         abort_job "Run on docker as root: ssh-keyscan -t ecdsa ${IPADDR} >> /etc/ssh/ssh_known_hosts"
     fi
 
+    if [ "${BENCHMARK_VULS_ISDEEP}" == "true" ]; then
+        if [ "${LOGIN}" != "root" ]; then
+            abort_job "You need to login as root for vuls deep scan"
+        fi
+    fi
+
     # Make sure that go was installed in the system
     is_on_target go PROGRAM_GO $PATH
     assert_define PROGRAM_GO "run /fuego-ro/scripts/fuego-vuls-docker-preparation"
Signature · 1 line
2.7.4
#3 lore raw

PATCH 3/11 vuls: rename all_version to versions

In reply to Daniel Sangorrin

versions referes to the distro version. E.g. for debian it could be 7, 8 etc.

---
 engine/tests/Benchmark.vuls/fuego_test.sh | 4 ++--
 engine/tests/Benchmark.vuls/spec.json     | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)
engine/tests/Benchmark.vuls/fuego_test.sh +2 -2
index d7d298e..8a434f1 100644
@@ -66,12 +66,12 @@ function test_build {
         for i in `seq 2002 $(date +"%Y")`; do
             go-cve-dictionary fetchnvd -http-proxy=$http_proxy -years $i
         done
-        goval-dictionary fetch-${BENCHMARK_VULS_DISTRO} ${BENCHMARK_VULS_ALL_VERSION} -http-proxy ${http_proxy}
+        goval-dictionary fetch-${BENCHMARK_VULS_DISTRO} ${BENCHMARK_VULS_VERSIONS} -http-proxy ${http_proxy}
     else
         for i in `seq 2002 $(date +"%Y")`; do
             go-cve-dictionary fetchnvd -years $i
         done
-        goval-dictionary fetch-${BENCHMARK_VULS_DISTRO} ${BENCHMARK_VULS_ALL_VERSION}
+        goval-dictionary fetch-${BENCHMARK_VULS_DISTRO} ${BENCHMARK_VULS_VERSIONS}
     fi
 
     # Create config.toml file
engine/tests/Benchmark.vuls/spec.json +2 -2
index 60e0d8a..89befeb 100644
@@ -4,13 +4,13 @@
         "default": {
             "gitrepo": "https://github.com/future-architect/vuls.git",
             "distro": "debian",
-            "all_version": "7 8 9 10",
+            "versions": "7 8 9 10",
             "isdeep": "false"
         },
         "ubuntu": {
             "gitrepo": "https://github.com/future-architect/vuls.git",
             "distro": "ubuntu",
-            "all_version": "12 14 16",
+            "versions": "12 14 16",
             "isdeep": "false"
         }
     }
Signature · 1 line
2.7.4
#4 lore raw

PATCH 4/11 vuls: remove gitrepo definition

In reply to Daniel Sangorrin

The test already clones the repository inside the build phase. Note that it also install other go packages during build (only once) so docker needs to have internet access.

---
 engine/tests/Benchmark.vuls/spec.json | 2 --
 1 file changed, 2 deletions(-)
engine/tests/Benchmark.vuls/spec.json +0 -2
index 89befeb..439c737 100644
@@ -2,13 +2,11 @@
     "testName": "Benchmark.vuls",
     "specs": {
         "default": {
-            "gitrepo": "https://github.com/future-architect/vuls.git",
             "distro": "debian",
             "versions": "7 8 9 10",
             "isdeep": "false"
         },
         "ubuntu": {
-            "gitrepo": "https://github.com/future-architect/vuls.git",
             "distro": "ubuntu",
             "versions": "12 14 16",
             "isdeep": "false"
Signature · 1 line
2.7.4
#5 lore raw

PATCH 5/11 vuls: add specs for all distros supported by vuls

In reply to Daniel Sangorrin

These specs are untested but they should work according to vuls help.

FIXTHIS: we need to add pre_checks for each distro, for the moment the user must check vuls documentation and make sure that the target machine has the necessary requirements.

---
 engine/tests/Benchmark.vuls/spec.json | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)
engine/tests/Benchmark.vuls/spec.json +20 -0
index 439c737..684bb70 100644
@@ -10,6 +10,26 @@
             "distro": "ubuntu",
             "versions": "12 14 16",
             "isdeep": "false"
+        },
+        "suse": {
+            "distro": "suse",
+            "versions": "13.2",
+            "isdeep": "false"
+        },
+        "redhat": {
+            "distro": "redhat",
+            "versions": "5 6 7",
+            "isdeep": "false"
+        },
+        "alpine": {
+            "distro": "alpine",
+            "versions": "3.3 3.4 3.5 3.6",
+            "isdeep": "false"
+        },
+        "oracle": {
+            "distro": "oracle",
+            "versions": "5 6 7",
+            "isdeep": "false"
         }
     }
 }
Signature · 1 line
2.7.4
#6 lore raw

PATCH 6/11 vuls: fix path during test_build

In reply to Daniel Sangorrin

This was working by chance.

---
 engine/tests/Benchmark.vuls/fuego_test.sh | 3 +++
 1 file changed, 3 insertions(+)
engine/tests/Benchmark.vuls/fuego_test.sh +3 -0
index 8a434f1..13f05ed 100644
@@ -61,6 +61,9 @@ function test_build {
         make install
     fi
 
+    # Share the database creation on a common directory
+    cd $GOPATH/src/github.com/future-architect/vuls
+
     # Fetch vulnerability data from NVD and OVAL data
     if [ -n "${http_proxy}" ]; then
         for i in `seq 2002 $(date +"%Y")`; do
Signature · 1 line
2.7.4
#7 lore raw

PATCH 7/11 vuls: remove one-line report

In reply to Daniel Sangorrin

This report was not being used/parsed and that line also appears in the normal report so it is not required.

---
 engine/tests/Benchmark.vuls/fuego_test.sh | 1 -
 1 file changed, 1 deletion(-)
engine/tests/Benchmark.vuls/fuego_test.sh +0 -1
index 13f05ed..086ad5c 100644
@@ -97,6 +97,5 @@ function test_run {
     # Start Scanning
     vuls scan ${BENCHMARK_VULS_DISTRO}
     # Reporting
-    vuls report -format-one-line-text -cvedb-path=$PWD/cve.sqlite3 -ovaldb-path=$PWD/oval.sqlite3
     vuls report -format-short-text | tee ${LOGDIR}/testlog.txt
 }
Signature · 1 line
2.7.4
#8 lore raw

PATCH 8/11 vuls: use full reports

In reply to Daniel Sangorrin

short reports do not show the packages affected so it is harder to know what to upgrade.

---
 engine/tests/Benchmark.vuls/fuego_test.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
engine/tests/Benchmark.vuls/fuego_test.sh +1 -1
index 086ad5c..3d1a7a9 100644
@@ -97,5 +97,5 @@ function test_run {
     # Start Scanning
     vuls scan ${BENCHMARK_VULS_DISTRO}
     # Reporting
-    vuls report -format-short-text | tee ${LOGDIR}/testlog.txt
+    vuls report -format-full-text | tee ${LOGDIR}/testlog.txt
 }
Signature · 1 line
2.7.4
#9 lore raw

PATCH 9/11 vuls: fix deep argument and do not configtest

In reply to Daniel Sangorrin

The deep argument was not correctly being passed, also configtest is not necessary because scan will report similar errors in case the ssh connection fails.

---
 engine/tests/Benchmark.vuls/fuego_test.sh | 12 +++++-------
 1 file changed, 5 insertions(+), 7 deletions(-)
engine/tests/Benchmark.vuls/fuego_test.sh +5 -7
index 3d1a7a9..d47bfca 100644
@@ -88,14 +88,12 @@ function test_build {
 
 function test_run {
     cd $GOPATH/src/github.com/future-architect/vuls
-    # Check config.toml and settings on the server before scanning
+
     if [ "${BENCHMARK_VULS_ISDEEP}" = "true" ]; then
-        vuls configtest -deep ${BENCHMARK_VULS_DISTRO}
-    else
-        vuls configtest ${BENCHMARK_VULS_DISTRO}
+        VULS_DEEP_FLAG="-deep"
     fi
-    # Start Scanning
-    vuls scan ${BENCHMARK_VULS_DISTRO}
-    # Reporting
+
+    cd $GOPATH/src/github.com/future-architect/vuls
+    vuls scan ${VULS_DEEP_FLAG} ${BENCHMARK_VULS_DISTRO}
     vuls report -format-full-text | tee ${LOGDIR}/testlog.txt
 }
Signature · 1 line
2.7.4
#10 lore raw

PATCH 10/11 vuls: move config.toml to the logdir

In reply to Daniel Sangorrin

config.toml was being generated on a folder that was shared by different boards and jobs. Instead generate the config.toml on the LOGDIR to have separate files for each board and to save the configuration used for possible inspection.

---
 engine/tests/Benchmark.vuls/fuego_test.sh | 21 ++++++++++-----------
 1 file changed, 10 insertions(+), 11 deletions(-)
engine/tests/Benchmark.vuls/fuego_test.sh +10 -11
index d47bfca..b9592ba 100644
@@ -76,24 +76,23 @@ function test_build {
         done
         goval-dictionary fetch-${BENCHMARK_VULS_DISTRO} ${BENCHMARK_VULS_VERSIONS}
     fi
-
-    # Create config.toml file
-    echo "[servers]" > config.toml
-    echo "[servers.${BENCHMARK_VULS_DISTRO}]" >> config.toml
-    echo "host        = \"${IPADDR}\"" >> config.toml
-    echo "port        = \"${SSH_PORT}\"" >> config.toml
-    echo "user        = \"${LOGIN}\"" >> config.toml
-    echo "keyPath     = \"${SSH_KEY}\"" >> config.toml
 }
 
 function test_run {
-    cd $GOPATH/src/github.com/future-architect/vuls
+    # Create config.toml file on the log folder (per run, not shared)
+    echo "[servers]" > ${LOGDIR}/config.toml
+    echo "[servers.${BENCHMARK_VULS_DISTRO}]" >> ${LOGDIR}/config.toml
+    echo "host        = \"${IPADDR}\"" >> ${LOGDIR}/config.toml
+    echo "port        = \"${SSH_PORT}\"" >> ${LOGDIR}/config.toml
+    echo "user        = \"${LOGIN}\"" >> ${LOGDIR}/config.toml
+    echo "keyPath     = \"${SSH_KEY}\"" >> ${LOGDIR}/config.toml
 
+    VULS_OPTIONSFLAG="-config=${LOGDIR}/config.toml -results-dir=${LOGDIR}"
     if [ "${BENCHMARK_VULS_ISDEEP}" = "true" ]; then
         VULS_DEEP_FLAG="-deep"
     fi
 
     cd $GOPATH/src/github.com/future-architect/vuls
-    vuls scan ${VULS_DEEP_FLAG} ${BENCHMARK_VULS_DISTRO}
-    vuls report -format-full-text | tee ${LOGDIR}/testlog.txt
+    vuls scan ${VULS_DEEP_FLAG} ${VULS_OPTIONSFLAG} ${BENCHMARK_VULS_DISTRO}
+    vuls report ${VULS_OPTIONSFLAG} -format-full-text | tee ${LOGDIR}/testlog.txt
 }
Signature · 1 line
2.7.4
#11 lore raw

PATCH 11/11 vuls: rewrite all parsing related files

In reply to Daniel Sangorrin

Before the parser was traversing the vuls report and creating a test case per CVE found. This was not very efficient and produced too many information in the run.json.

Instead we can do something more simple: parse only the summary line that classifies vulnerabilities in 4 groups (HIGH, MEDIUM, LOW and UNKNOWN) and apply a criteria on them. On Jenkins we will only show HIGH and MEDIUM as a plot.

---
 engine/tests/Benchmark.vuls/chart_config.json |  3 +-
 engine/tests/Benchmark.vuls/criteria.json     | 28 ++++++++++++-----
 engine/tests/Benchmark.vuls/parser.py         | 25 ++++++++-------
 engine/tests/Benchmark.vuls/reference.json    | 45 +++++++++++++++++++++++++++
 4 files changed, 81 insertions(+), 20 deletions(-)
 create mode 100644 engine/tests/Benchmark.vuls/reference.json
engine/tests/Benchmark.vuls/chart_config.json +2 -1
index 1849992..9f51d0f 100644
@@ -1,3 +1,4 @@
 {
-    "chart_type": "testset_summary_table"
+    "chart_type": "measure_plot",
+    "measures": ["default.HIGH.Count", "default.MEDIUM.Count"]
 }
engine/tests/Benchmark.vuls/criteria.json +20 -8
index 91c0f8a..beaccaf 100644
@@ -2,20 +2,32 @@
     "schema_version":"1.0",
     "criteria":[
         {
-            "tguid":"HIGH",
-            "max_fail": 0
+            "tguid":"default.HIGH.Count",
+            "reference":{
+                "value":0,
+                "operator":"le"
+            }
         },
         {
-            "tguid":"MEDIUM",
-            "max_fail": 100
+            "tguid":"default.MEDIUM.Count",
+            "reference":{
+                "value":100,
+                "operator":"le"
+            }
         },
         {
-            "tguid":"LOW",
-            "max_fail": 100
+            "tguid":"default.LOW.Count",
+            "reference":{
+                "value":200,
+                "operator":"le"
+            }
         },
         {
-            "tguid":"(unknown)",
-            "max_fail": 200
+            "tguid":"default.UNKNOWN.Count",
+            "reference":{
+                "value":100,
+                "operator":"le"
+            }
         }
     ]
 }
engine/tests/Benchmark.vuls/parser.py +14 -11
index 3bfa196..e287a53 100644
@@ -1,17 +1,20 @@
-#!/bin/python
+#!/usr/bin/python
+# See common.py for description of command-line arguments
 
-import os, re, sys
+import os, sys
 
 sys.path.insert(0, os.environ['FUEGO_CORE'] + '/engine/scripts/parser')
 import common as plib
 
-results = {}
-with open(plib.TEST_LOG) as f:
-    for line in f:
-        if line.startswith("CVE-"):
-            fields = line.split()
-            test_set = fields[2]
-            test_case = fields[0]
-            results[test_set+'.'+test_case] = "FAIL"
+measurements = {}
 
-sys.exit(plib.process(results))
+regex_string = '.*\(High:(\d+) Medium:(\d+) Low:(\d+) \?:(\d+)\)'
+matches = plib.parse_log(regex_string)
+
+if matches:
+    measurements['default.HIGH']    = [{"name": "Count", "measure" : float(matches[0][0])}]
+    measurements['default.MEDIUM']  = [{"name": "Count", "measure" : float(matches[0][1])}]
+    measurements['default.LOW']     = [{"name": "Count", "measure" : float(matches[0][2])}]
+    measurements['default.UNKNOWN'] = [{"name": "Count", "measure" : float(matches[0][3])}]
+
+sys.exit(plib.process(measurements))
engine/tests/Benchmark.vuls/reference.json new file +45 -0

45 lines not on this page · raw message

Signature · 1 line
2.7.4