Improvements to the vuls test
cover Improvements to the vuls test
Hi Tim,
I send you a few patches for vuls that I wrote on top of the ones that I sent last time. They are on my git repo as well.
Daniel Sangorrin (11):
vuls: make it a benchmark test
vuls: deep scans require root permissions
vuls: rename all_version to versions
vuls: remove gitrepo definition
vuls: add specs for all distros supported by vuls
vuls: fix path during test_build
vuls: remove one-line report
vuls: use full reports
vuls: fix deep argument and do not configtest
vuls: move config.toml to the logdir
vuls: rewrite all parsing related files
engine/tests/Benchmark.vuls/chart_config.json | 4 ++ engine/tests/Benchmark.vuls/criteria.json | 33 +++++++++ engine/tests/Benchmark.vuls/fuego_test.sh | 98 ++++++++++++++++++++++++++ engine/tests/Benchmark.vuls/parser.py | 20 ++++++ engine/tests/Benchmark.vuls/reference.json | 45 ++++++++++++ engine/tests/Benchmark.vuls/spec.json | 35 +++++++++ engine/tests/Functional.vuls/chart_config.json | 3 - engine/tests/Functional.vuls/criteria.json | 21 ------ engine/tests/Functional.vuls/fuego_test.sh | 93 ------------------------ engine/tests/Functional.vuls/parser.py | 17 ----- engine/tests/Functional.vuls/spec.json | 17 ----- 11 files changed, 235 insertions(+), 151 deletions(-) create mode 100644 engine/tests/Benchmark.vuls/chart_config.json create mode 100644 engine/tests/Benchmark.vuls/criteria.json create mode 100644 engine/tests/Benchmark.vuls/fuego_test.sh create mode 100644 engine/tests/Benchmark.vuls/parser.py create mode 100644 engine/tests/Benchmark.vuls/reference.json create mode 100644 engine/tests/Benchmark.vuls/spec.json delete mode 100644 engine/tests/Functional.vuls/chart_config.json delete mode 100644 engine/tests/Functional.vuls/criteria.json delete mode 100644 engine/tests/Functional.vuls/fuego_test.sh delete mode 100644 engine/tests/Functional.vuls/parser.py delete mode 100644 engine/tests/Functional.vuls/spec.json
Signature · 1 line
2.7.4
PATCH 1/11 vuls: make it a benchmark test
The reason is that we are going to use thresholds against the number of vulnerabilities (a number) so it is easier to represent as a benchmark.
In fact the variables inside fuego_test were already assuming to be a benchmark by mistake.
--- engine/tests/Benchmark.vuls/chart_config.json | 3 + engine/tests/Benchmark.vuls/criteria.json | 21 ++++++ engine/tests/Benchmark.vuls/fuego_test.sh | 93 ++++++++++++++++++++++++++ engine/tests/Benchmark.vuls/parser.py | 17 +++++ engine/tests/Benchmark.vuls/spec.json | 17 +++++ engine/tests/Functional.vuls/chart_config.json | 3 - engine/tests/Functional.vuls/criteria.json | 21 ------ engine/tests/Functional.vuls/fuego_test.sh | 93 -------------------------- engine/tests/Functional.vuls/parser.py | 17 ----- engine/tests/Functional.vuls/spec.json | 17 ----- 10 files changed, 151 insertions(+), 151 deletions(-) create mode 100644 engine/tests/Benchmark.vuls/chart_config.json create mode 100644 engine/tests/Benchmark.vuls/criteria.json create mode 100644 engine/tests/Benchmark.vuls/fuego_test.sh create mode 100644 engine/tests/Benchmark.vuls/parser.py create mode 100644 engine/tests/Benchmark.vuls/spec.json delete mode 100644 engine/tests/Functional.vuls/chart_config.json delete mode 100644 engine/tests/Functional.vuls/criteria.json delete mode 100644 engine/tests/Functional.vuls/fuego_test.sh delete mode 100644 engine/tests/Functional.vuls/parser.py delete mode 100644 engine/tests/Functional.vuls/spec.json
engine/tests/Benchmark.vuls/chart_config.json new file +3 -0
@@ -0,0 +1,3 @@ +{ + "chart_type": "testset_summary_table" +}
engine/tests/Benchmark.vuls/criteria.json new file +21 -0
@@ -0,0 +1,21 @@ +{ + "schema_version":"1.0", + "criteria":[ + { + "tguid":"HIGH", + "max_fail": 0 + }, + { + "tguid":"MEDIUM", + "max_fail": 100 + }, + { + "tguid":"LOW", + "max_fail": 100 + }, + { + "tguid":"(unknown)", + "max_fail": 200 + } + ] +}
engine/tests/Benchmark.vuls/fuego_test.sh new file +93 -0
93 lines not on this page · raw message
engine/tests/Benchmark.vuls/parser.py new file +17 -0
@@ -0,0 +1,17 @@ +#!/bin/python + +import os, re, sys + +sys.path.insert(0, os.environ['FUEGO_CORE'] + '/engine/scripts/parser') +import common as plib + +results = {} +with open(plib.TEST_LOG) as f: + for line in f: + if line.startswith("CVE-"): + fields = line.split() + test_set = fields[2] + test_case = fields[0] + results[test_set+'.'+test_case] = "FAIL" + +sys.exit(plib.process(results))
engine/tests/Benchmark.vuls/spec.json new file +17 -0
@@ -0,0 +1,17 @@ +{ + "testName": "Benchmark.vuls", + "specs": { + "default": { + "gitrepo": "https://github.com/future-architect/vuls.git", + "distro": "debian", + "all_version": "7 8 9 10", + "isdeep": "false" + }, + "ubuntu": { + "gitrepo": "https://github.com/future-architect/vuls.git", + "distro": "ubuntu", + "all_version": "12 14 16", + "isdeep": "false" + } + } +}
engine/tests/Functional.vuls/chart_config.json deleted +0 -3
@@ -1,3 +0,0 @@ -{ - "chart_type": "testset_summary_table" -}
engine/tests/Functional.vuls/criteria.json deleted +0 -21
@@ -1,21 +0,0 @@ -{ - "schema_version":"1.0", - "criteria":[ - { - "tguid":"HIGH", - "max_fail": 0 - }, - { - "tguid":"MEDIUM", - "max_fail": 100 - }, - { - "tguid":"LOW", - "max_fail": 100 - }, - { - "tguid":"(unknown)", - "max_fail": 200 - } - ] -}
engine/tests/Functional.vuls/fuego_test.sh deleted +0 -93
93 lines not on this page · raw message
engine/tests/Functional.vuls/parser.py deleted +0 -17
@@ -1,17 +0,0 @@ -#!/bin/python - -import os, re, sys - -sys.path.insert(0, os.environ['FUEGO_CORE'] + '/engine/scripts/parser') -import common as plib - -results = {} -with open(plib.TEST_LOG) as f: - for line in f: - if line.startswith("CVE-"): - fields = line.split() - test_set = fields[2] - test_case = fields[0] - results[test_set+'.'+test_case] = "FAIL" - -sys.exit(plib.process(results))
engine/tests/Functional.vuls/spec.json deleted +0 -17
@@ -1,17 +0,0 @@ -{ - "testName": "Benchmark.vuls", - "specs": { - "default": { - "gitrepo": "https://github.com/future-architect/vuls.git", - "distro": "debian", - "all_version": "7 8 9 10", - "isdeep": "false" - }, - "ubuntu": { - "gitrepo": "https://github.com/future-architect/vuls.git", - "distro": "ubuntu", - "all_version": "12 14 16", - "isdeep": "false" - } - } -}
Signature · 1 line
2.7.4
PATCH 2/11 vuls: deep scans require root permissions
In reply to Daniel Sangorrin--- engine/tests/Benchmark.vuls/fuego_test.sh | 6 ++++++ 1 file changed, 6 insertions(+)
engine/tests/Benchmark.vuls/fuego_test.sh +6 -0
@@ -27,6 +27,12 @@ function test_pre_check { abort_job "Run on docker as root: ssh-keyscan -t ecdsa ${IPADDR} >> /etc/ssh/ssh_known_hosts" fi + if [ "${BENCHMARK_VULS_ISDEEP}" == "true" ]; then + if [ "${LOGIN}" != "root" ]; then + abort_job "You need to login as root for vuls deep scan" + fi + fi + # Make sure that go was installed in the system is_on_target go PROGRAM_GO $PATH assert_define PROGRAM_GO "run /fuego-ro/scripts/fuego-vuls-docker-preparation"
Signature · 1 line
2.7.4
PATCH 3/11 vuls: rename all_version to versions
In reply to Daniel Sangorrinversions referes to the distro version. E.g. for debian it could be 7, 8 etc.
--- engine/tests/Benchmark.vuls/fuego_test.sh | 4 ++-- engine/tests/Benchmark.vuls/spec.json | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-)
engine/tests/Benchmark.vuls/fuego_test.sh +2 -2
@@ -66,12 +66,12 @@ function test_build { for i in `seq 2002 $(date +"%Y")`; do go-cve-dictionary fetchnvd -http-proxy=$http_proxy -years $i done - goval-dictionary fetch-${BENCHMARK_VULS_DISTRO} ${BENCHMARK_VULS_ALL_VERSION} -http-proxy ${http_proxy} + goval-dictionary fetch-${BENCHMARK_VULS_DISTRO} ${BENCHMARK_VULS_VERSIONS} -http-proxy ${http_proxy} else for i in `seq 2002 $(date +"%Y")`; do go-cve-dictionary fetchnvd -years $i done - goval-dictionary fetch-${BENCHMARK_VULS_DISTRO} ${BENCHMARK_VULS_ALL_VERSION} + goval-dictionary fetch-${BENCHMARK_VULS_DISTRO} ${BENCHMARK_VULS_VERSIONS} fi # Create config.toml file
engine/tests/Benchmark.vuls/spec.json +2 -2
@@ -4,13 +4,13 @@ "default": { "gitrepo": "https://github.com/future-architect/vuls.git", "distro": "debian", - "all_version": "7 8 9 10", + "versions": "7 8 9 10", "isdeep": "false" }, "ubuntu": { "gitrepo": "https://github.com/future-architect/vuls.git", "distro": "ubuntu", - "all_version": "12 14 16", + "versions": "12 14 16", "isdeep": "false" } }
Signature · 1 line
2.7.4
PATCH 4/11 vuls: remove gitrepo definition
In reply to Daniel SangorrinThe test already clones the repository inside the build phase. Note that it also install other go packages during build (only once) so docker needs to have internet access.
--- engine/tests/Benchmark.vuls/spec.json | 2 -- 1 file changed, 2 deletions(-)
engine/tests/Benchmark.vuls/spec.json +0 -2
@@ -2,13 +2,11 @@ "testName": "Benchmark.vuls", "specs": { "default": { - "gitrepo": "https://github.com/future-architect/vuls.git", "distro": "debian", "versions": "7 8 9 10", "isdeep": "false" }, "ubuntu": { - "gitrepo": "https://github.com/future-architect/vuls.git", "distro": "ubuntu", "versions": "12 14 16", "isdeep": "false"
Signature · 1 line
2.7.4
PATCH 5/11 vuls: add specs for all distros supported by vuls
In reply to Daniel SangorrinThese specs are untested but they should work according to vuls help.
FIXTHIS: we need to add pre_checks for each distro, for the moment the user must check vuls documentation and make sure that the target machine has the necessary requirements.
--- engine/tests/Benchmark.vuls/spec.json | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+)
engine/tests/Benchmark.vuls/spec.json +20 -0
@@ -10,6 +10,26 @@ "distro": "ubuntu", "versions": "12 14 16", "isdeep": "false" + }, + "suse": { + "distro": "suse", + "versions": "13.2", + "isdeep": "false" + }, + "redhat": { + "distro": "redhat", + "versions": "5 6 7", + "isdeep": "false" + }, + "alpine": { + "distro": "alpine", + "versions": "3.3 3.4 3.5 3.6", + "isdeep": "false" + }, + "oracle": { + "distro": "oracle", + "versions": "5 6 7", + "isdeep": "false" } } }
Signature · 1 line
2.7.4
PATCH 6/11 vuls: fix path during test_build
In reply to Daniel SangorrinThis was working by chance.
--- engine/tests/Benchmark.vuls/fuego_test.sh | 3 +++ 1 file changed, 3 insertions(+)
engine/tests/Benchmark.vuls/fuego_test.sh +3 -0
@@ -61,6 +61,9 @@ function test_build { make install fi + # Share the database creation on a common directory + cd $GOPATH/src/github.com/future-architect/vuls + # Fetch vulnerability data from NVD and OVAL data if [ -n "${http_proxy}" ]; then for i in `seq 2002 $(date +"%Y")`; do
Signature · 1 line
2.7.4
PATCH 7/11 vuls: remove one-line report
In reply to Daniel SangorrinThis report was not being used/parsed and that line also appears in the normal report so it is not required.
--- engine/tests/Benchmark.vuls/fuego_test.sh | 1 - 1 file changed, 1 deletion(-)
engine/tests/Benchmark.vuls/fuego_test.sh +0 -1
@@ -97,6 +97,5 @@ function test_run { # Start Scanning vuls scan ${BENCHMARK_VULS_DISTRO} # Reporting - vuls report -format-one-line-text -cvedb-path=$PWD/cve.sqlite3 -ovaldb-path=$PWD/oval.sqlite3 vuls report -format-short-text | tee ${LOGDIR}/testlog.txt }
Signature · 1 line
2.7.4
PATCH 8/11 vuls: use full reports
In reply to Daniel Sangorrinshort reports do not show the packages affected so it is harder to know what to upgrade.
--- engine/tests/Benchmark.vuls/fuego_test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
engine/tests/Benchmark.vuls/fuego_test.sh +1 -1
@@ -97,5 +97,5 @@ function test_run { # Start Scanning vuls scan ${BENCHMARK_VULS_DISTRO} # Reporting - vuls report -format-short-text | tee ${LOGDIR}/testlog.txt + vuls report -format-full-text | tee ${LOGDIR}/testlog.txt }
Signature · 1 line
2.7.4
PATCH 9/11 vuls: fix deep argument and do not configtest
In reply to Daniel SangorrinThe deep argument was not correctly being passed, also configtest is not necessary because scan will report similar errors in case the ssh connection fails.
--- engine/tests/Benchmark.vuls/fuego_test.sh | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-)
engine/tests/Benchmark.vuls/fuego_test.sh +5 -7
@@ -88,14 +88,12 @@ function test_build { function test_run { cd $GOPATH/src/github.com/future-architect/vuls - # Check config.toml and settings on the server before scanning + if [ "${BENCHMARK_VULS_ISDEEP}" = "true" ]; then - vuls configtest -deep ${BENCHMARK_VULS_DISTRO} - else - vuls configtest ${BENCHMARK_VULS_DISTRO} + VULS_DEEP_FLAG="-deep" fi - # Start Scanning - vuls scan ${BENCHMARK_VULS_DISTRO} - # Reporting + + cd $GOPATH/src/github.com/future-architect/vuls + vuls scan ${VULS_DEEP_FLAG} ${BENCHMARK_VULS_DISTRO} vuls report -format-full-text | tee ${LOGDIR}/testlog.txt }
Signature · 1 line
2.7.4
PATCH 10/11 vuls: move config.toml to the logdir
In reply to Daniel Sangorrinconfig.toml was being generated on a folder that was shared by different boards and jobs. Instead generate the config.toml on the LOGDIR to have separate files for each board and to save the configuration used for possible inspection.
--- engine/tests/Benchmark.vuls/fuego_test.sh | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-)
engine/tests/Benchmark.vuls/fuego_test.sh +10 -11
@@ -76,24 +76,23 @@ function test_build { done goval-dictionary fetch-${BENCHMARK_VULS_DISTRO} ${BENCHMARK_VULS_VERSIONS} fi - - # Create config.toml file - echo "[servers]" > config.toml - echo "[servers.${BENCHMARK_VULS_DISTRO}]" >> config.toml - echo "host = \"${IPADDR}\"" >> config.toml - echo "port = \"${SSH_PORT}\"" >> config.toml - echo "user = \"${LOGIN}\"" >> config.toml - echo "keyPath = \"${SSH_KEY}\"" >> config.toml } function test_run { - cd $GOPATH/src/github.com/future-architect/vuls + # Create config.toml file on the log folder (per run, not shared) + echo "[servers]" > ${LOGDIR}/config.toml + echo "[servers.${BENCHMARK_VULS_DISTRO}]" >> ${LOGDIR}/config.toml + echo "host = \"${IPADDR}\"" >> ${LOGDIR}/config.toml + echo "port = \"${SSH_PORT}\"" >> ${LOGDIR}/config.toml + echo "user = \"${LOGIN}\"" >> ${LOGDIR}/config.toml + echo "keyPath = \"${SSH_KEY}\"" >> ${LOGDIR}/config.toml + VULS_OPTIONSFLAG="-config=${LOGDIR}/config.toml -results-dir=${LOGDIR}" if [ "${BENCHMARK_VULS_ISDEEP}" = "true" ]; then VULS_DEEP_FLAG="-deep" fi cd $GOPATH/src/github.com/future-architect/vuls - vuls scan ${VULS_DEEP_FLAG} ${BENCHMARK_VULS_DISTRO} - vuls report -format-full-text | tee ${LOGDIR}/testlog.txt + vuls scan ${VULS_DEEP_FLAG} ${VULS_OPTIONSFLAG} ${BENCHMARK_VULS_DISTRO} + vuls report ${VULS_OPTIONSFLAG} -format-full-text | tee ${LOGDIR}/testlog.txt }
Signature · 1 line
2.7.4
PATCH 11/11 vuls: rewrite all parsing related files
In reply to Daniel SangorrinBefore the parser was traversing the vuls report and creating a test case per CVE found. This was not very efficient and produced too many information in the run.json.
Instead we can do something more simple: parse only the summary line that classifies vulnerabilities in 4 groups (HIGH, MEDIUM, LOW and UNKNOWN) and apply a criteria on them. On Jenkins we will only show HIGH and MEDIUM as a plot.
--- engine/tests/Benchmark.vuls/chart_config.json | 3 +- engine/tests/Benchmark.vuls/criteria.json | 28 ++++++++++++----- engine/tests/Benchmark.vuls/parser.py | 25 ++++++++------- engine/tests/Benchmark.vuls/reference.json | 45 +++++++++++++++++++++++++++ 4 files changed, 81 insertions(+), 20 deletions(-) create mode 100644 engine/tests/Benchmark.vuls/reference.json
engine/tests/Benchmark.vuls/chart_config.json +2 -1
@@ -1,3 +1,4 @@ { - "chart_type": "testset_summary_table" + "chart_type": "measure_plot", + "measures": ["default.HIGH.Count", "default.MEDIUM.Count"] }
engine/tests/Benchmark.vuls/criteria.json +20 -8
@@ -2,20 +2,32 @@ "schema_version":"1.0", "criteria":[ { - "tguid":"HIGH", - "max_fail": 0 + "tguid":"default.HIGH.Count", + "reference":{ + "value":0, + "operator":"le" + } }, { - "tguid":"MEDIUM", - "max_fail": 100 + "tguid":"default.MEDIUM.Count", + "reference":{ + "value":100, + "operator":"le" + } }, { - "tguid":"LOW", - "max_fail": 100 + "tguid":"default.LOW.Count", + "reference":{ + "value":200, + "operator":"le" + } }, { - "tguid":"(unknown)", - "max_fail": 200 + "tguid":"default.UNKNOWN.Count", + "reference":{ + "value":100, + "operator":"le" + } } ] }
engine/tests/Benchmark.vuls/parser.py +14 -11
@@ -1,17 +1,20 @@ -#!/bin/python +#!/usr/bin/python +# See common.py for description of command-line arguments -import os, re, sys +import os, sys sys.path.insert(0, os.environ['FUEGO_CORE'] + '/engine/scripts/parser') import common as plib -results = {} -with open(plib.TEST_LOG) as f: - for line in f: - if line.startswith("CVE-"): - fields = line.split() - test_set = fields[2] - test_case = fields[0] - results[test_set+'.'+test_case] = "FAIL" +measurements = {} -sys.exit(plib.process(results)) +regex_string = '.*\(High:(\d+) Medium:(\d+) Low:(\d+) \?:(\d+)\)' +matches = plib.parse_log(regex_string) + +if matches: + measurements['default.HIGH'] = [{"name": "Count", "measure" : float(matches[0][0])}] + measurements['default.MEDIUM'] = [{"name": "Count", "measure" : float(matches[0][1])}] + measurements['default.LOW'] = [{"name": "Count", "measure" : float(matches[0][2])}] + measurements['default.UNKNOWN'] = [{"name": "Count", "measure" : float(matches[0][3])}] + +sys.exit(plib.process(measurements))
engine/tests/Benchmark.vuls/reference.json new file +45 -0
45 lines not on this page · raw message
Signature · 1 line
2.7.4