Skip to contentSkip to conversation

iotests: delinting work-in-progress

RFC PATCH New John Snow 2 participants 8 messages May 13, 2020 – May 14, 2020

lore
Share & tools
lore
author
#2 lore raw

RFC PATCH 2/6 Makefile: add delint WIP

In reply to John Snow
---
 tests/qemu-iotests/Makefile | 6 ++++++
 1 file changed, 6 insertions(+)
tests/qemu-iotests/Makefile +6 -0
index 27380e60c1..7dbb7f0fff 100644
@@ -1,3 +1,6 @@
+PYMODULES = $(wildcard *.py)
+
+KNOWN_GOOD = iotests.py
 
 CLEANFILES= *.out.bad *.notrun check.log check.time*
 
@@ -7,3 +10,6 @@ default:
 clean:
 	rm -f $(CLEANFILES)
 
+delint:
+	pylint $(KNOWN_GOOD)
+	pylint --disable=R,C,W $(filter-out $(KNOWN_GOOD), $(PYMODULES))
Signature · 1 line
2.21.1
author
#3 lore raw

RFC PATCH 3/6 nbd-fault-injector: delint

In reply to John Snow
---
 tests/qemu-iotests/Makefile              |  2 +-
 tests/qemu-iotests/nbd-fault-injector.py | 34 ++++++++++++++----------
 2 files changed, 21 insertions(+), 15 deletions(-)
tests/qemu-iotests/Makefile +1 -1
index 7dbb7f0fff..64db48342f 100644
@@ -1,6 +1,6 @@
 PYMODULES = $(wildcard *.py)
 
-KNOWN_GOOD = iotests.py
+KNOWN_GOOD = iotests.py nbd-fault-injector.py
 
 CLEANFILES= *.out.bad *.notrun check.log check.time*
 
tests/qemu-iotests/nbd-fault-injector.py +20 -14
index 588d62aebf..0de6bc643e 100755
@@ -47,10 +47,7 @@
 import socket
 import struct
 import collections
-if sys.version_info.major >= 3:
-    import configparser
-else:
-    import ConfigParser as configparser
+import configparser
 
 FAKE_DISK_SIZE = 8 * 1024 * 1024 * 1024 # 8 GB
 
@@ -71,7 +68,9 @@
 export_tuple = collections.namedtuple('Export', 'reserved magic opt len')
 export_struct = struct.Struct('>IQII')
 neg2_struct = struct.Struct('>QH124x')
-request_tuple = collections.namedtuple('Request', 'magic type handle from_ len')
+request_tuple = collections.namedtuple(
+    'Request', 'magic type handle from_ len'
+)
 request_struct = struct.Struct('>IIQQI')
 reply_struct = struct.Struct('>IIQ')
 
@@ -84,13 +83,13 @@ def recvall(sock, bufsize):
     chunks = []
     while received < bufsize:
         chunk = sock.recv(bufsize - received)
-        if len(chunk) == 0:
+        if not chunk:
             raise Exception('unexpected disconnect')
         chunks.append(chunk)
         received += len(chunk)
     return b''.join(chunks)
 
-class Rule(object):
+class Rule:
     def __init__(self, name, event, io, when):
         self.name = name
         self.event = event
@@ -104,7 +103,7 @@ def match(self, event, io):
             return False
         return True
 
-class FaultInjectionSocket(object):
+class FaultInjectionSocket:
     def __init__(self, sock, rules):
         self.sock = sock
         self.rules = rules
@@ -150,7 +149,7 @@ def negotiate_export(conn):
     export = export_tuple._make(export_struct.unpack(buf))
     assert export.magic == NBD_OPTS_MAGIC
     assert export.opt == NBD_OPT_EXPORT_NAME
-    name = conn.recv(export.len, event='export-name')
+    _name = conn.recv(export.len, event='export-name')
 
     # Send negotiation part 2
     buf = neg2_struct.pack(FAKE_DISK_SIZE, 0)
@@ -200,7 +199,8 @@ def parse_inject_error(name, options):
     if 'event' not in options:
         err('missing \"event\" option in %s' % name)
     event = options['event']
-    if event not in ('neg-classic', 'neg1', 'export', 'neg2', 'request', 'reply', 'data'):
+    if event not in ('neg-classic', 'neg1', 'export',
+                     'neg2', 'request', 'reply', 'data'):
         err('invalid \"event\" option value \"%s\" in %s' % (event, name))
     io = options.get('io', 'readwrite')
     if io not in ('read', 'write', 'readwrite'):
@@ -229,8 +229,8 @@ def parse_config(config):
 
 def load_rules(filename):
     config = configparser.RawConfigParser()
-    with open(filename, 'rt') as f:
-        config.readfp(f, filename)
+    with open(filename, 'rt') as infile:
+        config.read_file(infile)
     return parse_config(config)
 
 def open_socket(path):
@@ -252,8 +252,14 @@ def open_socket(path):
     return sock
 
 def usage(args):
-    sys.stderr.write('usage: %s [--classic-negotiation] <tcp-port>|<unix-path> <config-file>\n' % args[0])
-    sys.stderr.write('Run an fault injector NBD server with rules defined in a config file.\n')
+    name = args[0]
+    sys.stderr.write(
+        f'usage: {name} [--classic-negotiation] '
+        '<tcp-port>|<unix-path> <config-file>\n'
+    )
+    sys.stderr.write(
+        'Run a fault injector NBD server with '
+        'rules defined in a config file.\n')
     sys.exit(1)
 
 def main(args):
Signature · 1 line
2.21.1
author
#1 lore raw

RFC PATCH 1/6 iotests: type hint wip

In reply to John Snow
---
 tests/qemu-iotests/iotests.py | 28 ++++++++++++++--------------
 1 file changed, 14 insertions(+), 14 deletions(-)
tests/qemu-iotests/iotests.py +14 -14
index 6c0e781af7..27c477c8a7 100644
@@ -90,7 +90,7 @@
 luks_default_key_secret_opt = 'key-secret=keysec0'
 
 
-def qemu_img(*args):
+def qemu_img(*args) -> int:
     '''Run qemu-img and return the exit code'''
     devnull = open('/dev/null', 'r+')
     exitcode = subprocess.call(qemu_img_args + list(args),
@@ -113,24 +113,24 @@ def ordered_qmp(qmsg, conv_keys=True):
         return od
     return qmsg
 
-def qemu_img_create(*args):
-    args = list(args)
+def qemu_img_create(*args: str) -> int:
+    qargs = list(args)
 
     # default luks support
-    if '-f' in args and args[args.index('-f') + 1] == 'luks':
-        if '-o' in args:
-            i = args.index('-o')
-            if 'key-secret' not in args[i + 1]:
-                args[i + 1].append(luks_default_key_secret_opt)
-                args.insert(i + 2, '--object')
-                args.insert(i + 3, luks_default_secret_object)
+    if '-f' in qargs and qargs[qargs.index('-f') + 1] == 'luks':
+        if '-o' in qargs:
+            i = qargs.index('-o')
+            if 'key-secret' not in qargs[i + 1]:
+                qargs[i + 1].append(luks_default_key_secret_opt)
+                qargs.insert(i + 2, '--object')
+                qargs.insert(i + 3, luks_default_secret_object)
         else:
-            args = ['-o', luks_default_key_secret_opt,
-                    '--object', luks_default_secret_object] + args
+            qargs = ['-o', luks_default_key_secret_opt,
+                     '--object', luks_default_secret_object] + qargs
 
-    args.insert(0, 'create')
+    qargs.insert(0, 'create')
 
-    return qemu_img(*args)
+    return qemu_img(*qargs)
 
 def qemu_img_verbose(*args):
     '''Run qemu-img without suppressing its output and return the exit code'''
Signature · 1 line
2.21.1
author
#5 lore raw

RFC PATCH 5/6 qcow2.py: delint

In reply to John Snow
---
 tests/qemu-iotests/Makefile |   5 +-
 tests/qemu-iotests/pylintrc |   1 +
 tests/qemu-iotests/qcow2.py | 156 +++++++++++++++++++++++-------------
 3 files changed, 104 insertions(+), 58 deletions(-)
tests/qemu-iotests/Makefile +1 -4
index 5a3a1e8092..77da9fd96d 100644
@@ -1,7 +1,5 @@
 PYMODULES = $(wildcard *.py)
 
-KNOWN_GOOD = iotests.py nbd-fault-injector.py qed.py
-
 CLEANFILES= *.out.bad *.notrun check.log check.time*
 
 # no default target
@@ -11,5 +9,4 @@ clean:
 	rm -f $(CLEANFILES)
 
 delint:
-	pylint $(KNOWN_GOOD)
-	pylint --disable=R,C,W $(filter-out $(KNOWN_GOOD), $(PYMODULES))
+	pylint $(PYMODULES)
tests/qemu-iotests/pylintrc +1 -0
index 5481afe528..a334e242b6 100644
@@ -17,6 +17,7 @@ disable=invalid-name,
         too-many-lines,
         too-many-locals,
         too-many-public-methods,
+        too-many-instance-attributes,
         # These are temporary, and should be removed:
         missing-docstring,
 
tests/qemu-iotests/qcow2.py +102 -54

266 lines not on this page · raw message

Signature · 1 line
2.21.1
author
#4 lore raw

RFC PATCH 4/6 qed.py: delint

In reply to John Snow
---
 tests/qemu-iotests/Makefile |  2 +-
 tests/qemu-iotests/qed.py   | 46 ++++++++++++++++++++++++-------------
 2 files changed, 31 insertions(+), 17 deletions(-)
tests/qemu-iotests/Makefile +1 -1
index 64db48342f..5a3a1e8092 100644
@@ -1,6 +1,6 @@
 PYMODULES = $(wildcard *.py)
 
-KNOWN_GOOD = iotests.py nbd-fault-injector.py
+KNOWN_GOOD = iotests.py nbd-fault-injector.py qed.py
 
 CLEANFILES= *.out.bad *.notrun check.log check.time*
 
tests/qemu-iotests/qed.py +30 -16

114 lines not on this page · raw message

Signature · 1 line
2.21.1
author
#6 lore raw

RFC PATCH 6/6 WIP: delint test files

In reply to John Snow
---
 tests/qemu-iotests/Makefile | 3 +++
 1 file changed, 3 insertions(+)
tests/qemu-iotests/Makefile +3 -0
index 77da9fd96d..fd85eb5bae 100644
@@ -1,4 +1,6 @@
 PYMODULES = $(wildcard *.py)
+PYTESTS := $(shell grep -rl '^\#!/usr/bin/env python3' *)
+PYTESTS := $(filter-out $(PYMODULES), $(PYTESTS))
 
 CLEANFILES= *.out.bad *.notrun check.log check.time*
 
@@ -10,3 +12,4 @@ clean:
 
 delint:
 	pylint $(PYMODULES)
+	pylint --disable=R,C,W $(PYTESTS)
Signature · 1 line
2.21.1

John Snow

author
#0 lore raw

cover iotests: delinting work-in-progress

I ran out of time, but I was briefly entertaining the idea of hitting everything else in the iotests folder with the pylint and mypy beam.

This is just a draft of what I had at the time, in case someone gets around to it before I do.

I forced all of the python scripts in this directory to Python3 and pass the formatter check imposed by the pylintrc that we make iotests.py adhere to, but there's no guarantee these scripts actually work. I was a little rough around the edges.

qed.py and qcow2.py in particular abuse some python features that pylint really doesn't like to see at all, and forcing these to pass a pylint check was clearly fighting against the idioms in-use in those files, with more than a few unsatisfactory hacks employed just to make pylint shut up. They are not necessarily improvements to those scripts.

I created a Makefile check to hit the iotests themselves with the pylint checker so we can identify when refactors to the testing infrastructure invalidate tests we don't get to run that often; but didn't get around to improving the quality of the tests themselves.

(Whether or not this is even worth our time is debatable, it depends on how much effort it would take to bring them up to par. Maybe it's a lot, I don't know.)

--js

John Snow (6):
iotests: type hint wip
Makefile: add delint WIP
nbd-fault-injector: delint
qed.py: delint
qcow2.py: delint
WIP: delint test files

 tests/qemu-iotests/Makefile              |   6 +
 tests/qemu-iotests/iotests.py            |  28 ++--
 tests/qemu-iotests/nbd-fault-injector.py |  34 +++--
 tests/qemu-iotests/pylintrc              |   1 +
 tests/qemu-iotests/qcow2.py              | 156 +++++++++++++++--------
 tests/qemu-iotests/qed.py                |  46 ++++---
 6 files changed, 173 insertions(+), 98 deletions(-)
Signature · 1 line
2.21.1
#7 lore raw
Patchew URL: https://patchew.org/QEMU/[email protected]/

Patchew URL: https://patchew.org/QEMU/[email protected]/

Hi,

This series failed the docker-quick@centos7 build test. Please find the testing commands and their output below. If you have Docker installed, you can probably reproduce it locally.

=== TEST SCRIPT BEGIN ===
#!/bin/bash
make docker-image-centos7 V=1 NETWORK=1
time make docker-test-quick@centos7 SHOW_ENV=1 J=14 NETWORK=1 === TEST SCRIPT END ===

Not run: 259
Failures: 031 036 054 061 114
Failed 5 of 118 iotests
make: *** [check-tests/check-block.sh] Error 1
make: *** Waiting for unfinished jobs....
TEST check-qtest-aarch64: tests/qtest/qos-test
Traceback (most recent call last):
---
raise CalledProcessError(retcode, cmd)
subprocess.CalledProcessError: Command '['sudo', '-n', 'docker', 'run', '--label', 'com.qemu.instance.uuid=a364a3916b304854887e4b7475175383', '-u', '1003', '--security-opt', 'seccomp=unconfined', '--rm', '-e', 'TARGET_LIST=', '-e', 'EXTRA_CONFIGURE_OPTS=', '-e', 'V=', '-e', 'J=14', '-e', 'DEBUG=', '-e', 'SHOW_ENV=1', '-e', 'CCACHE_DIR=/var/tmp/ccache', '-v', '/home/patchew2/.cache/qemu-docker-ccache:/var/tmp/ccache:z', '-v', '/var/tmp/patchew-tester-tmp-n73ihxu6/src/docker-src.2020-05-13-23.00.06.31673:/var/tmp/qemu:z,ro', 'qemu:centos7', '/var/tmp/qemu/run', 'test-quick']' returned non-zero exit status 2. filter=--filter=label=com.qemu.instance.uuid=a364a3916b304854887e4b7475175383
make[1]: *** [docker-run] Error 1
make[1]: Leaving directory `/var/tmp/patchew-tester-tmp-n73ihxu6/src'
make: *** [docker-run-test-quick@centos7] Error 2

real 15m47.213s
user 0m5.361s

The full log is available at
http://patchew.org/logs/[email protected]/testing.docker-quick@centos7/?type=message.
---
Email generated automatically by Patchew [https://patchew.org/].
Please send your feedback to [email protected]