python/qemu: refactor as installable package
RFC PATCH 1/32 python/qemu: create qemu.lib module
In reply to John Snowmove python/qemu/*.py to python/qemu/lib/*.py.
To create a namespace package, the 'qemu' directory itself shouldn't have module files in it. Thus, these files will go under a 'lib' package directory instead.
Bolster the lib/__init__.py file a little bit, Make the top-level classes and functions available directly inside the qemu.lib namespace, to facilitate a convenient shorthand:
Quoted 1 line
from qemu.lib import QEMUQtestMachine, QEMUMonitorProtocol
Lastly, update all of the existing import directives.
(Note: these scripts were not necessarily tested to see if they still work. Some of these scripts are in obvious states of disrepair and it is beyond the scope of this patch to attempt to fix them.)
---
python/qemu/__init__.py | 11 -----
python/qemu/lib/__init__.py | 57 +++++++++++++++++++++++
python/qemu/{ => lib}/accel.py | 0
python/qemu/{ => lib}/machine.py | 0
python/qemu/{ => lib}/qmp.py | 0
python/qemu/{ => lib}/qtest.py | 0
scripts/device-crash-test | 2 +-
scripts/qmp/qemu-ga-client | 2 +-
scripts/qmp/qmp-shell | 2 +-
scripts/render_block_graph.py | 3 +-
scripts/simplebench/bench_block_job.py | 4 +-
tests/acceptance/avocado_qemu/__init__.py | 2 +-
tests/acceptance/boot_linux.py | 3 +-
tests/acceptance/virtio_check_params.py | 2 +-
tests/acceptance/virtio_version.py | 2 +-
tests/migration/guestperf/engine.py | 2 +-
tests/qemu-iotests/235 | 2 +-
tests/qemu-iotests/iotests.py | 2 +-
tests/vm/basevm.py | 6 +--
19 files changed, 74 insertions(+), 28 deletions(-)
delete mode 100644 python/qemu/__init__.py
create mode 100644 python/qemu/lib/__init__.py
rename python/qemu/{ => lib}/accel.py (100%)
rename python/qemu/{ => lib}/machine.py (100%)
rename python/qemu/{ => lib}/qmp.py (100%)
rename python/qemu/{ => lib}/qtest.py (100%)python/qemu/__init__.py deleted +0 -11
@@ -1,11 +0,0 @@ -# QEMU library -# -# Copyright (C) 2015-2016 Red Hat Inc. -# Copyright (C) 2012 IBM Corp. -# -# Authors: -# Fam Zheng <[email protected]> -# -# This work is licensed under the terms of the GNU GPL, version 2. See -# the COPYING file in the top-level directory. -#
python/qemu/lib/__init__.py new file +57 -0
57 lines not on this page · raw message
python/qemu/lib/accel.py renamed from python/qemu/accel.py +0 -0
python/qemu/lib/machine.py renamed from python/qemu/machine.py +0 -0
python/qemu/lib/qmp.py renamed from python/qemu/qmp.py +0 -0
python/qemu/lib/qtest.py renamed from python/qemu/qtest.py +0 -0
scripts/device-crash-test +1 -1
@@ -35,7 +35,7 @@ import argparse from itertools import chain sys.path.append(os.path.join(os.path.dirname(__file__), '..', 'python')) -from qemu.machine import QEMUMachine +from qemu.lib import QEMUMachine logger = logging.getLogger('device-crash-test') dbg = logger.debug
scripts/qmp/qemu-ga-client +1 -1
@@ -42,7 +42,7 @@ import base64 import random sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'python')) -from qemu import qmp +from qemu.lib import qmp class QemuGuestAgent(qmp.QEMUMonitorProtocol):
scripts/qmp/qmp-shell +1 -1
@@ -75,7 +75,7 @@ import atexit import re sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'python')) -from qemu import qmp +from qemu.lib import qmp if sys.version_info[0] == 2: input = raw_input
scripts/render_block_graph.py +2 -1
@@ -25,7 +25,8 @@ from graphviz import Digraph sys.path.append(os.path.join(os.path.dirname(__file__), '..', 'python')) -from qemu.machine import MonitorResponseError +from qemu.lib import QEMUMonitorProtocol +from qemu.lib.machine import MonitorResponseError def perm(arr):
scripts/simplebench/bench_block_job.py +2 -2
@@ -25,8 +25,8 @@ import json sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'python')) -from qemu.machine import QEMUMachine -from qemu.qmp import QMPConnectError +from qemu.lib import QEMUMachine +from qemu.lib.qmp import QMPConnectError def bench_block_job(cmd, cmd_args, qemu_args):
tests/acceptance/avocado_qemu/__init__.py +1 -1
@@ -32,7 +32,7 @@ sys.path.append(os.path.join(SOURCE_DIR, 'python')) -from qemu.machine import QEMUMachine +from qemu.lib import QEMUMachine def is_readable_executable_file(path): return os.path.isfile(path) and os.access(path, os.R_OK | os.X_OK)
tests/acceptance/boot_linux.py +1 -2
@@ -12,8 +12,7 @@ from avocado_qemu import Test, BUILD_DIR -from qemu.accel import kvm_available -from qemu.accel import tcg_available +from qemu.lib import kvm_available, tcg_available from avocado.utils import cloudinit from avocado.utils import network
tests/acceptance/virtio_check_params.py +1 -1
@@ -23,7 +23,7 @@ import logging sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'python')) -from qemu.machine import QEMUMachine +from qemu.lib import QEMUMachine from avocado_qemu import Test from avocado import skip
tests/acceptance/virtio_version.py +1 -1
@@ -12,7 +12,7 @@ import os sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'python')) -from qemu.machine import QEMUMachine +from qemu.lib import QEMUMachine from avocado_qemu import Test # Virtio Device IDs:
tests/migration/guestperf/engine.py +1 -1
@@ -29,7 +29,7 @@ sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', '..', 'python')) -from qemu.machine import QEMUMachine +from qemu.lib import QEMUMachine class Engine(object):
tests/qemu-iotests/235 +1 -1
@@ -25,7 +25,7 @@ from iotests import qemu_img_create, qemu_io, file_path, log sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'python')) -from qemu.machine import QEMUMachine +from qemu.lib import QEMUMachine iotests.script_initialize(supported_fmts=['qcow2'])
tests/qemu-iotests/iotests.py +1 -1
@@ -34,7 +34,7 @@ # pylint: disable=import-error, wrong-import-position sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'python')) -from qemu import qtest +from qemu.lib import qtest assert sys.version_info >= (3, 6)
tests/vm/basevm.py +3 -3
@@ -18,9 +18,6 @@ import logging import time import datetime -sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'python')) -from qemu.accel import kvm_available -from qemu.machine import QEMUMachine import subprocess import hashlib import optparse
@@ -30,6 +27,9 @@ import multiprocessing import traceback +sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'python')) +from qemu.lib import kvm_available, QEMUMachine + SSH_KEY = open(os.path.join(os.path.dirname(__file__), "..", "keys", "id_rsa")).read() SSH_PUB_KEY = open(os.path.join(os.path.dirname(__file__),
Signature · 1 line
2.21.1
RFC PATCH 2/32 scripts/qmp: Fix shebang and imports
In reply to John SnowThere's more wrong with these scripts; They are in various stages of disrepair. That's beyond the scope of this current patchset.
This just mechanically corrects the imports and the shebangs, as part of ensuring that the python/qemu/lib refactoring didn't break anything needlessly.
--- scripts/qmp/qmp | 4 +++- scripts/qmp/qom-fuse | 4 +++- scripts/qmp/qom-get | 6 ++++-- scripts/qmp/qom-list | 4 +++- scripts/qmp/qom-set | 6 ++++-- scripts/qmp/qom-tree | 6 ++++-- 6 files changed, 21 insertions(+), 9 deletions(-)
scripts/qmp/qmp +3 -1
@@ -11,7 +11,9 @@ # See the COPYING file in the top-level directory. import sys, os -from qmp import QEMUMonitorProtocol + +sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'python')) +from qemu.lib import QEMUMonitorProtocol def print_response(rsp, prefix=[]): if type(rsp) == list:
scripts/qmp/qom-fuse +3 -1
@@ -15,7 +15,9 @@ import fuse, stat from fuse import Fuse import os, posix from errno import * -from qmp import QEMUMonitorProtocol + +sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'python')) +from qemu.lib import QEMUMonitorProtocol fuse.fuse_python_api = (0, 2)
scripts/qmp/qom-get +4 -2
@@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python3 ## # QEMU Object Model test tools #
@@ -13,7 +13,9 @@ import sys import os -from qmp import QEMUMonitorProtocol + +sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'python')) +from qemu.lib import QEMUMonitorProtocol cmd, args = sys.argv[0], sys.argv[1:] socket_path = None
scripts/qmp/qom-list +3 -1
@@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python3 ## # QEMU Object Model test tools #
@@ -13,6 +13,8 @@ import sys import os + +sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'python')) from qmp import QEMUMonitorProtocol cmd, args = sys.argv[0], sys.argv[1:]
scripts/qmp/qom-set +4 -2
@@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python3 ## # QEMU Object Model test tools #
@@ -13,7 +13,9 @@ import sys import os -from qmp import QEMUMonitorProtocol + +sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'python')) +from qemu.lib import QEMUMonitorProtocol cmd, args = sys.argv[0], sys.argv[1:] socket_path = None
scripts/qmp/qom-tree +4 -2
@@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python3 ## # QEMU Object Model test tools #
@@ -15,7 +15,9 @@ import sys import os -from qmp import QEMUMonitorProtocol + +sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'python')) +from qemu.lib import QEMUMonitorProtocol cmd, args = sys.argv[0], sys.argv[1:] socket_path = None
Signature · 1 line
2.21.1
RFC PATCH 4/32 python/qemu/lib: delint, add pylintrc
In reply to John SnowBring our these files up to speed with pylint 2.5.0. Add a pylintrc file to formalize which pylint subset we are targeting.
The similarity ignore is there to suppress similarity reports across imports, which for typing constants, are going to trigger this report erroneously.
--- python/qemu/lib/machine.py | 6 ++-- python/qemu/lib/pylintrc | 58 ++++++++++++++++++++++++++++++++++++++ python/qemu/lib/qtest.py | 42 +++++++++++++++++---------- 3 files changed, 88 insertions(+), 18 deletions(-) create mode 100644 python/qemu/lib/pylintrc
python/qemu/lib/machine.py +3 -3
@@ -58,7 +58,7 @@ def __init__(self, reply): self.reply = reply -class QEMUMachine(object): +class QEMUMachine: """ A QEMU VM
@@ -242,7 +242,7 @@ def _base_args(self): 'chardev=mon,mode=control']) if self._machine is not None: args.extend(['-machine', self._machine]) - for i in range(self._console_index): + for _ in range(self._console_index): args.extend(['-serial', 'null']) if self._console_set: self._console_address = os.path.join(self._sock_dir,
@@ -383,7 +383,7 @@ def shutdown(self, has_quit: bool = False) -> None: command = ' '.join(self._qemu_full_args) else: command = '' - LOG.warning(msg, -exitcode, command) + LOG.warning(msg, -int(exitcode), command) self._launched = False
python/qemu/lib/pylintrc new file +58 -0
58 lines not on this page · raw message
python/qemu/lib/qtest.py +27 -15
66 lines not on this page · raw message
Signature · 1 line
2.21.1
RFC PATCH 6/32 python/qemu: formalize as package
In reply to John SnowNB: I am choosing Python 3.6 here. Although our minimum requirement is 3.5, this code is used only by iotests (so far) under which we have been using a minimum version of 3.6.
3.6 is being preferred here for variable type hint capability, which enables us to use mypy for this package.
--- python/README.rst | 6 ++++++ python/setup.py | 50 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+) create mode 100644 python/README.rst create mode 100755 python/setup.py
python/README.rst new file +6 -0
@@ -0,0 +1,6 @@ +QEMU Python Tooling +------------------- + +This package provides QEMU tooling used by the QEMU project to build, +configure, and test QEMU. It is not a fully-fledged SDK and it is subject +to change at any time.
python/setup.py new file +50 -0
50 lines not on this page · raw message
Signature · 1 line
2.21.1
cover python/qemu: refactor as installable package
Hey, I got lost on my way to the store and I accidentally got 32 patches that convert our python library into something that passes pylint, flake8, and mypy --strict.
...So, a few things:
1. This is just an RFC. The actual design of these libraries really needs adjusted to be more pythonic. In general this means less Optional[T] return types and raising more exceptions. This could be handled later, but we ought to address it before publishing, if we do.
2. We still need to think carefully about how we package this, if we even want to package it, what the license on the top-level package should be, etc.
3. We should consider how to version it. For now, I'm using a lockstep versioning.
4. You can install this package using pip3 or python3 setup.py to a virtual environment or to your real one. From there, any python code in the QEMU tree that imports these modules will work with no sys.path hacking or custom PYTHONPATH exports.
5. You don't have to install it, though. I left all of the usual hacks in place in the rest of the tree so that everything will just keep working exactly as-is for right now. It's just that you COULD install it.
6. Here's a cool trick if you don't know about it yet:
Quoted 2 lines
cd qemu/python/qemu
pip3 install --user -e .
This will install the package in "develop" mode, which installs a forwarder package. When you update your source tree, the installed package stays "up to date" with the most recent edits.
Alright, have fun, stay safe!
John Snow (32):
python/qemu: create qemu.lib module scripts/qmp: Fix shebang and imports python//machine.py: remove bare except python/qemu/lib: delint, add pylintrc python/qemu/lib: delint; add flake8 config python/qemu: formalize as package python/qemu: add README.rst
python/qemu: Add Pipfile
python/qemu: add pylint to Pipfile python/qemu: Add flake8 to Pipfile python/qemu/lib: remove Python2 style super() calls python/qemu/lib: fix socket.makefile() typing python/qemu/lib: Adjust traceback typing python//qmp.py: use True/False for non/blocking modes python//qmp.py: Define common types python//qmp.py: re-absorb MonitorResponseError python//qmp.py: Do not return None from cmd_obj python//qmp.py: add casts to JSON deserialization python//qmp.py: add QMPProtocolError python//qmp.py: assert sockfile is not None python//machine.py: remove logging configuration python//machine.py: Fix monitor address typing python//machine.py: reorder __init__ python//machine.py: Don't modify state in _base_args() python//machine.py: Handle None events in event_wait python//machine.py: use qmp.command python//machine.py: Add _qmp access shim python//machine.py: fix _popen access python//qtest.py: Check before accessing _qtest python/qemu/lib: make 'args' style arguments immutable python/qemu: add mypy to Pipfile python/qemu/lib: Add mypy type annotations
python/README.rst | 6 +
python/qemu/README.rst | 8 +
python/Pipfile | 14 +
python/Pipfile.lock | 187 +++++++++++++
python/qemu/__init__.py | 11 -
python/qemu/lib/.flake8 | 2 +
python/qemu/lib/__init__.py | 57 ++++
python/qemu/{ => lib}/accel.py | 17 +-
python/qemu/{ => lib}/machine.py | 320 +++++++++++++---------
python/qemu/lib/pylintrc | 58 ++++
python/qemu/{ => lib}/qmp.py | 140 +++++++---
python/qemu/lib/qtest.py | 160 +++++++++++
python/qemu/qtest.py | 119 --------
python/setup.py | 50 ++++
scripts/device-crash-test | 2 +-
scripts/qmp/qemu-ga-client | 2 +-
scripts/qmp/qmp | 4 +-
scripts/qmp/qmp-shell | 2 +-
scripts/qmp/qom-fuse | 4 +-
scripts/qmp/qom-get | 6 +-
scripts/qmp/qom-list | 4 +-
scripts/qmp/qom-set | 6 +-
scripts/qmp/qom-tree | 6 +-
scripts/render_block_graph.py | 5 +-
scripts/simplebench/bench_block_job.py | 4 +-
tests/acceptance/avocado_qemu/__init__.py | 2 +-
tests/acceptance/boot_linux.py | 3 +-
tests/acceptance/virtio_check_params.py | 2 +-
tests/acceptance/virtio_version.py | 2 +-
tests/migration/guestperf/engine.py | 2 +-
tests/qemu-iotests/235 | 2 +-
tests/qemu-iotests/iotests.py | 2 +-
tests/vm/basevm.py | 6 +-
33 files changed, 881 insertions(+), 334 deletions(-)
create mode 100644 python/README.rst
create mode 100644 python/qemu/README.rst
create mode 100644 python/Pipfile
create mode 100644 python/Pipfile.lock
delete mode 100644 python/qemu/__init__.py
create mode 100644 python/qemu/lib/.flake8
create mode 100644 python/qemu/lib/__init__.py
rename python/qemu/{ => lib}/accel.py (86%)
rename python/qemu/{ => lib}/machine.py (67%)
create mode 100644 python/qemu/lib/pylintrc
rename python/qemu/{ => lib}/qmp.py (70%)
create mode 100644 python/qemu/lib/qtest.py
delete mode 100644 python/qemu/qtest.py
create mode 100755 python/setup.pySignature · 1 line
2.21.1
RFC PATCH 3/32 python//machine.py: remove bare except
Catch only the timeout error; if there are other problems, allow the stack trace to be visible.
--- python/qemu/lib/machine.py | 33 +++++++++++++++++++++------------ 1 file changed, 21 insertions(+), 12 deletions(-)
python/qemu/lib/machine.py +21 -12
@@ -342,7 +342,26 @@ def wait(self): self._load_io_log() self._post_shutdown() - def shutdown(self, has_quit=False): + def _issue_shutdown(self, has_quit: bool = False) -> None: + """ + Shutdown the VM. + """ + if not self.is_running(): + return + + if self._qmp is not None: + if not has_quit: + self._qmp.cmd('quit') + self._qmp.close() + + try: + self._popen.wait(timeout=3) + except subprocess.TimeoutExpired: + self._popen.kill() + + self._popen.wait() + + def shutdown(self, has_quit: bool = False) -> None: """ Terminate the VM and clean up """
@@ -353,17 +372,7 @@ def shutdown(self, has_quit=False): self._console_socket.close() self._console_socket = None - if self.is_running(): - if self._qmp: - try: - if not has_quit: - self._qmp.cmd('quit') - self._qmp.close() - self._popen.wait(timeout=3) - except: - self._popen.kill() - self._popen.wait() - + self._issue_shutdown(has_quit) self._load_io_log() self._post_shutdown()
Signature · 1 line
2.21.1
RFC PATCH 7/32 python/qemu: add README.rst
In reply to John SnowAdd a short readme that explains the package hierarchy, which will be visible while browsing the source on e.g. gitlab/github.
--- python/qemu/README.rst | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 python/qemu/README.rst
python/qemu/README.rst new file +8 -0
@@ -0,0 +1,8 @@ +python/qemu/ +------------ + +This directory serves as the root of a `Python PEP 420 implicit +namespace package <<https://www.python.org/dev/peps/pep-0420/>`_. + +Each directory below is assumed to be an installable Python module that +will be available under the ``qemu.<module>`` namespace.
Signature · 1 line
2.21.1
RFC PATCH 8/32 python/qemu: Add Pipfile
In reply to John Snowpipenv is a tool used for managing virtual environments with precisely specified dependencies. It is separate from the dependencies listed in setup.py, which are not pinned to precise versions.
Note that pipenv is not required to install or use this module; this is just a convenience for in-tree developing.
Here, a "blank" pipfile is added with no dependencies, but specifies Python 3.6 for the virtual environment.
--- python/Pipfile | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 python/Pipfile
python/Pipfile new file +11 -0
@@ -0,0 +1,11 @@ +[[source]] +name = "pypi" +url = "https://pypi.org/simple" +verify_ssl = true + +[dev-packages] + +[packages] + +[requires] +python_version = "3.6"
Signature · 1 line
2.21.1
RFC PATCH 5/32 python/qemu/lib: delint; add flake8 config
In reply to John SnowMostly, ignore the "no bare except" rule, because flake8 is not contextual and cannot determine if we re-raise. Pylint can, though, so always prefer pylint for that.
--- python/qemu/lib/.flake8 | 2 ++ python/qemu/lib/accel.py | 9 ++++++--- python/qemu/lib/machine.py | 13 +++++++++---- python/qemu/lib/qmp.py | 4 ++-- 4 files changed, 19 insertions(+), 9 deletions(-) create mode 100644 python/qemu/lib/.flake8
python/qemu/lib/.flake8 new file +2 -0
@@ -0,0 +1,2 @@ +[flake8] +extend-ignore = E722 # Pylint handles this, but smarter.
python/qemu/lib/accel.py +6 -3
@@ -23,11 +23,12 @@ # Mapping host architecture to any additional architectures it can # support which often includes its 32 bit cousin. ADDITIONAL_ARCHES = { - "x86_64" : "i386", - "aarch64" : "armhf", - "ppc64le" : "ppc64", + "x86_64": "i386", + "aarch64": "armhf", + "ppc64le": "ppc64", } + def list_accel(qemu_bin): """ List accelerators enabled in the QEMU binary.
@@ -47,6 +48,7 @@ def list_accel(qemu_bin): # Skip the first line which is the header. return [acc.strip() for acc in out.splitlines()[1:]] + def kvm_available(target_arch=None, qemu_bin=None): """ Check if KVM is available using the following heuristic:
@@ -69,6 +71,7 @@ def kvm_available(target_arch=None, qemu_bin=None): return False return True + def tcg_available(qemu_bin): """ Check if TCG is available.
python/qemu/lib/machine.py +9 -4
@@ -29,6 +29,7 @@ LOG = logging.getLogger(__name__) + class QEMUMachineError(Exception): """ Exception called when an error in QEMUMachine happens.
@@ -62,7 +63,8 @@ class QEMUMachine: """ A QEMU VM - Use this object as a context manager to ensure the QEMU process terminates:: + Use this object as a context manager to ensure + the QEMU process terminates:: with VM(binary) as vm: ...
@@ -188,8 +190,10 @@ def send_fd_scm(self, fd=None, file_path=None): fd_param.append(str(fd)) devnull = open(os.path.devnull, 'rb') - proc = subprocess.Popen(fd_param, stdin=devnull, stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, close_fds=False) + proc = subprocess.Popen( + fd_param, stdin=devnull, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, close_fds=False + ) output = proc.communicate()[0] if output: LOG.debug(output)
@@ -491,7 +495,8 @@ def event_wait(self, name, timeout=60.0, match=None): def events_wait(self, events, timeout=60.0): """ - events_wait waits for and returns a named event from QMP with a timeout. + events_wait waits for and returns a named event + from QMP with a timeout. events: a sequence of (name, match_criteria) tuples. The match criteria are optional and may be None.
python/qemu/lib/qmp.py +2 -2
@@ -168,8 +168,8 @@ def accept(self, timeout=15.0): @param timeout: timeout in seconds (nonnegative float number, or None). The value passed will set the behavior of the - underneath QMP socket as described in [1]. Default value - is set to 15.0. + underneath QMP socket as described in [1]. + Default value is set to 15.0. @return QMP greeting dict @raise OSError on socket connection errors @raise QMPConnectError if the greeting is not received
Signature · 1 line
2.21.1
RFC PATCH 13/32 python/qemu/lib: Adjust traceback typing
In reply to John Snowmypy considers it incorrect to use bool to statically return false, because it will assume that it could conceivably return True, and gives different analysis in that case. Use a None return to achieve the same effect, but make mypy happy.
Note: Pylint considers function signatures as code that might trip the duplicate-code checker. I'd rather not disable this as it does not trigger often in practice, so I'm disabling it as a one-off and filed a change request; see https://github.com/PyCQA/pylint/issues/3619
--- python/qemu/lib/machine.py | 8 ++++++-- python/qemu/lib/qmp.py | 10 ++++++++-- 2 files changed, 14 insertions(+), 4 deletions(-)
python/qemu/lib/machine.py +6 -2
@@ -24,6 +24,8 @@ import shutil import socket import tempfile +from typing import Optional, Type +from types import TracebackType from . import qmp
@@ -127,9 +129,11 @@ def __init__(self, binary, args=None, wrapper=None, name=None, def __enter__(self): return self - def __exit__(self, exc_type, exc_val, exc_tb): + def __exit__(self, + exc_type: Optional[Type[BaseException]], + exc_val: Optional[BaseException], + exc_tb: Optional[TracebackType]) -> None: self.shutdown() - return False def add_monitor_null(self): """
python/qemu/lib/qmp.py +8 -2
@@ -14,7 +14,9 @@ from typing import ( Optional, TextIO, + Type, ) +from types import TracebackType class QMPError(Exception):
@@ -146,10 +148,14 @@ def __enter__(self): # Implement context manager enter function. return self - def __exit__(self, exc_type, exc_value, exc_traceback): + def __exit__(self, + # pylint: disable=duplicate-code + # see https://github.com/PyCQA/pylint/issues/3619 + exc_type: Optional[Type[BaseException]], + exc_val: Optional[BaseException], + exc_tb: Optional[TracebackType]) -> None: # Implement context manager exit function. self.close() - return False def connect(self, negotiate=True): """
Signature · 1 line
2.21.1
RFC PATCH 11/32 python/qemu/lib: remove Python2 style super() calls
In reply to John SnowUse the Python3 style instead.
--- python/qemu/lib/machine.py | 2 +- python/qemu/lib/qtest.py | 15 +++++++-------- 2 files changed, 8 insertions(+), 9 deletions(-)
python/qemu/lib/machine.py +1 -1
@@ -55,7 +55,7 @@ def __init__(self, reply): desc = reply["error"]["desc"] except KeyError: desc = reply - super(MonitorResponseError, self).__init__(desc) + super().__init__(desc) self.reply = reply
python/qemu/lib/qtest.py +7 -8
@@ -101,29 +101,28 @@ def __init__(self, binary, args=None, name=None, test_dir="/var/tmp", name = "qemu-%d" % os.getpid() if sock_dir is None: sock_dir = test_dir - super(QEMUQtestMachine, - self).__init__(binary, args, name=name, test_dir=test_dir, - socket_scm_helper=socket_scm_helper, - sock_dir=sock_dir) + super().__init__(binary, args, name=name, test_dir=test_dir, + socket_scm_helper=socket_scm_helper, + sock_dir=sock_dir) self._qtest = None self._qtest_path = os.path.join(sock_dir, name + "-qtest.sock") def _base_args(self): - args = super(QEMUQtestMachine, self)._base_args() + args = super()._base_args() args.extend(['-qtest', 'unix:path=' + self._qtest_path, '-accel', 'qtest']) return args def _pre_launch(self): - super(QEMUQtestMachine, self)._pre_launch() + super()._pre_launch() self._qtest = QEMUQtestProtocol(self._qtest_path, server=True) def _post_launch(self): - super(QEMUQtestMachine, self)._post_launch() + super()._post_launch() self._qtest.accept() def _post_shutdown(self): - super(QEMUQtestMachine, self)._post_shutdown() + super()._post_shutdown() self._remove_if_exists(self._qtest_path) def qtest(self, cmd):
Signature · 1 line
2.21.1
RFC PATCH 10/32 python/qemu: Add flake8 to Pipfile
In reply to John Snow--- python/Pipfile | 1 + python/Pipfile.lock | 31 ++++++++++++++++++++++++++++++- 2 files changed, 31 insertions(+), 1 deletion(-)
python/Pipfile +1 -0
@@ -5,6 +5,7 @@ verify_ssl = true [dev-packages] pylint = "==2.5.0" +flake8 = "*" [packages]
python/Pipfile.lock +30 -1
@@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "7815dedfd7481b645389153dd45e9adb82c72956d0efc74d8f087497624b75e4" + "sha256": "58115144ace0f646b5c62da260fb4867ac0a0e485de3f5b0a56c7854afa21f5b" }, "pipfile-spec": 6, "requires": {
@@ -24,6 +24,21 @@ ], "version": "==2.4.1" }, + "entrypoints": { + "hashes": [ + "sha256:589f874b313739ad35be6e0cd7efde2a4e9b6fea91edcc34e58ecbb8dbe56d19", + "sha256:c70dd71abe5a8c85e55e12c19bd91ccfeec11a6e99044204511f9ed547d48451" + ], + "version": "==0.3" + }, + "flake8": { + "hashes": [ + "sha256:45681a117ecc81e870cbf1262835ae4af5e7a8b08e40b944a8a6e6b895914cfb", + "sha256:49356e766643ad15072a789a20915d3c91dc89fd313ccd71802303fd67e4deca" + ], + "index": "pypi", + "version": "==3.7.9" + }, "isort": { "hashes": [ "sha256:54da7e92468955c4fceacd0c86bd0ec997b0e1ee80d97f67c35a78b719dccab1",
@@ -64,6 +79,20 @@ ], "version": "==0.6.1" }, + "pycodestyle": { + "hashes": [ + "sha256:95a2219d12372f05704562a14ec30bc76b05a5b297b21a5dfe3f6fac3491ae56", + "sha256:e40a936c9a450ad81df37f549d676d127b1b66000a6c500caa2b085bc0ca976c" + ], + "version": "==2.5.0" + }, + "pyflakes": { + "hashes": [ + "sha256:17dbeb2e3f4d772725c777fabc446d5634d1038f234e77343108ce445ea69ce0", + "sha256:d976835886f8c5b31d47970ed689944a0262b5f3afa00a5a7b4dc81e5449f8a2" + ], + "version": "==2.1.1" + }, "pylint": { "hashes": [ "sha256:588e114e3f9a1630428c35b7dd1c82c1c93e1b0e78ee312ae4724c5e1a1e0245",
Signature · 1 line
2.21.1
RFC PATCH 26/32 python//machine.py: use qmp.command
In reply to John Snowmachine.py and qmp.py both do the same thing here; refactor machine.py to use qmp.py's functionality more directly.
--- python/qemu/lib/machine.py | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-)
python/qemu/lib/machine.py +15 -11
@@ -25,6 +25,8 @@ import socket import tempfile from typing import ( + Any, + Dict, List, Optional, Type,
@@ -416,17 +418,23 @@ def set_qmp_monitor(self, enabled=True): self._qmp_set = False self._qmp = None - def qmp(self, cmd, conv_keys=True, **args): - """ - Invoke a QMP command and return the response dict - """ + @classmethod + def _qmp_args(cls, _conv_keys: bool = True, **args: Any) -> Dict[str, Any]: qmp_args = dict() for key, value in args.items(): - if conv_keys: + if _conv_keys: qmp_args[key.replace('_', '-')] = value else: qmp_args[key] = value + return qmp_args + def qmp(self, cmd: str, + conv_keys: bool = True, + **args: Any) -> QMPMessage: + """ + Invoke a QMP command and return the response dict + """ + qmp_args = self._qmp_args(conv_keys, **args) return self._qmp.cmd(cmd, args=qmp_args) def command(self, cmd, conv_keys=True, **args):
@@ -435,12 +443,8 @@ def command(self, cmd, conv_keys=True, **args): On success return the response dict. On failure raise an exception. """ - reply = self.qmp(cmd, conv_keys, **args) - if reply is None: - raise qmp.QMPError("Monitor is closed") - if "error" in reply: - raise qmp.QMPResponseError(reply) - return reply["return"] + qmp_args = self._qmp_args(conv_keys, **args) + return self._qmp.command(cmd, **qmp_args) def get_qmp_event(self, wait=False): """
Signature · 1 line
2.21.1
RFC PATCH 14/32 python//qmp.py: use True/False for non/blocking modes
In reply to John SnowThe type system doesn't want integers.
--- python/qemu/lib/qmp.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)
python/qemu/lib/qmp.py +2 -2
@@ -120,14 +120,14 @@ def __get_events(self, wait=False): """ # Check for new events regardless and pull them into the cache: - self.__sock.setblocking(0) + self.__sock.setblocking(False) try: self.__json_read() except OSError as err: if err.errno == errno.EAGAIN: # No data available pass - self.__sock.setblocking(1) + self.__sock.setblocking(True) # Wait for new events, if needed. # if wait is 0.0, this means "no wait" and is also implicitly false.
Signature · 1 line
2.21.1
RFC PATCH 15/32 python//qmp.py: Define common types
In reply to John SnowDefine some common types that we'll need to annotate a lot of other functions going forward.
--- python/qemu/lib/qmp.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+)
python/qemu/lib/qmp.py +18 -0
@@ -12,13 +12,31 @@ import socket import logging from typing import ( + Any, + Dict, Optional, TextIO, Type, + Tuple, + Union, ) from types import TracebackType +# QMPMessage is a QMP Message of any kind. +# e.g. {'yee': 'haw'} +# +# QMPReturnValue is the inner value of return values only. +# {'return': {}} is the QMPMessage, +# {} is the QMPReturnValue. +QMPMessage = Dict[str, Any] +QMPReturnValue = Dict[str, Any] + +InternetAddrT = Tuple[str, str] +UnixAddrT = str +SocketAddrT = Union[InternetAddrT, UnixAddrT] + + class QMPError(Exception): """ QMP base exception
Signature · 1 line
2.21.1
RFC PATCH 9/32 python/qemu: add pylint to Pipfile
In reply to John SnowA bug in pylint 2.5.1 and 2.5.2 causes false positives for relative imports. This version is pinned at 2.5.0 until a fix is available.
--- python/Pipfile | 1 + python/Pipfile.lock | 123 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 124 insertions(+) create mode 100644 python/Pipfile.lock
python/Pipfile +1 -0
@@ -4,6 +4,7 @@ url = "https://pypi.org/simple" verify_ssl = true [dev-packages] +pylint = "==2.5.0" [packages]
python/Pipfile.lock new file +123 -0
123 lines not on this page · raw message
Signature · 1 line
2.21.1
RFC PATCH 30/32 python/qemu/lib: make 'args' style arguments immutable
In reply to John SnowThese arguments don't need to be mutable and aren't really used as such. Clarify their types as immutable and adjust code to match where necessary.
In general, It's probably best not to accept a user-defined mutable object and store it as internal object state unless there's a strong justification for doing so. Instead, try to use generic types as input with empty tuples as the default, and coerce to list where necessary.
--- python/qemu/lib/machine.py | 30 +++++++++++++++++------------- python/qemu/lib/qtest.py | 16 ++++++++++++---- 2 files changed, 29 insertions(+), 17 deletions(-)
python/qemu/lib/machine.py +17 -13
@@ -18,6 +18,7 @@ # import errno +from itertools import chain import logging import os import subprocess
@@ -29,6 +30,8 @@ Dict, List, Optional, + Sequence, + Tuple, Type, ) from types import TracebackType
@@ -67,8 +70,12 @@ class QEMUMachine: # vm is guaranteed to be shut down here """ - def __init__(self, binary, args=None, wrapper=None, name=None, - test_dir="/var/tmp", + def __init__(self, + binary: str, + args: Sequence[str] = (), + wrapper: Sequence[str] = (), + name: Optional[str] = None, + test_dir: str = "/var/tmp", monitor_address: Optional[SocketAddrT] = None, socket_scm_helper=None, sock_dir=None): '''
@@ -86,14 +93,7 @@ def __init__(self, binary, args=None, wrapper=None, name=None, # Direct user configuration self._binary = binary - - if args is None: - args = [] - # Copy mutable input: we will be modifying our copy self._args = list(args) - - if wrapper is None: - wrapper = [] self._wrapper = wrapper self._name = name or "qemu-%d" % os.getpid()
@@ -118,7 +118,7 @@ def __init__(self, binary, args=None, wrapper=None, name=None, self._iolog = None self._qmp_set = True # Enable QMP monitor by default. self._qmp_connection: Optional[qmp.QEMUMonitorProtocol] = None - self._qemu_full_args = None + self._qemu_full_args: Tuple[str, ...] = () self._temp_dir = None self._launched = False self._machine = None
@@ -323,7 +323,7 @@ def launch(self): raise QEMUMachineError('VM already launched') self._iolog = None - self._qemu_full_args = None + self._qemu_full_args = () try: self._launch() self._launched = True
@@ -343,8 +343,12 @@ def _launch(self): """ devnull = open(os.path.devnull, 'rb') self._pre_launch() - self._qemu_full_args = (self._wrapper + [self._binary] + - self._base_args + self._args) + self._qemu_full_args = tuple( + chain(self._wrapper, + [self._binary], + self._base_args, + self._args) + ) LOG.debug('VM launch command: %r', ' '.join(self._qemu_full_args)) self._popen = subprocess.Popen(self._qemu_full_args, stdin=devnull,
python/qemu/lib/qtest.py +12 -4
@@ -22,6 +22,7 @@ from typing import ( List, Optional, + Sequence, TextIO, )
@@ -103,8 +104,13 @@ class QEMUQtestMachine(QEMUMachine): A QEMU VM, with a qtest socket available. """ - def __init__(self, binary, args=None, name=None, test_dir="/var/tmp", - socket_scm_helper=None, sock_dir=None): + def __init__(self, + binary: str, + args: Sequence[str] = (), + name: Optional[str] = None, + test_dir: str = "/var/tmp", + socket_scm_helper: Optional[str] = None, + sock_dir: Optional[str] = None): if name is None: name = "qemu-%d" % os.getpid() if sock_dir is None:
@@ -118,8 +124,10 @@ def __init__(self, binary, args=None, name=None, test_dir="/var/tmp", @property def _base_args(self) -> List[str]: args = super()._base_args - args.extend(['-qtest', 'unix:path=' + self._qtest_path, - '-accel', 'qtest']) + args.extend([ + '-qtest', f"unix:path={self._qtest_path}", + '-accel', 'qtest' + ]) return args def _pre_launch(self):
Signature · 1 line
2.21.1
RFC PATCH 20/32 python//qmp.py: assert sockfile is not None
In reply to John SnowIn truth, if you don't do this, you'll just get a TypeError exception. Now, you'll get an AssertionError.
Is this tangibly better? No.
Does mypy complain less? Yes.
--- python/qemu/lib/qmp.py | 1 + 1 file changed, 1 insertion(+)
python/qemu/lib/qmp.py +1 -0
@@ -132,6 +132,7 @@ def __negotiate_capabilities(self): raise QMPCapabilitiesError def __json_read(self, only_event=False): + assert self.__sockfile is not None while True: data = self.__sockfile.readline() if not data:
Signature · 1 line
2.21.1
RFC PATCH 18/32 python//qmp.py: add casts to JSON deserialization
In reply to John Snowmypy and python type hints are not powerful enough to properly describe JSON messages in Python 3.6. The best we can do, generally, is describe them as Dict[str, Any].
Add casts to coerce this type for static analysis; but do NOT enforce this type at runtime in any way.
Note: Python 3.8 adds a TypedDict construct which allows for the description of more arbitrary Dictionary shapes. There is a third-party module, "Pydantic", which is compatible with 3.6 that can be used instead of the JSON library that parses JSON messages to fully-typed Python objects, and may be preferable in some cases.
(That is well beyond the scope of this commit or series.)
--- python/qemu/lib/qmp.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-)
python/qemu/lib/qmp.py +6 -2
@@ -13,6 +13,7 @@ import logging from typing import ( Any, + cast, Dict, Optional, TextIO,
@@ -129,7 +130,10 @@ def __json_read(self, only_event=False): data = self.__sockfile.readline() if not data: return None - resp = json.loads(data) + # By definition, any JSON received from QMP is a QMPMessage, + # and we are asserting only at static analysis time that it + # has a particular shape. + resp = cast(QMPMessage, json.loads(data)) if 'event' in resp: self.logger.debug("<<< %s", resp) self.__events.append(resp)
@@ -261,7 +265,7 @@ def command(self, cmd, **kwds): ret = self.cmd(cmd, kwds) if 'error' in ret: raise QMPResponseError(ret) - return ret['return'] + return cast(QMPReturnValue, ret['return']) def pull_event(self, wait=False): """
Signature · 1 line
2.21.1
RFC PATCH 16/32 python//qmp.py: re-absorb MonitorResponseError
In reply to John SnowWhen I initially split this out, I considered this more of a machine error than a QMP protocol error, but I think that's misguided.
Move this back to qmp.py and name it QMPResponseError. Convert qmp.command() to use this exception type.
--- python/qemu/lib/machine.py | 15 +-------------- python/qemu/lib/qmp.py | 17 +++++++++++++++-- scripts/render_block_graph.py | 4 ++-- 3 files changed, 18 insertions(+), 18 deletions(-)
python/qemu/lib/machine.py +1 -14
@@ -48,19 +48,6 @@ class QEMUMachineAddDeviceError(QEMUMachineError): """ -class MonitorResponseError(qmp.QMPError): - """ - Represents erroneous QMP monitor reply - """ - def __init__(self, reply): - try: - desc = reply["error"]["desc"] - except KeyError: - desc = reply - super().__init__(desc) - self.reply = reply - - class QEMUMachine: """ A QEMU VM
@@ -433,7 +420,7 @@ def command(self, cmd, conv_keys=True, **args): if reply is None: raise qmp.QMPError("Monitor is closed") if "error" in reply: - raise MonitorResponseError(reply) + raise qmp.QMPResponseError(reply) return reply["return"] def get_qmp_event(self, wait=False):
python/qemu/lib/qmp.py +15 -2
@@ -61,6 +61,19 @@ class QMPTimeoutError(QMPError): """ +class QMPResponseError(QMPError): + """ + Represents erroneous QMP monitor reply + """ + def __init__(self, reply: QMPMessage): + try: + desc = reply['error']['desc'] + except KeyError: + desc = reply + super().__init__(desc) + self.reply = reply + + class QEMUMonitorProtocol: """ Provide an API to connect to QEMU via QEMU Monitor Protocol (QMP) and then
@@ -250,8 +263,8 @@ def command(self, cmd, **kwds): Build and send a QMP command to the monitor, report errors if any """ ret = self.cmd(cmd, kwds) - if "error" in ret: - raise Exception(ret['error']['desc']) + if 'error' in ret: + raise QMPResponseError(ret) return ret['return'] def pull_event(self, wait=False):
scripts/render_block_graph.py +2 -2
@@ -26,7 +26,7 @@ sys.path.append(os.path.join(os.path.dirname(__file__), '..', 'python')) from qemu.lib import QEMUMonitorProtocol -from qemu.lib.machine import MonitorResponseError +from qemu.lib.qmp import QMPResponseError def perm(arr):
@@ -103,7 +103,7 @@ def command(self, cmd): reply = json.loads(subprocess.check_output(ar)) if 'error' in reply: - raise MonitorResponseError(reply) + raise QEMUResponseError(reply) return reply['return']
Signature · 1 line
2.21.1
RFC PATCH 21/32 python//machine.py: remove logging configuration
In reply to John SnowPython 3.5 and above do not print a warning when logging is not configured. As a library, it's best practice to leave logging configuration to the client executable.
--- python/qemu/lib/machine.py | 3 --- 1 file changed, 3 deletions(-)
python/qemu/lib/machine.py +0 -3
@@ -110,9 +110,6 @@ def __init__(self, binary, args=None, wrapper=None, name=None, self._console_socket = None self._remove_files = [] - # just in case logging wasn't configured by the main script: - logging.basicConfig() - def __enter__(self): return self
Signature · 1 line
2.21.1
RFC PATCH 22/32 python//machine.py: Fix monitor address typing
In reply to John SnowPrior to this, it's difficult for mypy to intuit what the concrete type of the monitor address is; it has difficulty inferring the type across two variables.
Create _monitor_address as a property that always returns a valid address to simply static type analysis.
To preserve our ability to clean up, use a simple boolean to indicate whether or not we should try to clean up the sock file after execution.
--- python/qemu/lib/machine.py | 45 ++++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 16 deletions(-)
python/qemu/lib/machine.py +29 -16
@@ -24,10 +24,14 @@ import shutil import socket import tempfile -from typing import Optional, Type +from typing import ( + Optional, + Type, +) from types import TracebackType from . import qmp +from .qmp import SocketAddrT LOG = logging.getLogger(__name__)
@@ -61,7 +65,8 @@ class QEMUMachine: """ def __init__(self, binary, args=None, wrapper=None, name=None, - test_dir="/var/tmp", monitor_address=None, + test_dir="/var/tmp", + monitor_address: Optional[SocketAddrT] = None, socket_scm_helper=None, sock_dir=None): ''' Initialize a QEMUMachine
@@ -84,8 +89,14 @@ def __init__(self, binary, args=None, wrapper=None, name=None, if sock_dir is None: sock_dir = test_dir self._name = name - self._monitor_address = monitor_address - self._vm_monitor = None + if monitor_address is not None: + self._monitor_address = monitor_address + self._remove_monitor_sockfile = False + else: + self._monitor_address = os.path.join( + sock_dir, f"{name}-monitor.sock" + ) + self._remove_monitor_sockfile = True self._qemu_log_path = None self._qemu_log_file = None self._popen = None
@@ -223,15 +234,17 @@ def _load_io_log(self): def _base_args(self): args = ['-display', 'none', '-vga', 'none'] + if self._qmp_set: if isinstance(self._monitor_address, tuple): - moncdev = "socket,id=mon,host=%s,port=%s" % ( - self._monitor_address[0], - self._monitor_address[1]) + moncdev = "socket,id=mon,host={},port={}".format( + *self._monitor_address + ) else: - moncdev = 'socket,id=mon,path=%s' % self._vm_monitor + moncdev = f"socket,id=mon,path={self._monitor_address}" args.extend(['-chardev', moncdev, '-mon', 'chardev=mon,mode=control']) + if self._machine is not None: args.extend(['-machine', self._machine]) for _ in range(self._console_index):
@@ -256,14 +269,14 @@ def _pre_launch(self): self._qemu_log_file = open(self._qemu_log_path, 'wb') if self._qmp_set: - if self._monitor_address is not None: - self._vm_monitor = self._monitor_address - else: - self._vm_monitor = os.path.join(self._sock_dir, - self._name + "-monitor.sock") - self._remove_files.append(self._vm_monitor) - self._qmp = qmp.QEMUMonitorProtocol(self._vm_monitor, server=True, - nickname=self._name) + if self._remove_monitor_sockfile: + assert isinstance(self._monitor_address, str) + self._remove_files.append(self._monitor_address) + self._qmp = qmp.QEMUMonitorProtocol( + self._monitor_address, + server=True, + nickname=self._name + ) def _post_launch(self): if self._qmp:
Signature · 1 line
2.21.1
RFC PATCH 17/32 python//qmp.py: Do not return None from cmd_obj
In reply to John SnowThis makes typing the qmp library difficult, as it necessitates wrapping Optional[] around the type for every return type up the stack. At some point, it becomes difficult to discern or remember why it's None instead of the expected object.
Use the python exception system to tell us exactly why we didn't get an object. Remove this special-cased return.
--- python/qemu/lib/qmp.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-)
python/qemu/lib/qmp.py +5 -9
@@ -224,22 +224,18 @@ def accept(self, timeout=15.0): self.__sockfile = self.__sock.makefile(mode='r') return self.__negotiate_capabilities() - def cmd_obj(self, qmp_cmd): + def cmd_obj(self, qmp_cmd: QMPMessage) -> QMPMessage: """ Send a QMP command to the QMP Monitor. @param qmp_cmd: QMP command to be sent as a Python dict - @return QMP response as a Python dict or None if the connection has - been closed + @return QMP response as a Python dict """ self.logger.debug(">>> %s", qmp_cmd) - try: - self.__sock.sendall(json.dumps(qmp_cmd).encode('utf-8')) - except OSError as err: - if err.errno == errno.EPIPE: - return None - raise err + self.__sock.sendall(json.dumps(qmp_cmd).encode('utf-8')) resp = self.__json_read() + if resp is None: + raise QMPConnectError("Unexpected empty reply from server") self.logger.debug("<<< %s", resp) return resp
Signature · 1 line
2.21.1
RFC PATCH 23/32 python//machine.py: reorder __init__
In reply to John SnowPut the init arg handling all at the top, and mostly in order (deviating when one is dependent on another), and put what is effectively runtime state declaration at the bottom.
--- python/qemu/lib/machine.py | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-)
python/qemu/lib/machine.py +17 -12
@@ -80,38 +80,43 @@ def __init__(self, binary, args=None, wrapper=None, name=None, @param socket_scm_helper: helper program, required for send_fd_scm() @note: Qemu process is not started until launch() is used. ''' + # Direct user configuration + + self._binary = binary + if args is None: args = [] + # Copy mutable input: we will be modifying our copy + self._args = list(args) + if wrapper is None: wrapper = [] - if name is None: - name = "qemu-%d" % os.getpid() - if sock_dir is None: - sock_dir = test_dir - self._name = name + self._wrapper = wrapper + + self._name = name or "qemu-%d" % os.getpid() + self._test_dir = test_dir + self._sock_dir = sock_dir or self._test_dir + self._socket_scm_helper = socket_scm_helper + if monitor_address is not None: self._monitor_address = monitor_address self._remove_monitor_sockfile = False else: self._monitor_address = os.path.join( - sock_dir, f"{name}-monitor.sock" + self._sock_dir, f"{self._name}-monitor.sock" ) self._remove_monitor_sockfile = True + + # Runstate self._qemu_log_path = None self._qemu_log_file = None self._popen = None - self._binary = binary - self._args = list(args) # Force copy args in case we modify them - self._wrapper = wrapper self._events = [] self._iolog = None - self._socket_scm_helper = socket_scm_helper self._qmp_set = True # Enable QMP monitor by default. self._qmp = None self._qemu_full_args = None - self._test_dir = test_dir self._temp_dir = None - self._sock_dir = sock_dir self._launched = False self._machine = None self._console_index = 0
Signature · 1 line
2.21.1
RFC PATCH 12/32 python/qemu/lib: fix socket.makefile() typing
In reply to John SnowNote:
A bug in typeshed (https://github.com/python/typeshed/issues/3977)
misinterprets the type of makefile(). Work around this by explicitly stating that we are opening a text-mode file.
--- python/qemu/lib/qmp.py | 10 +++++++--- python/qemu/lib/qtest.py | 12 ++++++++---- 2 files changed, 15 insertions(+), 7 deletions(-)
python/qemu/lib/qmp.py +7 -3
@@ -11,6 +11,10 @@ import errno import socket import logging +from typing import ( + Optional, + TextIO, +) class QMPError(Exception):
@@ -61,7 +65,7 @@ def __init__(self, address, server=False, nickname=None): self.__events = [] self.__address = address self.__sock = self.__get_sock() - self.__sockfile = None + self.__sockfile: Optional[TextIO] = None self._nickname = nickname if self._nickname: self.logger = logging.getLogger('QMP').getChild(self._nickname)
@@ -157,7 +161,7 @@ def connect(self, negotiate=True): @raise QMPCapabilitiesError if fails to negotiate capabilities """ self.__sock.connect(self.__address) - self.__sockfile = self.__sock.makefile() + self.__sockfile = self.__sock.makefile(mode='r') if negotiate: return self.__negotiate_capabilities() return None
@@ -180,7 +184,7 @@ def accept(self, timeout=15.0): """ self.__sock.settimeout(timeout) self.__sock, _ = self.__sock.accept() - self.__sockfile = self.__sock.makefile() + self.__sockfile = self.__sock.makefile(mode='r') return self.__negotiate_capabilities() def cmd_obj(self, qmp_cmd):
python/qemu/lib/qtest.py +8 -4
@@ -19,6 +19,7 @@ import socket import os +from typing import Optional, TextIO from .machine import QEMUMachine
@@ -40,7 +41,7 @@ class QEMUQtestProtocol: def __init__(self, address, server=False): self._address = address self._sock = self._get_sock() - self._sockfile = None + self._sockfile: Optional[TextIO] = None if server: self._sock.bind(self._address) self._sock.listen(1)
@@ -59,7 +60,7 @@ def connect(self): @raise socket.error on socket connection errors """ self._sock.connect(self._address) - self._sockfile = self._sock.makefile() + self._sockfile = self._sock.makefile(mode='r') def accept(self): """
@@ -68,7 +69,7 @@ def accept(self): @raise socket.error on socket connection errors """ self._sock, _ = self._sock.accept() - self._sockfile = self._sock.makefile() + self._sockfile = self._sock.makefile(mode='r') def cmd(self, qtest_cmd): """
@@ -76,6 +77,7 @@ def cmd(self, qtest_cmd): @param qtest_cmd: qtest command text to be sent """ + assert self._sockfile is not None self._sock.sendall((qtest_cmd + "\n").encode('utf-8')) resp = self._sockfile.readline() return resp
@@ -83,7 +85,9 @@ def cmd(self, qtest_cmd): def close(self): """Close this socket.""" self._sock.close() - self._sockfile.close() + if self._sockfile: + self._sockfile.close() + self._sockfile = None def settimeout(self, timeout): """Set a timeout, in seconds."""
Signature · 1 line
2.21.1
RFC PATCH 28/32 python//machine.py: fix _popen access
In reply to John SnowAs always, Optional[T] causes problems with unchecked access. Add a helper that asserts the pipe is present before we attempt to talk with it.
--- python/qemu/lib/machine.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-)
python/qemu/lib/machine.py +12 -6
@@ -113,7 +113,7 @@ def __init__(self, binary, args=None, wrapper=None, name=None, # Runstate self._qemu_log_path = None self._qemu_log_file = None - self._popen = None + self._popen: Optional['subprocess.Popen[bytes]'] = None self._events = [] self._iolog = None self._qmp_set = True # Enable QMP monitor by default.
@@ -225,6 +225,12 @@ def is_running(self): """Returns true if the VM is running.""" return self._popen is not None and self._popen.poll() is None + @property + def _subp(self) -> 'subprocess.Popen[bytes]': + if self._popen is None: + raise QEMUMachineError('Subprocess pipe not present') + return self._popen + def exitcode(self): """Returns the exit code if possible, or None.""" if self._popen is None:
@@ -235,7 +241,7 @@ def get_pid(self): """Returns the PID of the running process, or None.""" if not self.is_running(): return None - return self._popen.pid + return self._subp.pid def _load_io_log(self): if self._qemu_log_path is not None:
@@ -352,7 +358,7 @@ def wait(self): """ Wait for the VM to power off """ - self._popen.wait() + self._subp.wait() if self._qmp: self._qmp.close() self._load_io_log()
@@ -371,11 +377,11 @@ def _issue_shutdown(self, has_quit: bool = False) -> None: self._qmp.close() try: - self._popen.wait(timeout=3) + self._subp.wait(timeout=3) except subprocess.TimeoutExpired: - self._popen.kill() + self._subp.kill() - self._popen.wait() + self._subp.wait() def shutdown(self, has_quit: bool = False) -> None: """
Signature · 1 line
2.21.1
RFC PATCH 24/32 python//machine.py: Don't modify state in _base_args()
In reply to John SnowDon't append to the _remove_files list during _base_args; instead do so during _launch. Rework _base_args as a @property to help facilitate this impression.
This has the additional benefit of making the type of _console_address easier to analyze statically.
--- python/qemu/lib/machine.py | 16 ++++++++++------ python/qemu/lib/qtest.py | 11 ++++++++--- 2 files changed, 18 insertions(+), 9 deletions(-)
python/qemu/lib/machine.py +10 -6
@@ -25,6 +25,7 @@ import socket import tempfile from typing import ( + List, Optional, Type, )
@@ -122,7 +123,9 @@ def __init__(self, binary, args=None, wrapper=None, name=None, self._console_index = 0 self._console_set = False self._console_device_type = None - self._console_address = None + self._console_address = os.path.join( + self._sock_dir, f"{self._name}-console.sock" + ) self._console_socket = None self._remove_files = []
@@ -237,7 +240,8 @@ def _load_io_log(self): with open(self._qemu_log_path, "r") as iolog: self._iolog = iolog.read() - def _base_args(self): + @property + def _base_args(self) -> List[str]: args = ['-display', 'none', '-vga', 'none'] if self._qmp_set:
@@ -255,9 +259,6 @@ def _base_args(self): for _ in range(self._console_index): args.extend(['-serial', 'null']) if self._console_set: - self._console_address = os.path.join(self._sock_dir, - self._name + "-console.sock") - self._remove_files.append(self._console_address) chardev = ('socket,id=console,path=%s,server,nowait' % self._console_address) args.extend(['-chardev', chardev])
@@ -273,6 +274,9 @@ def _pre_launch(self): self._qemu_log_path = os.path.join(self._temp_dir, self._name + ".log") self._qemu_log_file = open(self._qemu_log_path, 'wb') + if self._console_set: + self._remove_files.append(self._console_address) + if self._qmp_set: if self._remove_monitor_sockfile: assert isinstance(self._monitor_address, str)
@@ -332,7 +336,7 @@ def _launch(self): devnull = open(os.path.devnull, 'rb') self._pre_launch() self._qemu_full_args = (self._wrapper + [self._binary] + - self._base_args() + self._args) + self._base_args + self._args) LOG.debug('VM launch command: %r', ' '.join(self._qemu_full_args)) self._popen = subprocess.Popen(self._qemu_full_args, stdin=devnull,
python/qemu/lib/qtest.py +8 -3
@@ -19,7 +19,11 @@ import socket import os -from typing import Optional, TextIO +from typing import ( + List, + Optional, + TextIO, +) from .machine import QEMUMachine
@@ -111,8 +115,9 @@ def __init__(self, binary, args=None, name=None, test_dir="/var/tmp", self._qtest = None self._qtest_path = os.path.join(sock_dir, name + "-qtest.sock") - def _base_args(self): - args = super()._base_args() + @property + def _base_args(self) -> List[str]: + args = super()._base_args args.extend(['-qtest', 'unix:path=' + self._qtest_path, '-accel', 'qtest']) return args
Signature · 1 line
2.21.1
RFC PATCH 25/32 python//machine.py: Handle None events in event_wait
In reply to John SnowIf the timeout is 0, we can get None back. Handle this explicitly.
--- python/qemu/lib/machine.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-)
python/qemu/lib/machine.py +5 -1
@@ -32,7 +32,7 @@ from types import TracebackType from . import qmp -from .qmp import SocketAddrT +from .qmp import SocketAddrT, QMPMessage LOG = logging.getLogger(__name__)
@@ -519,6 +519,8 @@ def _match(event): return True return False + event: Optional[QMPMessage] + # Search cached events for event in self._events: if _match(event):
@@ -528,6 +530,8 @@ def _match(event): # Poll for new events while True: event = self._qmp.pull_event(wait=timeout) + if event is None: + break if _match(event): return event self._events.append(event)
Signature · 1 line
2.21.1
RFC PATCH 29/32 python//qtest.py: Check before accessing _qtest
In reply to John SnowIt can be None; so add assertions or exceptions where appropriate to guard the access accordingly.
--- python/qemu/lib/qtest.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-)
python/qemu/lib/qtest.py +11 -3
@@ -126,7 +126,8 @@ def _pre_launch(self): super()._pre_launch() self._qtest = QEMUQtestProtocol(self._qtest_path, server=True) - def _post_launch(self): + def _post_launch(self) -> None: + assert self._qtest is not None super()._post_launch() self._qtest.accept()
@@ -134,6 +135,13 @@ def _post_shutdown(self): super()._post_shutdown() self._remove_if_exists(self._qtest_path) - def qtest(self, cmd): - '''Send a qtest command to guest''' + def qtest(self, cmd: str) -> str: + """ + Send a qtest command to the guest. + + :param cmd: qtest command to send + :return: qtest server response + """ + if self._qtest is None: + raise RuntimeError("qtest socket not available") return self._qtest.cmd(cmd)
Signature · 1 line
2.21.1
Re: [PATCH RFC 16/32] python//qmp.py: re-absorb MonitorResponseError
In reply to John SnowQuoted 7 lines · John Snow
When I initially split this out, I considered this more of a machine error than a QMP protocol error, but I think that's misguided.
Move this back to
qmp.pyand name it QMPResponseError. Convertqmp.command()to use this exception type.Signed-off-by John Snow <[email protected]>
Quoted 97 lines
--- python/qemu/lib/machine.py | 15 +-------------- python/qemu/lib/qmp.py | 17 +++++++++++++++-- scripts/render_block_graph.py | 4 ++-- 3 files changed, 18 insertions(+), 18 deletions(-)python/qemu/lib/machine.py +1
-14@@ -48,19 +48,6 @@ class QEMUMachineAddDeviceError(QEMUMachineError): """ -class MonitorResponseError(qmp.QMPError): - """ - Represents erroneous QMP monitor reply - """ - def __init__(self, reply): - try: - desc = reply["error"]["desc"] - except KeyError: - desc = reply - super().__init__(desc) - self.reply = reply - - class QEMUMachine: """ A QEMU VM@@ -433,7 +420,7 @@ def command(self, cmd, conv_keys=True, **args): if reply is None: raise qmp.QMPError("Monitor is closed") if "error" in reply: - raise MonitorResponseError(reply) + raise qmp.QMPResponseError(reply) return reply["return"] def get_qmp_event(self, wait=False):python/qemu/lib/qmp.py +15
-2@@ -61,6 +61,19 @@ class QMPTimeoutError(QMPError): """ +class QMPResponseError(QMPError): + """ + Represents erroneous QMP monitor reply + """ + def __init__(self, reply: QMPMessage): + try: + desc = reply['error']['desc'] + except KeyError: + desc = reply + super().__init__(desc) + self.reply = reply + + class QEMUMonitorProtocol: """ Provide an API to connect to QEMU via QEMU Monitor Protocol (QMP) and then@@ -250,8 +263,8 @@ def command(self, cmd, **kwds): Build and send a QMP command to the monitor, report errors if any """ ret = self.cmd(cmd, kwds) - if "error" in ret: - raise Exception(ret['error']['desc']) + if 'error' in ret: + raise QMPResponseError(ret) return ret['return'] def pull_event(self, wait=False):scripts/render_block_graph.py +2
-2@@ -26,7 +26,7 @@ sys.path.append(os.path.join(os.path.dirname(__file__), '..', 'python')) from qemu.lib import QEMUMonitorProtocol -from qemu.lib.machine import MonitorResponseError +from qemu.lib.qmp import QMPResponseError def perm(arr):@@ -103,7 +103,7 @@ def command(self, cmd): reply = json.loads(subprocess.check_output(ar)) if 'error' in reply: - raise MonitorResponseError(reply) + raise QEMUResponseError(reply) return reply['return']
RFC PATCH 27/32 python//machine.py: Add _qmp access shim
In reply to John SnowLike many other Optional[] types, it's not always a given that this object will be set. Wrap it in a type-shim that raises a meaningful error and will always return a concrete type.
--- python/qemu/lib/machine.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-)
python/qemu/lib/machine.py +9 -3
@@ -117,7 +117,7 @@ def __init__(self, binary, args=None, wrapper=None, name=None, self._events = [] self._iolog = None self._qmp_set = True # Enable QMP monitor by default. - self._qmp = None + self._qmp_connection: Optional[qmp.QEMUMonitorProtocol] = None self._qemu_full_args = None self._temp_dir = None self._launched = False
@@ -283,7 +283,7 @@ def _pre_launch(self): if self._remove_monitor_sockfile: assert isinstance(self._monitor_address, str) self._remove_files.append(self._monitor_address) - self._qmp = qmp.QEMUMonitorProtocol( + self._qmp_connection = qmp.QEMUMonitorProtocol( self._monitor_address, server=True, nickname=self._name
@@ -416,7 +416,13 @@ def set_qmp_monitor(self, enabled=True): self._qmp_set = True else: self._qmp_set = False - self._qmp = None + self._qmp_connection = None + + @property + def _qmp(self) -> qmp.QEMUMonitorProtocol: + if self._qmp_connection is None: + raise QEMUMachineError("Attempt to access QMP with no connection") + return self._qmp_connection @classmethod def _qmp_args(cls, _conv_keys: bool = True, **args: Any) -> Dict[str, Any]:
Signature · 1 line
2.21.1
Re: [PATCH RFC 21/32] python//machine.py: remove logging configuration
In reply to John SnowQuoted 15 lines · John Snow
Python 3.5 and above do not print a warning when logging is not configured. As a library, it's best practice to leave logging configuration to the client executable.
Signed-off-by John Snow <[email protected]>--- python/qemu/lib/machine.py | 3 --- 1 file changed, 3 deletions(-)python/qemu/lib/machine.py +0
-0
python/qemu/lib/machine.py def __init__(self, binary, args=None, wrapper=None, name=None, · line 115
Quoted 9 lines
python/qemu/lib/machine.py +0
-3@@ -110,9 +110,6 @@ def __init__(self, binary, args=None, wrapper=None, name=None, self._console_socket = None self._remove_files = [] - # just in case logging wasn't configured by the main script: - logging.basicConfig() - def __enter__(self): return self
Re: [PATCH RFC 23/32] python//machine.py: reorder __init__
In reply to John SnowQuoted 60 lines · John Snow
Put the init arg handling all at the top, and mostly in order (deviating when one is dependent on another), and put what is effectively runtime state declaration at the bottom.
Signed-off-by John Snow <[email protected]>--- python/qemu/lib/machine.py | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-)python/qemu/lib/machine.py +17
-9@@ -80,38 +80,43 @@ def __init__(self, binary, args=None, wrapper=None, name=None, @param socket_scm_helper: helper program, required for send_fd_scm() @note: Qemu process is not started until launch() is used. ''' + # Direct user configuration + + self._binary = binary + if args is None: args = [] + # Copy mutable input: we will be modifying our copy + self._args = list(args) + if wrapper is None: wrapper = [] - if name is None: - name = "qemu-%d" % os.getpid() - if sock_dir is None: - sock_dir = test_dir - self._name = name + self._wrapper = wrapper + + self._name = name or "qemu-%d" % os.getpid() + self._test_dir = test_dir + self._sock_dir = sock_dir or self._test_dir + self._socket_scm_helper = socket_scm_helper + if monitor_address is not None: self._monitor_address = monitor_address self._remove_monitor_sockfile = False else: self._monitor_address = os.path.join( - sock_dir, f"{name}-monitor.sock" + self._sock_dir, f"{self._name}-monitor.sock" ) self._remove_monitor_sockfile = True + + # Runstate self._qemu_log_path = None self._qemu_log_file = None self._popen = None - self._binary = binary - self._args = list(args) # Force copy args in case we modify them - self._wrapper = wrapper self._events = [] self._iolog = None
python/qemu/lib/machine.py def __init__(self, binary, args=None, wrapper=None, name=None, · line 122
Quoted 10 lines
python/qemu/lib/machine.py +0
-3- self._socket_scm_helper = socket_scm_helper self._qmp_set = True # Enable QMP monitor by default. self._qmp = None self._qemu_full_args = None - self._test_dir = test_dir self._temp_dir = None - self._sock_dir = sock_dir self._launched = False self._machine = None self._console_index = 0
RFC PATCH 19/32 python//qmp.py: add QMPProtocolError
In reply to John SnowIn the case that we receive a reply but are unable to understand it, use this exception name to indicate that case.
--- python/qemu/lib/qmp.py | 10 ++++++++++ 1 file changed, 10 insertions(+)
python/qemu/lib/qmp.py +10 -0
@@ -62,6 +62,12 @@ class QMPTimeoutError(QMPError): """ +class QMPProtocolError(QMPError): + """ + QMP protocol error; unexpected response + """ + + class QMPResponseError(QMPError): """ Represents erroneous QMP monitor reply
@@ -265,6 +271,10 @@ def command(self, cmd, **kwds): ret = self.cmd(cmd, kwds) if 'error' in ret: raise QMPResponseError(ret) + if 'return' not in ret: + raise QMPProtocolError( + "'return' key not found in QMP response '{}'".format(str(ret)) + ) return cast(QMPReturnValue, ret['return']) def pull_event(self, wait=False):
Signature · 1 line
2.21.1
Re: [PATCH RFC 29/32] python//qtest.py: Check before accessing _qtest
In reply to John SnowQuoted 29 lines · John Snow
It can be None; so add assertions or exceptions where appropriate to guard the access accordingly.
Signed-off-by John Snow <[email protected]>--- python/qemu/lib/qtest.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-)python/qemu/lib/qtest.py +2
-3@@ -126,7 +126,8 @@ def _pre_launch(self): super()._pre_launch() self._qtest = QEMUQtestProtocol(self._qtest_path, server=True) - def _post_launch(self): + def _post_launch(self) -> None: + assert self._qtest is not None super()._post_launch() self._qtest.accept()@@ -134,6 +135,13 @@ def _post_shutdown(self): super()._post_shutdown() self._remove_if_exists(self._qtest_path) - def qtest(self, cmd): - '''Send a qtest command to guest'''
python/qemu/lib/qtest.py def _post_shutdown(self): · line 147
Quoted 10 lines
python/qemu/lib/qtest.py +9
-0+ def qtest(self, cmd: str) -> str: + """ + Send a qtest command to the guest. + + :param cmd: qtest command to send + :return: qtest server response + """ + if self._qtest is None: + raise RuntimeError("qtest socket not available") return self._qtest.cmd(cmd)
RFC PATCH 31/32 python/qemu: add mypy to Pipfile
In reply to John Snow--- python/Pipfile | 1 + python/Pipfile.lock | 37 ++++++++++++++++++++++++++++++++++++- 2 files changed, 37 insertions(+), 1 deletion(-)
python/Pipfile +1 -0
@@ -6,6 +6,7 @@ verify_ssl = true [dev-packages] pylint = "==2.5.0" flake8 = "*" +mypy = "*" [packages]
python/Pipfile.lock +36 -1
@@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "58115144ace0f646b5c62da260fb4867ac0a0e485de3f5b0a56c7854afa21f5b" + "sha256": "17a30ead8719d80349ff5473bda3133fd4559a4e83d3f9d669631ac4b2171502" }, "pipfile-spec": 6, "requires": {
@@ -79,6 +79,33 @@ ], "version": "==0.6.1" }, + "mypy": { + "hashes": [ + "sha256:15b948e1302682e3682f11f50208b726a246ab4e6c1b39f9264a8796bb416aa2", + "sha256:219a3116ecd015f8dca7b5d2c366c973509dfb9a8fc97ef044a36e3da66144a1", + "sha256:3b1fc683fb204c6b4403a1ef23f0b1fac8e4477091585e0c8c54cbdf7d7bb164", + "sha256:3beff56b453b6ef94ecb2996bea101a08f1f8a9771d3cbf4988a61e4d9973761", + "sha256:7687f6455ec3ed7649d1ae574136835a4272b65b3ddcf01ab8704ac65616c5ce", + "sha256:7ec45a70d40ede1ec7ad7f95b3c94c9cf4c186a32f6bacb1795b60abd2f9ef27", + "sha256:86c857510a9b7c3104cf4cde1568f4921762c8f9842e987bc03ed4f160925754", + "sha256:8a627507ef9b307b46a1fea9513d5c98680ba09591253082b4c48697ba05a4ae", + "sha256:8dfb69fbf9f3aeed18afffb15e319ca7f8da9642336348ddd6cab2713ddcf8f9", + "sha256:a34b577cdf6313bf24755f7a0e3f3c326d5c1f4fe7422d1d06498eb25ad0c600", + "sha256:a8ffcd53cb5dfc131850851cc09f1c44689c2812d0beb954d8138d4f5fc17f65", + "sha256:b90928f2d9eb2f33162405f32dde9f6dcead63a0971ca8a1b50eb4ca3e35ceb8", + "sha256:c56ffe22faa2e51054c5f7a3bc70a370939c2ed4de308c690e7949230c995913", + "sha256:f91c7ae919bbc3f96cd5e5b2e786b2b108343d1d7972ea130f7de27fdd547cf3" + ], + "index": "pypi", + "version": "==0.770" + }, + "mypy-extensions": { + "hashes": [ + "sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d", + "sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8" + ], + "version": "==0.4.3" + }, "pycodestyle": { "hashes": [ "sha256:95a2219d12372f05704562a14ec30bc76b05a5b297b21a5dfe3f6fac3491ae56",
@@ -142,6 +169,14 @@ "markers": "implementation_name == 'cpython' and python_version < '3.8'", "version": "==1.4.1" }, + "typing-extensions": { + "hashes": [ + "sha256:6e95524d8a547a91e08f404ae485bbb71962de46967e1b71a0cb89af24e761c5", + "sha256:79ee589a3caca649a9bfd2a8de4709837400dfa00b6cc81962a1e6a1815969ae", + "sha256:f8d2bd89d25bc39dabe7d23df520442fa1d8969b82544370e03d88b5a591c392" + ], + "version": "==3.7.4.2" + }, "wrapt": { "hashes": [ "sha256:b62ffa81fb85f4332a4f609cab4ac40709470da05643a082ec1eb88e6d9b97d7"
Signature · 1 line
2.21.1
RFC PATCH 32/32 python/qemu/lib: Add mypy type annotations
In reply to John SnowThese should all be purely annotations with no changes in behavior at all.
--- python/qemu/lib/accel.py | 8 ++-- python/qemu/lib/machine.py | 94 +++++++++++++++++++++----------------- python/qemu/lib/qmp.py | 44 ++++++++++-------- python/qemu/lib/qtest.py | 27 ++++++----- 4 files changed, 99 insertions(+), 74 deletions(-)
python/qemu/lib/accel.py +5 -3
@@ -17,6 +17,7 @@ import logging import os import subprocess +from typing import List, Optional LOG = logging.getLogger(__name__)
@@ -29,7 +30,7 @@ } -def list_accel(qemu_bin): +def list_accel(qemu_bin: str) -> List[str]: """ List accelerators enabled in the QEMU binary.
@@ -49,7 +50,8 @@ def list_accel(qemu_bin): return [acc.strip() for acc in out.splitlines()[1:]] -def kvm_available(target_arch=None, qemu_bin=None): +def kvm_available(target_arch: Optional[str] = None, + qemu_bin: Optional[str] = None) -> bool: """ Check if KVM is available using the following heuristic: - Kernel module is present in the host;
@@ -72,7 +74,7 @@ def kvm_available(target_arch=None, qemu_bin=None): return True -def tcg_available(qemu_bin): +def tcg_available(qemu_bin: str) -> bool: """ Check if TCG is available.
python/qemu/lib/machine.py +53 -41
292 lines not on this page · raw message
python/qemu/lib/qmp.py +25 -19
163 lines not on this page · raw message
python/qemu/lib/qtest.py +16 -11
@@ -24,6 +24,8 @@ Optional, Sequence, TextIO, + Tuple, + Union, ) from .machine import QEMUMachine
@@ -43,7 +45,8 @@ class QEMUQtestProtocol: No conection is estabalished by __init__(), this is done by the connect() or accept() methods. """ - def __init__(self, address, server=False): + def __init__(self, address: Union[Tuple[str, str], str], + server: bool = False): self._address = address self._sock = self._get_sock() self._sockfile: Optional[TextIO] = None
@@ -51,14 +54,14 @@ def __init__(self, address, server=False): self._sock.bind(self._address) self._sock.listen(1) - def _get_sock(self): + def _get_sock(self) -> socket.socket: if isinstance(self._address, tuple): family = socket.AF_INET else: family = socket.AF_UNIX return socket.socket(family, socket.SOCK_STREAM) - def connect(self): + def connect(self) -> None: """ Connect to the qtest socket.
@@ -67,7 +70,7 @@ def connect(self): self._sock.connect(self._address) self._sockfile = self._sock.makefile(mode='r') - def accept(self): + def accept(self) -> None: """ Await connection from QEMU.
@@ -76,7 +79,7 @@ def accept(self): self._sock, _ = self._sock.accept() self._sockfile = self._sock.makefile(mode='r') - def cmd(self, qtest_cmd): + def cmd(self, qtest_cmd: str) -> str: """ Send a qtest command on the wire.
@@ -87,14 +90,16 @@ def cmd(self, qtest_cmd): resp = self._sockfile.readline() return resp - def close(self): - """Close this socket.""" + def close(self) -> None: + """ + Close this socket. + """ self._sock.close() if self._sockfile: self._sockfile.close() self._sockfile = None - def settimeout(self, timeout): + def settimeout(self, timeout: Optional[float]) -> None: """Set a timeout, in seconds.""" self._sock.settimeout(timeout)
@@ -118,7 +123,7 @@ def __init__(self, super().__init__(binary, args, name=name, test_dir=test_dir, socket_scm_helper=socket_scm_helper, sock_dir=sock_dir) - self._qtest = None + self._qtest: Optional[QEMUQtestProtocol] = None self._qtest_path = os.path.join(sock_dir, name + "-qtest.sock") @property
@@ -130,7 +135,7 @@ def _base_args(self) -> List[str]: ]) return args - def _pre_launch(self): + def _pre_launch(self) -> None: super()._pre_launch() self._qtest = QEMUQtestProtocol(self._qtest_path, server=True)
@@ -139,7 +144,7 @@ def _post_launch(self) -> None: super()._post_launch() self._qtest.accept() - def _post_shutdown(self): + def _post_shutdown(self) -> None: super()._post_shutdown() self._remove_if_exists(self._qtest_path)
Signature · 1 line
2.21.1
Re: [PATCH RFC 11/32] python/qemu/lib: remove Python2 style super() calls
In reply to John SnowQuoted 3 lines · John Snow
Use the Python3 style instead.
Signed-off-by John Snow <[email protected]>
Quoted 60 lines
--- python/qemu/lib/machine.py | 2 +- python/qemu/lib/qtest.py | 15 +++++++-------- 2 files changed, 8 insertions(+), 9 deletions(-)python/qemu/lib/machine.py +1
-1@@ -55,7 +55,7 @@ def __init__(self, reply): desc = reply["error"]["desc"] except KeyError: desc = reply - super(MonitorResponseError, self).__init__(desc) + super().__init__(desc) self.reply = replypython/qemu/lib/qtest.py +7
-8@@ -101,29 +101,28 @@ def __init__(self, binary, args=None, name=None, test_dir="/var/tmp", name = "qemu-%d" % os.getpid() if sock_dir is None: sock_dir = test_dir - super(QEMUQtestMachine, - self).__init__(binary, args, name=name, test_dir=test_dir, - socket_scm_helper=socket_scm_helper, - sock_dir=sock_dir) + super().__init__(binary, args, name=name, test_dir=test_dir, + socket_scm_helper=socket_scm_helper, + sock_dir=sock_dir) self._qtest = None self._qtest_path = os.path.join(sock_dir, name + "-qtest.sock") def _base_args(self): - args = super(QEMUQtestMachine, self)._base_args() + args = super()._base_args() args.extend(['-qtest', 'unix:path=' + self._qtest_path, '-accel', 'qtest']) return args def _pre_launch(self): - super(QEMUQtestMachine, self)._pre_launch() + super()._pre_launch() self._qtest = QEMUQtestProtocol(self._qtest_path, server=True) def _post_launch(self): - super(QEMUQtestMachine, self)._post_launch() + super()._post_launch() self._qtest.accept() def _post_shutdown(self): - super(QEMUQtestMachine, self)._post_shutdown() + super()._post_shutdown() self._remove_if_exists(self._qtest_path) def qtest(self, cmd):
Re: [PATCH RFC 14/32] python//qmp.py: use True/False for non/blocking modes
In reply to John SnowQuoted 19 lines · John Snow
The type system doesn't want integers.
Signed-off-by John Snow <[email protected]>--- python/qemu/lib/qmp.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)python/qemu/lib/qmp.py +1
-1@@ -120,14 +120,14 @@ def __get_events(self, wait=False): """ # Check for new events regardless and pull them into the cache: - self.__sock.setblocking(0) + self.__sock.setblocking(False) try:
python/qemu/lib/qmp.py def __get_events(self, wait=False): · line 133
Quoted 10 lines
python/qemu/lib/qmp.py +1
-1self.__json_read() except OSError as err: if err.errno == errno.EAGAIN: # No data available pass - self.__sock.setblocking(1) + self.__sock.setblocking(True) # Wait for new events, if needed. # if wait is 0.0, this means "no wait" and is also implicitly false.
Re: [PATCH RFC 19/32] python//qmp.py: add QMPProtocolError
In reply to John SnowQuoted 27 lines · John Snow
In the case that we receive a reply but are unable to understand it, use this exception name to indicate that case.
Signed-off-by John Snow <[email protected]>--- python/qemu/lib/qmp.py | 10 ++++++++++ 1 file changed, 10 insertions(+)python/qemu/lib/qmp.py +6
-0@@ -62,6 +62,12 @@ class QMPTimeoutError(QMPError): """ +class QMPProtocolError(QMPError): + """ + QMP protocol error; unexpected response + """ + + class QMPResponseError(QMPError): """ Represents erroneous QMP monitor reply
python/qemu/lib/qmp.py def command(self, cmd, **kwds): · line 280
Quoted 10 lines
python/qemu/lib/qmp.py +4
-0@@ -265,6 +271,10 @@ def command(self, cmd, **kwds): ret = self.cmd(cmd, kwds) if 'error' in ret: raise QMPResponseError(ret) + if 'return' not in ret: + raise QMPProtocolError( + "'return' key not found in QMP response '{}'".format(str(ret)) + ) return cast(QMPReturnValue, ret['return']) def pull_event(self, wait=False):
Re: [PATCH RFC 03/32] python//machine.py: remove bare except
In reply to John SnowQuoted 3 lines · John Snow
Catch only the timeout error; if there are other problems, allow the stack trace to be visible.
A lot of patches in this series start with "python//" - is that intentional, or should it be a single slash?
Quoted 5 lines
Signed-off-by John Snow <[email protected]>--- python/qemu/lib/machine.py | 33 +++++++++++++++++++++------------ 1 file changed, 21 insertions(+), 12 deletions(-)
Signature · 3 lines
Eric Blake, Principal Software Engineer Red Hat, Inc. +1-919-301-3226 Virtualization: qemu.org | libvirt.org
Re: [PATCH RFC 03/32] python//machine.py: remove bare except
Quoted 8 lines · Eric Blake
Quoted 3 lines · John Snow
Catch only the timeout error; if there are other problems, allow the stack trace to be visible.
A lot of patches in this series start with "python//" - is that intentional, or should it be a single slash?
Was trying to imply an elided path structure, because "python/qemu/lib/qmp.py" et al is way too chatty.
Feel free to suggest better subject names!
Quoted 8 lines · John Snow
python//qmp.py: use True/False for non/blocking modespython//qmp.py: Define common typespython//qmp.py: re-absorb MonitorResponseErrorpython//qmp.py: Do not return None fromcmd_objpython//qmp.py: add casts to JSON deserializationpython//qmp.py: add QMPProtocolErrorpython//qmp.py: assert sockfile is not Nonepython//machine.py: remove logging configuration
Quoted 8 lines
python//machine.py: Fix monitor address typingpython//machine.py: reorder__init__python//machine.py: Don't modify state in_base_args()python//machine.py: Handle None events inevent_waitpython//machine.py: use qmp.commandpython//machine.py: Add_qmpaccess shimpython//machine.py: fix_popenaccesspython//qtest.py: Check before accessing_qtest
Maybe remove double // in patch subjects :)
Quoted 19 lines · Philippe Mathieu-Daudé
Quoted 8 lines · John Snow
python//qmp.py: use True/False for non/blocking modespython//qmp.py: Define common typespython//qmp.py: re-absorb MonitorResponseErrorpython//qmp.py: Do not return None fromcmd_objpython//qmp.py: add casts to JSON deserializationpython//qmp.py: add QMPProtocolErrorpython//qmp.py: assert sockfile is not Nonepython//machine.py: remove logging configurationQuoted 8 lines
python//machine.py: Fix monitor address typingpython//machine.py: reorder__init__python//machine.py: Don't modify state in_base_args()python//machine.py: Handle None events inevent_waitpython//machine.py: use qmp.commandpython//machine.py: Add_qmpaccess shimpython//machine.py: fix_popenaccesspython//qtest.py: Check before accessing_qtest
Quoted 2 lines
Maybe remove double // in patch subjects :)
Sure, if it's causing problems. I meant to imply an elided path structure. I guess I can do e.g.
"python/qmp"
"python/machine"
and so on, if it matters. We don't really have a standard or anything. Let me know what you'd prefer and I'll happily change it.
¯\_(ツ)_/¯
Re: [PATCH RFC 01/32] python/qemu: create qemu.lib module
In reply to John SnowQuoted 5 lines · John Snow
move python/qemu/*.py to python/qemu/lib/*.py.
To create a namespace package, the 'qemu' directory itself shouldn't have module files in it. Thus, these files will go under a 'lib' package directory instead.
Hmm..
On the first glance, it looks better to have
from qemu import QEMUMachine
than
from qemu.lib import QEMUMachine
why do we need this extra ".lib" part?
Is it needed only for internal use?
Assume we have installed qemu package. Can we write
from qemu import QEMUMachine
? Or we still need qemu.lib ?
I don't remember any python package, which made me to write "import from package_name.lib ..."
Signature · 2 lines
Best regards, Vladimir
Re: [PATCH RFC 01/32] python/qemu: create qemu.lib module
Quoted 21 lines · Vladimir Sementsov-Ogievskiy
Quoted 5 lines · John Snow
move python/qemu/*.py to python/qemu/lib/*.py.
To create a namespace package, the 'qemu' directory itself shouldn't have module files in it. Thus, these files will go under a 'lib' package directory instead.
Hmm..
On the first glance, it looks better to have
from qemu import QEMUMachine
than
from qemu.lib import QEMUMachinewhy do we need this extra ".lib" part?
Is it needed only for internal use?
Quoted 10 lines
Assume we have installed qemu package. Can we write
from qemu import QEMUMachine
? Or we still need qemu.lib ?
I don't remember any python package, which made me to write "import from package_name.lib ..."
It's a strategy to create "qemu" as a PEP420 namespace package; i.e. "qemu" forms a namespace, but you need a name for the actual package underneath it.
"qemu.lib" is one package, with qmp, qtest, and machine modules. "qemu" isn't really a package in this system, it's just a namespace.
The idea is that this allows us to create a more modular rollout of various python scripts and services as desired instead of monolithically bundling them all inside of a "qemu" package.
It also allows us to fork or split out the sub-packages to separate repos, if we wish. i.e., let's say we create a "qemu.sdk" subpackage, we can eventually fork it off into its own repo with its own installer and so forth. These subpackages can be installed and managed separately.
Re: [PATCH RFC 01/32] python/qemu: create qemu.lib module
Quoted 42 lines · John Snow
Quoted 21 lines · Vladimir Sementsov-Ogievskiy
Quoted 5 lines · John Snow
move python/qemu/*.py to python/qemu/lib/*.py.
To create a namespace package, the 'qemu' directory itself shouldn't have module files in it. Thus, these files will go under a 'lib' package directory instead.
Hmm..
On the first glance, it looks better to have
from qemu import QEMUMachine
than
from qemu.lib import QEMUMachine
why do we need this extra ".lib" part?
Is it needed only for internal use?
Quoted 10 lines
Assume we have installed qemu package. Can we write
from qemu import QEMUMachine
? Or we still need qemu.lib ?
I don't remember any python package, which made me to write "import from package_name.lib ..."
It's a strategy to create "qemu" as a PEP420 namespace package; i.e. "qemu" forms a namespace, but you need a name for the actual package underneath it.
"qemu.lib" is one package, with qmp, qtest, and machine modules. "qemu" isn't really a package in this system, it's just a namespace.
Quoted 9 lines
The idea is that this allows us to create a more modular rollout of various python scripts and services as desired instead of monolithically bundling them all inside of a "qemu" package.
It also allows us to fork or split out the sub-packages to separate repos, if we wish. i.e., let's say we create a "qemu.sdk" subpackage, we can eventually fork it off into its own repo with its own installer and so forth. These subpackages can be installed and managed separately.
Okay, I understand.. No real objections than.
Still, maybe, everything should not go into lib, maybe something like
qemu/vm/ - qmp, QEMUMachine, etc
qemu/qtest/ - qtest
would be more user friendly? But I'm not sure. I just thought that "lib" is too generic.
Signature · 2 lines
Best regards, Vladimir
Re: [PATCH RFC 01/32] python/qemu: create qemu.lib module
Quoted 55 lines · Vladimir Sementsov-Ogievskiy
Quoted 42 lines · John Snow
Quoted 21 lines · Vladimir Sementsov-Ogievskiy
Quoted 6 lines · John Snow
move python/qemu/*.py to python/qemu/lib/*.py.
To create a namespace package, the 'qemu' directory itself shouldn't have module files in it. Thus, these files will go under a 'lib' package
directory instead.Hmm..
On the first glance, it looks better to have
from qemu import QEMUMachine
than
from qemu.lib import QEMUMachinewhy do we need this extra ".lib" part?
Is it needed only for internal use?
Quoted 10 lines
Assume we have installed qemu package. Can we write
from qemu import QEMUMachine
? Or we still need qemu.lib ?
I don't remember any python package, which made me to write "import from package_name.lib ..."
It's a strategy to create "qemu" as a PEP420 namespace package; i.e. "qemu" forms a namespace, but you need a name for the actual package underneath it.
"qemu.lib" is one package, with qmp, qtest, and machine modules. "qemu" isn't really a package in this system, it's just a namespace.
Quoted 9 lines
The idea is that this allows us to create a more modular rollout of various python scripts and services as desired instead of monolithically bundling them all inside of a "qemu" package.
It also allows us to fork or split out the sub-packages to separate repos, if we wish. i.e., let's say we create a "qemu.sdk" subpackage, we can eventually fork it off into its own repo with its own installer and so forth. These subpackages can be installed and managed separately.
Okay, I understand.. No real objections than.
Quoted 8 lines
Still, maybe, everything should not go into lib, maybe something like
qemu/vm/ - qmp, QEMUMachine, etc
qemu/qtest/ - qtestwould be more user friendly? But I'm not sure. I just thought that "lib" is too generic.
lib is a very generic name, I agree.
Splitting accel, qmp and QEMUMachine in one package and keeping qtest in another is fine too. I'm not sure if I like "vm" for the name of that core package, though.
I want to avoid using "qemu/sdk" because I have some plans for trying to generate and package a "real" SDK using that namespace.
"devkit"? "testkit"? "core"? Naming things is always the worst part.
--js
Re: [PATCH RFC 01/32] python/qemu: create qemu.lib module
Quoted 69 lines · John Snow
Quoted 55 lines · Vladimir Sementsov-Ogievskiy
Quoted 42 lines · John Snow
Quoted 21 lines · Vladimir Sementsov-Ogievskiy
Quoted 6 lines · John Snow
move python/qemu/*.py to python/qemu/lib/*.py.
To create a namespace package, the 'qemu' directory itself shouldn't have module files in it. Thus, these files will go under a 'lib' package
directory instead.Hmm..
On the first glance, it looks better to have
from qemu import QEMUMachine
than
from qemu.lib import QEMUMachinewhy do we need this extra ".lib" part?
Is it needed only for internal use?
Quoted 10 lines
Assume we have installed qemu package. Can we write
from qemu import QEMUMachine
? Or we still need qemu.lib ?
I don't remember any python package, which made me to write "import from package_name.lib ..."
It's a strategy to create "qemu" as a PEP420 namespace package; i.e. "qemu" forms a namespace, but you need a name for the actual package underneath it.
"qemu.lib" is one package, with qmp, qtest, and machine modules. "qemu" isn't really a package in this system, it's just a namespace.
Quoted 9 lines
The idea is that this allows us to create a more modular rollout of various python scripts and services as desired instead of monolithically bundling them all inside of a "qemu" package.
It also allows us to fork or split out the sub-packages to separate repos, if we wish. i.e., let's say we create a "qemu.sdk" subpackage, we can eventually fork it off into its own repo with its own installer and so forth. These subpackages can be installed and managed separately.
Okay, I understand.. No real objections than.
Quoted 8 lines
Still, maybe, everything should not go into lib, maybe something like
qemu/vm/ - qmp, QEMUMachine, etc
qemu/qtest/ - qtestwould be more user friendly? But I'm not sure. I just thought that "lib" is too generic.
lib is a very generic name, I agree.
Quoted 9 lines
Splitting accel, qmp and QEMUMachine in one package and keeping qtest in another is fine too. I'm not sure if I like "vm" for the name of that core package, though.
I want to avoid using "qemu/sdk" because I have some plans for trying to generate and package a "real" SDK using that namespace.
"devkit"? "testkit"? "core"? Naming things is always the worst part.
I think, "core" sounds good.
Signature · 2 lines
Best regards, Vladimir
Ping, please give this series a look-over. It looks big, but the changes themselves are actually fairly tiny.
Here's a TOC:
Patch 1 moves files and renames import statements. Patches 2-3 do some basic delinting.
Patches 4-5 do more basic delinting and add flake8/pylintrc cfg Patches 6-10 add setup.py, pipfile, a readme, etc. Patch 11 is a final bit of removing python2 isms.
Patches 12-32 (!) are all mypy typing fixes, and I have broken these out in great care:
Patches 12-30 all fix one mypy issue each with bite-sized refactors each.
Patch 31 adds a mypy configuration to the package.
Patch 32 is a giant patch that has __NO__ runtime side-effects, it JUST adds the remaining mypy annotations.
At the end of the series, you should find that mypy *strict*, flake8, and pylint all pass 100%.
(Note: you MAY need specific versions of these tools to have them pass. The pipfile included will help you target the correct versions.)
--js
Quoted 121 lines · John Snow
Hey, I got lost on my way to the store and I accidentally got 32 patches that convert our python library into something that passes pylint, flake8, and mypy --strict.
...So, a few things:
1. This is just an RFC. The actual design of these libraries really needs adjusted to be more pythonic. In general this means less Optional[T] return types and raising more exceptions. This could be handled later, but we ought to address it before publishing, if we do.
2. We still need to think carefully about how we package this, if we even want to package it, what the license on the top-level package should be, etc.
3. We should consider how to version it. For now, I'm using a lockstep versioning.
4. You can install this package using pip3 or python3
setup.pyto a virtual environment or to your real one. From there, any python code in the QEMU tree that imports these modules will work with no sys.path hacking or custom PYTHONPATH exports.5. You don't have to install it, though. I left all of the usual hacks in place in the rest of the tree so that everything will just keep working exactly as-is for right now. It's just that you COULD install it.
6. Here's a cool trick if you don't know about it yet:
Quoted 2 lines
cd qemu/python/qemu
pip3 install --user -e .This will install the package in "develop" mode, which installs a forwarder package. When you update your source tree, the installed package stays "up to date" with the most recent edits.
Alright, have fun, stay safe!
John Snow (32):
python/qemu: create qemu.lib modulescripts/qmp: Fix shebang and importspython//machine.py: remove bare except python/qemu/lib: delint, add pylintrc python/qemu/lib: delint; add flake8 config python/qemu: formalize as package python/qemu: addREADME.rst
python/qemu: Add Pipfile
python/qemu: add pylint to Pipfile python/qemu: Add flake8 to Pipfile python/qemu/lib: remove Python2 stylesuper()calls python/qemu/lib: fixsocket.makefile()typing python/qemu/lib: Adjust traceback typingpython//qmp.py: use True/False for non/blocking modespython//qmp.py: Define common typespython//qmp.py: re-absorb MonitorResponseErrorpython//qmp.py: Do not return None fromcmd_objpython//qmp.py: add casts to JSON deserializationpython//qmp.py: add QMPProtocolErrorpython//qmp.py: assert sockfile is not Nonepython//machine.py: remove logging configurationpython//machine.py: Fix monitor address typingpython//machine.py: reorder__init__python//machine.py: Don't modify state in_base_args()python//machine.py: Handle None events inevent_waitpython//machine.py: use qmp.commandpython//machine.py: Add_qmpaccess shimpython//machine.py: fix_popenaccesspython//qtest.py: Check before accessing_qtestpython/qemu/lib: make 'args' style arguments immutable python/qemu: add mypy to Pipfile python/qemu/lib: Add mypy type annotationspython/README.rst | 6 + python/qemu/README.rst | 8 + python/Pipfile | 14 + python/Pipfile.lock | 187 +++++++++++++ python/qemu/__init__.py | 11 - python/qemu/lib/.flake8 | 2 + python/qemu/lib/__init__.py | 57 ++++ python/qemu/{ => lib}/accel.py | 17 +- python/qemu/{ => lib}/machine.py | 320 +++++++++++++--------- python/qemu/lib/pylintrc | 58 ++++ python/qemu/{ => lib}/qmp.py | 140 +++++++--- python/qemu/lib/qtest.py | 160 +++++++++++ python/qemu/qtest.py | 119 -------- python/setup.py | 50 ++++ scripts/device-crash-test | 2 +- scripts/qmp/qemu-ga-client | 2 +- scripts/qmp/qmp | 4 +- scripts/qmp/qmp-shell | 2 +- scripts/qmp/qom-fuse | 4 +- scripts/qmp/qom-get | 6 +- scripts/qmp/qom-list | 4 +- scripts/qmp/qom-set | 6 +- scripts/qmp/qom-tree | 6 +- scripts/render_block_graph.py | 5 +- scripts/simplebench/bench_block_job.py | 4 +- tests/acceptance/avocado_qemu/__init__.py | 2 +- tests/acceptance/boot_linux.py | 3 +- tests/acceptance/virtio_check_params.py | 2 +- tests/acceptance/virtio_version.py | 2 +- tests/migration/guestperf/engine.py | 2 +- tests/qemu-iotests/235 | 2 +- tests/qemu-iotests/iotests.py | 2 +- tests/vm/basevm.py | 6 +- 33 files changed, 881 insertions(+), 334 deletions(-) create mode 100644 python/README.rst create mode 100644 python/qemu/README.rst create mode 100644 python/Pipfile create mode 100644 python/Pipfile.lock delete mode 100644 python/qemu/__init__.py create mode 100644 python/qemu/lib/.flake8 create mode 100644 python/qemu/lib/__init__.py rename python/qemu/{ => lib}/accel.py (86%) rename python/qemu/{ => lib}/machine.py (67%) create mode 100644 python/qemu/lib/pylintrc rename python/qemu/{ => lib}/qmp.py (70%) create mode 100644 python/qemu/lib/qtest.py delete mode 100644 python/qemu/qtest.py create mode 100755 python/setup.py
Signature · 1 line
—js
Re: [PATCH RFC 01/32] python/qemu: create qemu.lib module
In reply to Vladimir Sementsov-OgievskiyQuoted 60 lines · Vladimir Sementsov-Ogievskiy
Quoted 59 lines · John Snow
Quoted 56 lines · Vladimir Sementsov-Ogievskiy
Quoted 54 lines · John Snow
Quoted 23 lines · Vladimir Sementsov-Ogievskiy
Quoted 6 lines · John Snow
move python/qemu/*.py to python/qemu/lib/*.py.
To create a namespace package, the 'qemu' directory itself shouldn't have module files in it. Thus, these files will go under a 'lib' package
directory instead.Hmm..
On the first glance, it looks better to have
from qemu import QEMUMachine
than
from qemu.lib import QEMUMachinewhy do we need this extra ".lib" part?
Is it needed only for internal use?
Assume we have installed qemu package. Can we write
Quoted 9 lines
from qemu import QEMUMachine
? Or we still need qemu.lib ?
I don't remember any python package, which made me to write "import from
package_name.lib ..."It's a strategy to create "qemu" as a PEP420 namespace package; i.e. "qemu" forms a namespace, but you need a name for the actual package underneath it.
"qemu.lib" is one package, with qmp, qtest, and machine modules. "qemu" isn't really a package in this system, it's just a namespace.
The idea is that this allows us to create a more modular rollout of various python scripts and services as desired instead of monolithically
bundling them all inside of a "qemu" package.It also allows us to fork or split out the sub-packages to separate repos, if we wish. i.e., let's say we create a "qemu.sdk" subpackage, we
can eventually fork it off into its own repo with its own installer and so forth. These subpackages can be installed and managed separately.
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Okay, I understand.. No real objections than.
Still, maybe, everything should not go into lib, maybe something like
qemu/vm/ - qmp, QEMUMachine, etc
qemu/qtest/ - qtest
I'm not sure this part is relevant now, as we have not good projection of what/who/how this package will be consumed.
I suppose by VM you mean VirtualMachine. I find it confusing. Maybe simply "machine"? We also have 'tools' and 'user-space processes'.
QMP is protocol, common to all. "qemu.core.qmp"?
We also have the gdb(stub) protocol, common to machine(system) & user.
The block layer has its classes, "qemu.block"?
Quoted 11 lines
Quoted 11 lines
Quoted 4 lines
would be more user friendly? But I'm not sure. I just thought that "lib" is too generic.
lib is a very generic name, I agree.
Splitting accel, qmp and QEMUMachine in one package and keeping qtest in another is fine too. I'm not sure if I like "vm" for the name of that core package, though.
Quoted 7 lines
Quoted 5 lines
I want to avoid using "qemu/sdk" because I have some plans for trying to generate and package a "real" SDK using that namespace.
"devkit"? "testkit"? "core"? Naming things is always the worst part.
I think, "core" sounds good.
Agreed.
Re: [PATCH RFC 03/32] python//machine.py: remove bare except
In reply to John SnowQuoted 8 lines · John Snow
Quoted 5 lines · Eric Blake
Quoted 3 lines · John Snow
Catch only the timeout error; if there are other problems, allow the stack trace to be visible.
Quoted 9 lines
Quoted 3 lines
A lot of patches in this series start with "python//" - is that intentional, or should it be a single slash?
Was trying to imply an elided path structure, because "
python/qemu/lib/qmp.py" et al is way too chatty.Feel free to suggest better subject names!
Eliding "qemu/lib/" is OK. I wouldn't not use double //, it sounds like a recurrent typo.
Re: [PATCH RFC 03/32] python//machine.py: remove bare except
In reply to John SnowQuoted 50 lines · John Snow
Catch only the timeout error; if there are other problems, allow the stack trace to be visible.
Signed-off-by John Snow <[email protected]>--- python/qemu/lib/machine.py | 33 +++++++++++++++++++++------------ 1 file changed, 21 insertions(+), 12 deletions(-)python/qemu/lib/machine.py +20
-6@@ -342,7 +342,26 @@ def wait(self): self._load_io_log() self._post_shutdown() - def shutdown(self, has_quit=False): + def _issue_shutdown(self, has_quit: bool = False) -> None: + """ + Shutdown the VM. + """ + if not self.is_running(): + return + + if self._qmp is not None: + if not has_quit: + self._qmp.cmd('quit') + self._qmp.close() + + try: + self._popen.wait(timeout=3) + except subprocess.TimeoutExpired: + self._popen.kill() + + self._popen.wait() + + def shutdown(self, has_quit: bool = False) -> None: """ Terminate the VM and clean up """@@ -353,17 +372,7 @@ def shutdown(self, has_quit=False): self._console_socket.close() self._console_socket = None - if self.is_running(): - if self._qmp: - try: - if not has_quit: - self._qmp.cmd('quit')
python/qemu/lib/machine.py def shutdown(self, has_quit=False): · line 378
Quoted 10 lines
python/qemu/lib/machine.py +1
-6- self._qmp.close() - self._popen.wait(timeout=3) - except: - self._popen.kill() - self._popen.wait() - + self._issue_shutdown(has_quit) self._load_io_log() self._post_shutdown()
Re: [PATCH RFC 01/32] python/qemu: create qemu.lib module
In reply to John SnowQuoted 69 lines · John Snow
Quoted 55 lines · Vladimir Sementsov-Ogievskiy
Quoted 42 lines · John Snow
Quoted 21 lines · Vladimir Sementsov-Ogievskiy
Quoted 6 lines · John Snow
move python/qemu/*.py to python/qemu/lib/*.py.
To create a namespace package, the 'qemu' directory itself shouldn't have module files in it. Thus, these files will go under a 'lib' package
directory instead.Hmm..
On the first glance, it looks better to have
from qemu import QEMUMachine
than
from qemu.lib import QEMUMachinewhy do we need this extra ".lib" part?
Is it needed only for internal use?
Quoted 10 lines
Assume we have installed qemu package. Can we write
from qemu import QEMUMachine
? Or we still need qemu.lib ?
I don't remember any python package, which made me to write "import from package_name.lib ..."
It's a strategy to create "qemu" as a PEP420 namespace package; i.e. "qemu" forms a namespace, but you need a name for the actual package underneath it.
"qemu.lib" is one package, with qmp, qtest, and machine modules. "qemu" isn't really a package in this system, it's just a namespace.
Quoted 9 lines
The idea is that this allows us to create a more modular rollout of various python scripts and services as desired instead of monolithically bundling them all inside of a "qemu" package.
It also allows us to fork or split out the sub-packages to separate repos, if we wish. i.e., let's say we create a "qemu.sdk" subpackage, we can eventually fork it off into its own repo with its own installer and so forth. These subpackages can be installed and managed separately.
Okay, I understand.. No real objections than.
Quoted 8 lines
Still, maybe, everything should not go into lib, maybe something like
qemu/vm/ - qmp, QEMUMachine, etc
qemu/qtest/ - qtestwould be more user friendly? But I'm not sure. I just thought that "lib" is too generic.
lib is a very generic name, I agree.
Quoted 8 lines
Splitting accel, qmp and QEMUMachine in one package and keeping qtest in another is fine too. I'm not sure if I like "vm" for the name of that core package, though.
I want to avoid using "qemu/sdk" because I have some plans for trying to generate and package a "real" SDK using that namespace.
"devkit"? "testkit"? "core"? Naming things is always the worst part.
I'd suggest "machine", as in
from qemu.machine import kvm_available, QEMUMachine
I wouldn't over-think the module naming as it has so little impact on the code usage - it usually only appears in the "import" statement.
Regards,
Daniel
Signature · 3 lines
|: https://berrange.com -o- https://www.flickr.com/photos/dberrange :| |: https://libvirt.org -o- https://fstop138.berrange.com :| |: https://entangle-photo.org -o- https://www.instagram.com/dberrange :|
Re: [PATCH RFC 01/32] python/qemu: create qemu.lib module
Quoted 77 lines · Daniel P. Berrangé
Quoted 76 lines · John Snow
Quoted 55 lines · Vladimir Sementsov-Ogievskiy
Quoted 42 lines · John Snow
Quoted 21 lines · Vladimir Sementsov-Ogievskiy
Quoted 6 lines · John Snow
move python/qemu/*.py to python/qemu/lib/*.py.
To create a namespace package, the 'qemu' directory itself shouldn't have module files in it. Thus, these files will go under a 'lib' package
directory instead.Hmm..
On the first glance, it looks better to have
from qemu import QEMUMachine
than
from qemu.lib import QEMUMachinewhy do we need this extra ".lib" part?
Is it needed only for internal use?
Quoted 10 lines
Assume we have installed qemu package. Can we write
from qemu import QEMUMachine
? Or we still need qemu.lib ?
I don't remember any python package, which made me to write "import from package_name.lib ..."
It's a strategy to create "qemu" as a PEP420 namespace package; i.e. "qemu" forms a namespace, but you need a name for the actual package underneath it.
"qemu.lib" is one package, with qmp, qtest, and machine modules. "qemu" isn't really a package in this system, it's just a namespace.
Quoted 9 lines
The idea is that this allows us to create a more modular rollout of various python scripts and services as desired instead of monolithically bundling them all inside of a "qemu" package.
It also allows us to fork or split out the sub-packages to separate repos, if we wish. i.e., let's say we create a "qemu.sdk" subpackage, we can eventually fork it off into its own repo with its own installer and so forth. These subpackages can be installed and managed separately.
Okay, I understand.. No real objections than.
Quoted 8 lines
Still, maybe, everything should not go into lib, maybe something like
qemu/vm/ - qmp, QEMUMachine, etc
qemu/qtest/ - qtestwould be more user friendly? But I'm not sure. I just thought that "lib" is too generic.
lib is a very generic name, I agree.
Splitting accel, qmp and QEMUMachine in one package and keeping qtest in another is fine too. I'm not sure if I like "vm" for the name of that core package, though.
I want to avoid using "qemu/sdk" because I have some plans for trying to generate and package a "real" SDK using that namespace.
Quoted 8 lines
Quoted 1 line
"devkit"? "testkit"? "core"? Naming things is always the worst part.
I'd suggest "machine", as in
from qemu.machine import
kvm_available, QEMUMachineI wouldn't over-think the module naming as it has so little impact on the code usage - it usually only appears in the "import" statement.
Don't forget linux-user binaries.
Quoted 5 lines
Regards,
Daniel
Re: [PATCH RFC 01/32] python/qemu: create qemu.lib module
Quoted 80 lines · Philippe Mathieu-Daudé
Quoted 79 lines · Daniel P. Berrangé
Quoted 77 lines · John Snow
Quoted 55 lines · Vladimir Sementsov-Ogievskiy
Quoted 42 lines · John Snow
Quoted 21 lines · Vladimir Sementsov-Ogievskiy
Quoted 6 lines · John Snow
move python/qemu/*.py to python/qemu/lib/*.py.
To create a namespace package, the 'qemu' directory itself shouldn't have module files in it. Thus, these files will go under a 'lib' package
directory instead.Hmm..
On the first glance, it looks better to have
from qemu import QEMUMachine
than
from qemu.lib import QEMUMachinewhy do we need this extra ".lib" part?
Is it needed only for internal use?
Quoted 10 lines
Assume we have installed qemu package. Can we write
from qemu import QEMUMachine
? Or we still need qemu.lib ?
I don't remember any python package, which made me to write "import from package_name.lib ..."
It's a strategy to create "qemu" as a PEP420 namespace package; i.e. "qemu" forms a namespace, but you need a name for the actual package underneath it.
"qemu.lib" is one package, with qmp, qtest, and machine modules. "qemu" isn't really a package in this system, it's just a namespace.
Quoted 9 lines
The idea is that this allows us to create a more modular rollout of various python scripts and services as desired instead of monolithically bundling them all inside of a "qemu" package.
It also allows us to fork or split out the sub-packages to separate repos, if we wish. i.e., let's say we create a "qemu.sdk" subpackage, we can eventually fork it off into its own repo with its own installer and so forth. These subpackages can be installed and managed separately.
Okay, I understand.. No real objections than.
Quoted 8 lines
Still, maybe, everything should not go into lib, maybe something like
qemu/vm/ - qmp, QEMUMachine, etc
qemu/qtest/ - qtestwould be more user friendly? But I'm not sure. I just thought that "lib" is too generic.
lib is a very generic name, I agree.
Splitting accel, qmp and QEMUMachine in one package and keeping qtest in another is fine too. I'm not sure if I like "vm" for the name of that core package, though.
I want to avoid using "qemu/sdk" because I have some plans for trying to generate and package a "real" SDK using that namespace.
"devkit"? "testkit"? "core"? Naming things is always the worst part.
Quoted 8 lines
Quoted 6 lines
I'd suggest "machine", as in
from qemu.machine import
kvm_available, QEMUMachineI wouldn't over-think the module naming as it has so little impact on the code usage - it usually only appears in the "import" statement.
Don't forget linux-user binaries.
That's why I suggested ".machine", as all the APIs there currently are focused on the machine emulators, and the linx-user binaries share essentially nothing in common with softmmu binaries in terms of control APIs / CLI config. We can add a "qemu.user" package later if we have stuff related to that to expose
Regards,
Daniel
Signature · 3 lines
|: https://berrange.com -o- https://www.flickr.com/photos/dberrange :| |: https://libvirt.org -o- https://fstop138.berrange.com :| |: https://entangle-photo.org -o- https://www.instagram.com/dberrange :|
Re: [PATCH RFC 02/32] scripts/qmp: Fix shebang and imports
In reply to John SnowQuoted 8 lines · John Snow
There's more wrong with these scripts; They are in various stages of disrepair. That's beyond the scope of this current patchset.
This just mechanically corrects the imports and the shebangs, as part of ensuring that the python/qemu/lib refactoring didn't break anything needlessly.
Signed-off-by John Snow <[email protected]>
Whoops, I address this in a patch series I'm working on, too. I'll keep my patch in there for now until this one makes it in, or vice-versa.
https://www.mail-archive.com/[email protected]/msg66801.html
Quoted 122 lines
--- scripts/qmp/qmp | 4 +++- scripts/qmp/qom-fuse | 4 +++- scripts/qmp/qom-get | 6 ++++-- scripts/qmp/qom-list | 4 +++- scripts/qmp/qom-set | 6 ++++-- scripts/qmp/qom-tree | 6 ++++-- 6 files changed, 21 insertions(+), 9 deletions(-)scripts/qmp/qmp +3
-1@@ -11,7 +11,9 @@ # See the COPYING file in the top-level directory. import sys, os -from qmp import QEMUMonitorProtocol + +sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'python')) +from qemu.lib import QEMUMonitorProtocol def print_response(rsp, prefix=[]): if type(rsp) == list:scripts/qmp/qom-fuse +3
-1@@ -15,7 +15,9 @@ import fuse, stat from fuse import Fuse import os, posix from errno import * -from qmp import QEMUMonitorProtocol + +sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'python')) +from qemu.lib import QEMUMonitorProtocol fuse.fuse_python_api = (0, 2)scripts/qmp/qom-get +4
-2@@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python3 ## # QEMU Object Model test tools #@@ -13,7 +13,9 @@ import sys import os -from qmp import QEMUMonitorProtocol + +sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'python')) +from qemu.lib import QEMUMonitorProtocol cmd, args = sys.argv[0], sys.argv[1:] socket_path = Nonescripts/qmp/qom-list +3
-1@@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python3 ## # QEMU Object Model test tools #@@ -13,6 +13,8 @@ import sys import os + +sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'python')) from qmp import QEMUMonitorProtocol cmd, args = sys.argv[0], sys.argv[1:]scripts/qmp/qom-set +4
-2@@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python3 ## # QEMU Object Model test tools #@@ -13,7 +13,9 @@ import sys import os -from qmp import QEMUMonitorProtocol + +sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'python')) +from qemu.lib import QEMUMonitorProtocol cmd, args = sys.argv[0], sys.argv[1:] socket_path = Nonescripts/qmp/qom-tree +4
-2@@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python3 ## # QEMU Object Model test tools #@@ -15,7 +15,9 @@ import sys import os -from qmp import QEMUMonitorProtocol + +sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'python')) +from qemu.lib import QEMUMonitorProtocol cmd, args = sys.argv[0], sys.argv[1:] socket_path = None
Re: [PATCH RFC 04/32] python/qemu/lib: delint, add pylintrc
In reply to John SnowQuoted 59 lines · John Snow
Bring our these files up to speed with pylint 2.5.0. Add a pylintrc file to formalize which pylint subset we are targeting.
The similarity ignore is there to suppress similarity reports across imports, which for typing constants, are going to trigger this report erroneously.
Signed-off-by John Snow <[email protected]>--- python/qemu/lib/machine.py | 6 ++-- python/qemu/lib/pylintrc | 58 ++++++++++++++++++++++++++++++++++++++ python/qemu/lib/qtest.py | 42 +++++++++++++++++---------- 3 files changed, 88 insertions(+), 18 deletions(-) create mode 100644 python/qemu/lib/pylintrcpython/qemu/lib/machine.py +3
-3@@ -58,7 +58,7 @@ def __init__(self, reply): self.reply = reply -class QEMUMachine(object): +class QEMUMachine: """ A QEMU VM@@ -242,7 +242,7 @@ def _base_args(self): 'chardev=mon,mode=control']) if self._machine is not None: args.extend(['-machine', self._machine]) - for i in range(self._console_index): + for _ in range(self._console_index): args.extend(['-serial', 'null']) if self._console_set: self._console_address = os.path.join(self._sock_dir,@@ -383,7 +383,7 @@ def shutdown(self, has_quit: bool = False) -> None: command = ' '.join(self._qemu_full_args) else: command = '' - LOG.warning(msg, -exitcode, command) + LOG.warning(msg, -int(exitcode), command) self._launched = Falsepython/qemu/lib/pylintrc new file +6
-0@@ -0,0 +1,58 @@ +[MASTER] + +[MESSAGES CONTROL] + +# Disable the message, report, category or checker with the given id(s). You +# can either give multiple identifiers separated by comma (,) or put this
python/qemu/lib/pylintrc line 16
Quoted 10 lines
python/qemu/lib/pylintrc new file +10
-0+# option multiple times (only on the command line, not in the configuration +# file where it should appear only once). You can also use "--disable=all" to +# disable everything first and then reenable specific checks. For example, if +# you want to run only the similarities checker, you can use "--disable=all +# --enable=similarities". If you want to run only the classes checker, but have +# no Warning level messages displayed, use "--disable=all --enable=classes +# --disable=W". +disable=too-many-arguments, + too-many-instance-attributes, + too-many-public-methods,
Quoted 116 lines
python/qemu/lib/pylintrc new file +42
-0+ +[REPORTS] + +[REFACTORING] + +[MISCELLANEOUS] + +[LOGGING] + +[BASIC] + +# Good variable names which should always be accepted, separated by a comma. +good-names=i, + j, + k, + ex, + Run, + _, + fd, + +[VARIABLES] + +[STRING] + +[SPELLING] + +[FORMAT] + +[SIMILARITIES] + +# Ignore imports when computing similarities. +ignore-imports=yes + +[TYPECHECK] + +[CLASSES] + +[IMPORTS] + +[DESIGN] + +[EXCEPTIONS]python/qemu/lib/qtest.py +27
-15@@ -1,5 +1,11 @@ -# QEMU qtest library -# +""" +QEMU qtest library + +qtest offers the QEMUQtestProtocol and QEMUQTestMachine classes, which +offer a connection to QEMU's qtest protocol socket, and a qtest-enabled +subclass of QEMUMachine, respectively. +""" + # Copyright (C) 2015 Red Hat Inc. # # Authors:@@ -17,19 +23,21 @@ from .machine import QEMUMachine -class QEMUQtestProtocol(object): +class QEMUQtestProtocol: + """ + QEMUQtestProtocol implements a connection to a qtest socket. + + :param address: QEMU address, can be either a unix socket path (string) + or a tuple in the form ( address, port ) for a TCP + connection + :param server: server mode, listens on the socket (bool) + :raise socket.error: on socket connection errors + + .. note:: + No conection is estabalished by __init__(), this is done + by the connect() or accept() methods. + """ def __init__(self, address, server=False): - """ - Create a QEMUQtestProtocol object. - - @param address: QEMU address, can be either a unix socket path (string) - or a tuple in the form ( address, port ) for a TCP - connection - @param server: server mode, listens on the socket (bool) - @raise socket.error on socket connection errors - @note No connection is established, this is done by the connect() or - accept() methods - """ self._address = address self._sock = self._get_sock() self._sockfile = None@@ -73,15 +81,19 @@ def cmd(self, qtest_cmd): return resp def close(self): + """Close this socket.""" self._sock.close() self._sockfile.close() def settimeout(self, timeout): + """Set a timeout, in seconds.""" self._sock.settimeout(timeout) class QEMUQtestMachine(QEMUMachine): - '''A QEMU VM''' + """ + A QEMU VM, with a qtest socket available. + """ def __init__(self, binary, args=None, name=None, test_dir="/var/tmp", socket_scm_helper=None, sock_dir=None):
Re: [PATCH RFC 05/32] python/qemu/lib: delint; add flake8 config
In reply to John SnowQuoted 109 lines · John Snow
Mostly, ignore the "no bare except" rule, because flake8 is not contextual and cannot determine if we re-raise. Pylint can, though, so always prefer pylint for that.
Signed-off-by John Snow <[email protected]>--- python/qemu/lib/.flake8 | 2 ++ python/qemu/lib/accel.py | 9 ++++++--- python/qemu/lib/machine.py | 13 +++++++++---- python/qemu/lib/qmp.py | 4 ++-- 4 files changed, 19 insertions(+), 9 deletions(-) create mode 100644 python/qemu/lib/.flake8python/qemu/lib/.flake8 new file +2
-0@@ -0,0 +1,2 @@ +[flake8] +extend-ignore = E722 # Pylint handles this, but smarter.python/qemu/lib/accel.py +6
-3@@ -23,11 +23,12 @@ # Mapping host architecture to any additional architectures it can # support which often includes its 32 bit cousin. ADDITIONAL_ARCHES = { - "x86_64" : "i386", - "aarch64" : "armhf", - "ppc64le" : "ppc64", + "x86_64": "i386", + "aarch64": "armhf", + "ppc64le": "ppc64", } + def list_accel(qemu_bin): """ List accelerators enabled in the QEMU binary.@@ -47,6 +48,7 @@ def list_accel(qemu_bin): # Skip the first line which is the header. return [acc.strip() for acc in out.splitlines()[1:]] + def kvm_available(target_arch=None, qemu_bin=None): """ Check if KVM is available using the following heuristic:@@ -69,6 +71,7 @@ def kvm_available(target_arch=None, qemu_bin=None): return False return True + def tcg_available(qemu_bin): """ Check if TCG is available.python/qemu/lib/machine.py +9
-4@@ -29,6 +29,7 @@ LOG = logging.getLogger(__name__) + class QEMUMachineError(Exception): """ Exception called when an error in QEMUMachine happens.@@ -62,7 +63,8 @@ class QEMUMachine: """ A QEMU VM - Use this object as a context manager to ensure the QEMU process terminates:: + Use this object as a context manager to ensure + the QEMU process terminates:: with VM(binary) as vm:@@ -188,8 +190,10 @@ def send_fd_scm(self, fd=None, file_path=None): fd_param.append(str(fd)) devnull = open(os.path.devnull, 'rb') - proc = subprocess.Popen(fd_param, stdin=devnull, stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, close_fds=False) + proc = subprocess.Popen( + fd_param, stdin=devnull, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, close_fds=False + ) output = proc.communicate()[0] if output: LOG.debug(output)@@ -491,7 +495,8 @@ def event_wait(self, name, timeout=60.0, match=None): def events_wait(self, events, timeout=60.0): """ - events_wait waits for and returns a named event from QMP with a timeout. + events_wait waits for and returns a named event + from QMP with a timeout. events: a sequence of (name, match_criteria) tuples. The match criteria are optional and may be None.python/qemu/lib/qmp.py +0
-0
python/qemu/lib/qmp.py def accept(self, timeout=15.0): · line 175
Quoted 10 lines
python/qemu/lib/qmp.py +2
-2@@ -168,8 +168,8 @@ def accept(self, timeout=15.0): @param timeout: timeout in seconds (nonnegative float number, or None). The value passed will set the behavior of the - underneath QMP socket as described in [1]. Default value - is set to 15.0. + underneath QMP socket as described in [1]. + Default value is set to 15.0. @return QMP greeting dict @raise OSError on socket connection errors @raise QMPConnectError if the greeting is not received
Re: [PATCH RFC 18/32] python//qmp.py: add casts to JSON deserialization
In reply to John SnowQuoted 46 lines · John Snow
mypy and python type hints are not powerful enough to properly describe JSON messages in Python 3.6. The best we can do, generally, is describe them as Dict[str, Any].
Add casts to coerce this type for static analysis; but do NOT enforce this type at runtime in any way.
Note: Python 3.8 adds a TypedDict construct which allows for the description of more arbitrary Dictionary shapes. There is a third-party module, "Pydantic", which is compatible with 3.6 that can be used instead of the JSON library that parses JSON messages to fully-typed Python objects, and may be preferable in some cases.
(That is well beyond the scope of this commit or series.)
Signed-off-by John Snow <[email protected]>--- python/qemu/lib/qmp.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-)python/qemu/lib/qmp.py +5
-1@@ -13,6 +13,7 @@ import logging from typing import ( Any, + cast, Dict, Optional, TextIO,@@ -129,7 +130,10 @@ def __json_read(self, only_event=False): data = self.__sockfile.readline() if not data: return None - resp = json.loads(data) + # By definition, any JSON received from QMP is a QMPMessage, + # and we are asserting only at static analysis time that it + # has a particular shape. + resp = cast(QMPMessage, json.loads(data)) if 'event' in resp: self.logger.debug("<<< %s", resp) self.__events.append(resp)
python/qemu/lib/qmp.py def command(self, cmd, **kwds): · line 271
Quoted 8 lines
python/qemu/lib/qmp.py +1
-1@@ -261,7 +265,7 @@ def command(self, cmd, **kwds): ret = self.cmd(cmd, kwds) if 'error' in ret: raise QMPResponseError(ret) - return ret['return'] + return cast(QMPReturnValue, ret['return']) def pull_event(self, wait=False): """
Re: [PATCH RFC 06/32] python/qemu: formalize as package
In reply to John SnowQuoted 3 lines · John Snow
NB: I am choosing Python 3.6 here. Although our minimum requirement is 3.5, this code is used only by iotests (so far) under which we have been using a minimum version of 3.6.
Preferably with another Ack-by from someone familiar with Python iotests:
Quoted 81 lines
3.6 is being preferred here for variable type hint capability, which enables us to use mypy for this package.
Signed-off-by John Snow <[email protected]>--- python/README.rst | 6 ++++++ python/setup.py | 50 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+) create mode 100644 python/README.rst create mode 100755 python/setup.pypython/README.rst new file +6
-0@@ -0,0 +1,6 @@ +QEMU Python Tooling +------------------- + +This package provides QEMU tooling used by the QEMU project to build, +configure, and test QEMU. It is not a fully-fledged SDK and it is subject +to change at any time.python/setup.py new file +50
-0@@ -0,0 +1,50 @@ +#!/usr/bin/env3 python +""" +QEMU tooling installer script +Copyright (c) 2020 John Snow for Red Hat, Inc. +""" + +import setuptools + +def main(): + """ + QEMU tooling installer + """ + + kwargs = { + 'name': 'qemu', + 'use_scm_version': { + 'root': '..', + 'relative_to': __file__, + }, + 'maintainer': 'QEMU Developer Team', + 'maintainer_email': '[email protected]', + 'url': 'https://www.qemu.org/', + 'download_url': 'https://www.qemu.org/download/', + 'packages': setuptools.find_namespace_packages(), + 'description': 'QEMU Python Build, Debug and SDK tooling.', + 'classifiers': [ + 'Development Status :: 5 - Production/Stable', + 'License :: OSI Approved :: GNU General Public License v2 (GPLv2)', + 'Natural Language :: English', + 'Operating System :: OS Independent', + ], + 'platforms': [], + 'keywords': [], + 'setup_requires': [ + 'setuptools', + 'setuptools_scm', + ], + 'install_requires': [ + ], + 'python_requires': '>=3.6', + 'long_description_content_type': 'text/x-rst', + } + + with open("README.rst", "r") as fh: + kwargs['long_description'] = fh.read() + + setuptools.setup(**kwargs) + +if __name__ == '__main__': + main()
Re: [PATCH RFC 13/32] python/qemu/lib: Adjust traceback typing
In reply to John SnowQuoted 65 lines · John Snow
mypy considers it incorrect to use
boolto statically return false, because it will assume that it could conceivably return True, and gives different analysis in that case. Use a None return to achieve the same effect, but make mypy happy.Note: Pylint considers function signatures as code that might trip the duplicate-code checker. I'd rather not disable this as it does not trigger often in practice, so I'm disabling it as a one-off and filed a change request; see https://github.com/PyCQA/pylint/issues/3619
Signed-off-by John Snow <[email protected]>--- python/qemu/lib/machine.py | 8 ++++++-- python/qemu/lib/qmp.py | 10 ++++++++-- 2 files changed, 14 insertions(+), 4 deletions(-)python/qemu/lib/machine.py +6
-2@@ -24,6 +24,8 @@ import shutil import socket import tempfile +from typing import Optional, Type +from types import TracebackType from . import qmp@@ -127,9 +129,11 @@ def __init__(self, binary, args=None, wrapper=None, name=None, def __enter__(self): return self - def __exit__(self, exc_type, exc_val, exc_tb): + def __exit__(self, + exc_type: Optional[Type[BaseException]], + exc_val: Optional[BaseException], + exc_tb: Optional[TracebackType]) -> None: self.shutdown() - return False def add_monitor_null(self): """python/qemu/lib/qmp.py +4
-1@@ -14,7 +14,9 @@ from typing import ( Optional, TextIO, + Type, ) +from types import TracebackType class QMPError(Exception):@@ -146,10 +148,14 @@ def __enter__(self): # Implement context manager enter function. return self - def __exit__(self, exc_type, exc_value, exc_traceback): + def __exit__(self, + # pylint: disable=duplicate-code
python/qemu/lib/qmp.py def __enter__(self): · line 161
Quoted 10 lines
python/qemu/lib/qmp.py +4
-1+ # see https://github.com/PyCQA/pylint/issues/3619 + exc_type: Optional[Type[BaseException]], + exc_val: Optional[BaseException], + exc_tb: Optional[TracebackType]) -> None: # Implement context manager exit function. self.close() - return False def connect(self, negotiate=True): """
Re: [PATCH RFC 20/32] python//qmp.py: assert sockfile is not None
In reply to John SnowQuoted 17 lines · John Snow
In truth, if you don't do this, you'll just get a TypeError exception. Now, you'll get an AssertionError.
Is this tangibly better? No.
Does mypy complain less? Yes.Signed-off-by John Snow <[email protected]>--- python/qemu/lib/qmp.py | 1 + 1 file changed, 1 insertion(+)python/qemu/lib/qmp.py +0
-0
python/qemu/lib/qmp.py def __negotiate_capabilities(self): · line 138
Quoted 7 lines
python/qemu/lib/qmp.py +1
-0@@ -132,6 +132,7 @@ def __negotiate_capabilities(self): raise QMPCapabilitiesError def __json_read(self, only_event=False): + assert self.__sockfile is not None while True: data = self.__sockfile.readline() if not data:
Philippe Mathieu-Daudé
Reviewed-byRe: [PATCH RFC 20/32] python//qmp.py: assert sockfile is not None
Quoted 19 lines · Philippe Mathieu-Daudé
Quoted 18 lines · John Snow
In truth, if you don't do this, you'll just get a TypeError exception. Now, you'll get an AssertionError.
Is this tangibly better? No.
Does mypy complain less? Yes.Signed-off-by John Snow <[email protected]>--- python/qemu/lib/qmp.py | 1 + 1 file changed, 1 insertion(+)python/qemu/lib/qmp.py +0
-0@@ -132,6 +132,7 @@ def __negotiate_capabilities(self): raise QMPCapabilitiesError
Quoted 8 lines
python/qemu/lib/qmp.py def __negotiate_capabilities(self): · line 138
Quoted 6 lines
python/qemu/lib/qmp.py +1
-0def __json_read(self, only_event=False): + assert self.__sockfile is not None while True: data = self.__sockfile.readline() if not data:
I meant:
Re: [PATCH RFC 01/32] python/qemu: create qemu.lib module
In reply to Daniel P. BerrangéQuoted 88 lines · Daniel P. Berrangé
Quoted 87 lines · Philippe Mathieu-Daudé
Quoted 85 lines · Daniel P. Berrangé
Quoted 69 lines · John Snow
Quoted 55 lines · Vladimir Sementsov-Ogievskiy
Quoted 42 lines · John Snow
Quoted 21 lines · Vladimir Sementsov-Ogievskiy
Quoted 6 lines · John Snow
move python/qemu/*.py to python/qemu/lib/*.py.
To create a namespace package, the 'qemu' directory itself shouldn't have module files in it. Thus, these files will go under a 'lib' package
directory instead.Hmm..
On the first glance, it looks better to have
from qemu import QEMUMachine
than
from qemu.lib import QEMUMachinewhy do we need this extra ".lib" part?
Is it needed only for internal use?
Quoted 10 lines
Assume we have installed qemu package. Can we write
from qemu import QEMUMachine
? Or we still need qemu.lib ?
I don't remember any python package, which made me to write "import from package_name.lib ..."
It's a strategy to create "qemu" as a PEP420 namespace package; i.e. "qemu" forms a namespace, but you need a name for the actual package underneath it.
"qemu.lib" is one package, with qmp, qtest, and machine modules. "qemu" isn't really a package in this system, it's just a namespace.
Quoted 9 lines
The idea is that this allows us to create a more modular rollout of various python scripts and services as desired instead of monolithically bundling them all inside of a "qemu" package.
It also allows us to fork or split out the sub-packages to separate repos, if we wish. i.e., let's say we create a "qemu.sdk" subpackage, we can eventually fork it off into its own repo with its own installer and so forth. These subpackages can be installed and managed separately.
Okay, I understand.. No real objections than.
Quoted 8 lines
Still, maybe, everything should not go into lib, maybe something like
qemu/vm/ - qmp, QEMUMachine, etc
qemu/qtest/ - qtestwould be more user friendly? But I'm not sure. I just thought that "lib" is too generic.
lib is a very generic name, I agree.
Quoted 8 lines
Splitting accel, qmp and QEMUMachine in one package and keeping qtest in another is fine too. I'm not sure if I like "vm" for the name of that core package, though.
I want to avoid using "qemu/sdk" because I have some plans for trying to generate and package a "real" SDK using that namespace.
"devkit"? "testkit"? "core"? Naming things is always the worst part.
I'd suggest "machine", as in
from qemu.machine import
kvm_available, QEMUMachineI wouldn't over-think the module naming as it has so little impact on the code usage - it usually only appears in the "import" statement.
Quoted 8 lines
Quoted 1 line
Don't forget linux-user binaries.
That's why I suggested ".machine", as all the APIs there currently are focused on the machine emulators, and the linx-user binaries share essentially nothing in common with softmmu binaries in terms of control APIs / CLI config. We can add a "qemu.user" package later if we have stuff related to that to expose
I'm re-ordering the series to front-load the linting and type-checking; and the package organization will now come second, in a separate series.
Module naming isn't a big deal right now, but if we package it and upload to PyPI it will be something we shouldn't change frivolously.
Daniel, are you suggesting we split it like this? --
- qemu.machine (machine.py, qtest.py, accel.py?)
- qemu.monitor (qmp.py)
the only one that's really truly weird is accel.py?, which is just kind of a misc function. I guess it can go in qemu.machine for now and if we adopt a qemu.user later, we can pull it out into a common area if we need to.
--js
Re: [PATCH RFC 01/32] python/qemu: create qemu.lib module
Quoted 102 lines · John Snow
Quoted 88 lines · Daniel P. Berrangé
Quoted 87 lines · Philippe Mathieu-Daudé
Quoted 85 lines · Daniel P. Berrangé
Quoted 69 lines · John Snow
Quoted 55 lines · Vladimir Sementsov-Ogievskiy
Quoted 42 lines · John Snow
Quoted 21 lines · Vladimir Sementsov-Ogievskiy
Quoted 6 lines · John Snow
move python/qemu/*.py to python/qemu/lib/*.py.
To create a namespace package, the 'qemu' directory itself shouldn't have module files in it. Thus, these files will go under a 'lib' package
directory instead.Hmm..
On the first glance, it looks better to have
from qemu import QEMUMachine
than
from qemu.lib import QEMUMachinewhy do we need this extra ".lib" part?
Is it needed only for internal use?
Quoted 10 lines
Assume we have installed qemu package. Can we write
from qemu import QEMUMachine
? Or we still need qemu.lib ?
I don't remember any python package, which made me to write "import from package_name.lib ..."
It's a strategy to create "qemu" as a PEP420 namespace package; i.e. "qemu" forms a namespace, but you need a name for the actual package underneath it.
"qemu.lib" is one package, with qmp, qtest, and machine modules. "qemu" isn't really a package in this system, it's just a namespace.
Quoted 9 lines
The idea is that this allows us to create a more modular rollout of various python scripts and services as desired instead of monolithically bundling them all inside of a "qemu" package.
It also allows us to fork or split out the sub-packages to separate repos, if we wish. i.e., let's say we create a "qemu.sdk" subpackage, we can eventually fork it off into its own repo with its own installer and so forth. These subpackages can be installed and managed separately.
Okay, I understand.. No real objections than.
Quoted 8 lines
Still, maybe, everything should not go into lib, maybe something like
qemu/vm/ - qmp, QEMUMachine, etc
qemu/qtest/ - qtestwould be more user friendly? But I'm not sure. I just thought that "lib" is too generic.
lib is a very generic name, I agree.
Quoted 8 lines
Splitting accel, qmp and QEMUMachine in one package and keeping qtest in another is fine too. I'm not sure if I like "vm" for the name of that core package, though.
I want to avoid using "qemu/sdk" because I have some plans for trying to generate and package a "real" SDK using that namespace.
"devkit"? "testkit"? "core"? Naming things is always the worst part.
I'd suggest "machine", as in
from qemu.machine import
kvm_available, QEMUMachineI wouldn't over-think the module naming as it has so little impact on the code usage - it usually only appears in the "import" statement.
Quoted 8 lines
Quoted 1 line
Don't forget linux-user binaries.
That's why I suggested ".machine", as all the APIs there currently are focused on the machine emulators, and the linx-user binaries share essentially nothing in common with softmmu binaries in terms of control APIs / CLI config. We can add a "qemu.user" package later if we have stuff related to that to expose
I'm re-ordering the series to front-load the linting and type-checking; and the package organization will now come second, in a separate series.
Quoted 7 lines
Module naming isn't a big deal right now, but if we package it and upload to PyPI it will be something we shouldn't change frivolously.
Daniel, are you suggesting we split it like this? --
- qemu.machine (
machine.py,qtest.py,accel.py?)
- qemu.monitor (qmp.py)
I was actually suggesting everything in qemu.machine, but I guess qemu.monitor makes sense, given that this is an interface both for controlling QEMU and the guest agent.
Quoted 4 lines
the only one that's really truly weird is
accel.py?, which is just kind of a misc function. I guess it can go inqemu.machinefor now and if we adopt aqemu.userlater, we can pull it out into a common area if we need to.
Regards,
Daniel
Signature · 3 lines
|: https://berrange.com -o- https://www.flickr.com/photos/dberrange :| |: https://libvirt.org -o- https://fstop138.berrange.com :| |: https://entangle-photo.org -o- https://www.instagram.com/dberrange :|
Re: [PATCH RFC 26/32] python//machine.py: use qmp.command
In reply to John Snow[...]
Quoted 9 lines
python/qemu/lib/machine.py +2
-4- def qmp(self, cmd, conv_keys=True, **args): - """ - Invoke a QMP command and return the response dict - """ + @classmethod + def _qmp_args(cls, _conv_keys: bool = True, **args: Any) -> Dict[str, Any]: qmp_args = dict() for key, value in args.items():
python/qemu/lib/machine.py def set_qmp_monitor(self, enabled=True): · line 433
Quoted 10 lines
python/qemu/lib/machine.py +5
-1- if conv_keys: + if _conv_keys: qmp_args[key.replace('_', '-')] = value else: qmp_args[key] = value + return qmp_args + def qmp(self, cmd: str, + conv_keys: bool = True, + **args: Any) -> QMPMessage:
This creates an interesting problem with iotests 297:
-Success: no issues found in 1 source file +iotests.py:563: error: Argument 2 to "qmp" of "QEMUMachine" has incompatible type "**Dict[str, str]"; expected "bool" +Found 1 error in 1 file (checked 1 source file)
def hmp(self, command_line: str, use_log: bool = False) -> QMPResponse:
cmd = 'human-monitor-command'
kwargs = {'command-line': command_line}
if use_log:
return self.qmp_log(cmd, **kwargs)
else:
return self.qmp(cmd, **kwargs)
It seems like mypy is unable to understand that we are passing keyword arguments, and instead believes we're passing something to the conv_keys parameter.
(Is this a bug...?)
Even amending the function signature to indicate that conv_keys should only ever appear as a keyword argument doesn't seem to help.
I'll have to think about a nice way to fix this; removing conv_keys out of the argument namespace seems like the best approach.
qmp(cmd, foo=bar, hello=world)qmp(cmd, **conv_keys(foo=bar, hello=world))
...but now this function looks really annoying to call.
Uh, I'll play around with this, but let me know if you have any cool ideas.
--js
Philippe Mathieu-Daudé
AppliedRe: [PATCH RFC 05/32] python/qemu/lib: delint; add flake8 config
In reply to John SnowQuoted 4 lines · John Snow
Mostly, ignore the "no bare except" rule, because flake8 is not contextual and cannot determine if we re-raise. Pylint can, though, so always prefer pylint for that.
Quoted 8 lines
Signed-off-by John Snow <[email protected]>--- python/qemu/lib/.flake8 | 2 ++ python/qemu/lib/accel.py | 9 ++++++--- python/qemu/lib/machine.py | 13 +++++++++---- python/qemu/lib/qmp.py | 4 ++-- 4 files changed, 19 insertions(+), 9 deletions(-) create mode 100644 python/qemu/lib/.flake8
Thanks, applied to my python-next tree:
https://gitlab.com/philmd/qemu/commits/python-next
Philippe Mathieu-Daudé
AppliedRe: [PATCH RFC 11/32] python/qemu/lib: remove Python2 style super() calls
In reply to John SnowQuoted 7 lines · John Snow
Use the Python3 style instead.
Signed-off-by John Snow <[email protected]>--- python/qemu/lib/machine.py | 2 +- python/qemu/lib/qtest.py | 15 +++++++-------- 2 files changed, 8 insertions(+), 9 deletions(-)
Thanks, applied to my python-next tree:
https://gitlab.com/philmd/qemu/commits/python-next
Re: [PATCH RFC 12/32] python/qemu/lib: fix socket.makefile() typing
In reply to John SnowQuoted 103 lines · John Snow
Note:
A bug in typeshed (https://github.com/python/typeshed/issues/3977)
misinterprets the type ofmakefile(). Work around this by explicitly stating that we are opening a text-mode file.Signed-off-by John Snow <[email protected]>--- python/qemu/lib/qmp.py | 10 +++++++--- python/qemu/lib/qtest.py | 12 ++++++++---- 2 files changed, 15 insertions(+), 7 deletions(-)python/qemu/lib/qmp.py +7
-3@@ -11,6 +11,10 @@ import errno import socket import logging +from typing import ( + Optional, + TextIO, +) class QMPError(Exception):@@ -61,7 +65,7 @@ def __init__(self, address, server=False, nickname=None): self.__events = [] self.__address = address self.__sock = self.__get_sock() - self.__sockfile = None + self.__sockfile: Optional[TextIO] = None self._nickname = nickname if self._nickname: self.logger = logging.getLogger('QMP').getChild(self._nickname)@@ -157,7 +161,7 @@ def connect(self, negotiate=True): @raise QMPCapabilitiesError if fails to negotiate capabilities """ self.__sock.connect(self.__address) - self.__sockfile = self.__sock.makefile() + self.__sockfile = self.__sock.makefile(mode='r') if negotiate: return self.__negotiate_capabilities() return None@@ -180,7 +184,7 @@ def accept(self, timeout=15.0): """ self.__sock.settimeout(timeout) self.__sock, _ = self.__sock.accept() - self.__sockfile = self.__sock.makefile() + self.__sockfile = self.__sock.makefile(mode='r') return self.__negotiate_capabilities() def cmd_obj(self, qmp_cmd):python/qemu/lib/qtest.py +5
-3@@ -19,6 +19,7 @@ import socket import os +from typing import Optional, TextIO from .machine import QEMUMachine@@ -40,7 +41,7 @@ class QEMUQtestProtocol: def __init__(self, address, server=False): self._address = address self._sock = self._get_sock() - self._sockfile = None + self._sockfile: Optional[TextIO] = None if server: self._sock.bind(self._address) self._sock.listen(1)@@ -59,7 +60,7 @@ def connect(self): @raise socket.error on socket connection errors """ self._sock.connect(self._address) - self._sockfile = self._sock.makefile() + self._sockfile = self._sock.makefile(mode='r') def accept(self): """@@ -68,7 +69,7 @@ def accept(self): @raise socket.error on socket connection errors """ self._sock, _ = self._sock.accept() - self._sockfile = self._sock.makefile() + self._sockfile = self._sock.makefile(mode='r') def cmd(self, qtest_cmd): """@@ -76,6 +77,7 @@ def cmd(self, qtest_cmd): @param qtest_cmd: qtest command text to be sent """ + assert self._sockfile is not None self._sock.sendall((qtest_cmd + "\n").encode('utf-8')) resp = self._sockfile.readline() return resp
python/qemu/lib/qtest.py def cmd(self, qtest_cmd): · line 93
Quoted 10 lines
python/qemu/lib/qtest.py +3
-1@@ -83,7 +85,9 @@ def cmd(self, qtest_cmd): def close(self): """Close this socket.""" self._sock.close() - self._sockfile.close() + if self._sockfile: + self._sockfile.close() + self._sockfile = None def settimeout(self, timeout): """Set a timeout, in seconds."""
And applied to my python-next tree:
https://gitlab.com/philmd/qemu/commits/python-next
Philippe Mathieu-Daudé
AppliedRe: [PATCH RFC 13/32] python/qemu/lib: Adjust traceback typing
In reply to John SnowQuoted 10 lines · John Snow
mypy considers it incorrect to use
boolto statically return false, because it will assume that it could conceivably return True, and gives different analysis in that case. Use a None return to achieve the same effect, but make mypy happy.Note: Pylint considers function signatures as code that might trip the duplicate-code checker. I'd rather not disable this as it does not trigger often in practice, so I'm disabling it as a one-off and filed a change request; see https://github.com/PyCQA/pylint/issues/3619
Quoted 5 lines
Signed-off-by John Snow <[email protected]>--- python/qemu/lib/machine.py | 8 ++++++-- python/qemu/lib/qmp.py | 10 ++++++++-- 2 files changed, 14 insertions(+), 4 deletions(-)
Thanks, applied to my python-next tree:
https://gitlab.com/philmd/qemu/commits/python-next
Philippe Mathieu-Daudé
AppliedRe: [PATCH RFC 20/32] python//qmp.py: assert sockfile is not None
In reply to John SnowQuoted 17 lines · John Snow
In truth, if you don't do this, you'll just get a TypeError exception. Now, you'll get an AssertionError.
Is this tangibly better? No.
Does mypy complain less? Yes.Signed-off-by John Snow <[email protected]>--- python/qemu/lib/qmp.py | 1 + 1 file changed, 1 insertion(+)python/qemu/lib/qmp.py +0
-0
python/qemu/lib/qmp.py def __negotiate_capabilities(self): · line 138
Quoted 7 lines
python/qemu/lib/qmp.py +1
-0@@ -132,6 +132,7 @@ def __negotiate_capabilities(self): raise QMPCapabilitiesError def __json_read(self, only_event=False): + assert self.__sockfile is not None while True: data = self.__sockfile.readline() if not data:
Thanks, applied to my python-next tree:
https://gitlab.com/philmd/qemu/commits/python-next
Philippe Mathieu-Daudé
AppliedRe: [PATCH RFC 21/32] python//machine.py: remove logging configuration
In reply to John SnowQuoted 15 lines · John Snow
Python 3.5 and above do not print a warning when logging is not configured. As a library, it's best practice to leave logging configuration to the client executable.
Signed-off-by John Snow <[email protected]>--- python/qemu/lib/machine.py | 3 --- 1 file changed, 3 deletions(-)python/qemu/lib/machine.py +0
-0
python/qemu/lib/machine.py def __init__(self, binary, args=None, wrapper=None, name=None, · line 115
Quoted 9 lines
python/qemu/lib/machine.py +0
-3@@ -110,9 +110,6 @@ def __init__(self, binary, args=None, wrapper=None, name=None, self._console_socket = None self._remove_files = [] - # just in case logging wasn't configured by the main script: - logging.basicConfig() - def __enter__(self): return self
Thanks, applied to my python-next tree:
https://gitlab.com/philmd/qemu/commits/python-next
Philippe Mathieu-Daudé
AppliedRe: [PATCH RFC 14/32] python//qmp.py: use True/False for non/blocking modes
In reply to John SnowQuoted 19 lines · John Snow
The type system doesn't want integers.
Signed-off-by John Snow <[email protected]>--- python/qemu/lib/qmp.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)python/qemu/lib/qmp.py +1
-1@@ -120,14 +120,14 @@ def __get_events(self, wait=False): """ # Check for new events regardless and pull them into the cache: - self.__sock.setblocking(0) + self.__sock.setblocking(False) try:
python/qemu/lib/qmp.py def __get_events(self, wait=False): · line 133
Quoted 10 lines
python/qemu/lib/qmp.py +1
-1self.__json_read() except OSError as err: if err.errno == errno.EAGAIN: # No data available pass - self.__sock.setblocking(1) + self.__sock.setblocking(True) # Wait for new events, if needed. # if wait is 0.0, this means "no wait" and is also implicitly false.
Thanks, applied to my python-next tree:
https://gitlab.com/philmd/qemu/commits/python-next
Philippe Mathieu-Daudé
AppliedRe: [PATCH RFC 29/32] python//qtest.py: Check before accessing _qtest
In reply to John SnowQuoted 29 lines · John Snow
It can be None; so add assertions or exceptions where appropriate to guard the access accordingly.
Signed-off-by John Snow <[email protected]>--- python/qemu/lib/qtest.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-)python/qemu/lib/qtest.py +2
-3@@ -126,7 +126,8 @@ def _pre_launch(self): super()._pre_launch() self._qtest = QEMUQtestProtocol(self._qtest_path, server=True) - def _post_launch(self): + def _post_launch(self) -> None: + assert self._qtest is not None super()._post_launch() self._qtest.accept()@@ -134,6 +135,13 @@ def _post_shutdown(self): super()._post_shutdown() self._remove_if_exists(self._qtest_path) - def qtest(self, cmd): - '''Send a qtest command to guest'''
python/qemu/lib/qtest.py def _post_shutdown(self): · line 147
Quoted 10 lines
python/qemu/lib/qtest.py +9
-0+ def qtest(self, cmd: str) -> str: + """ + Send a qtest command to the guest. + + :param cmd: qtest command to send + :return: qtest server response + """ + if self._qtest is None: + raise RuntimeError("qtest socket not available") return self._qtest.cmd(cmd)
Thanks, applied to my python-next tree:
https://gitlab.com/philmd/qemu/commits/python-next
Re: [PATCH RFC 01/32] python/qemu: create qemu.lib module
In reply to John SnowAm 14.05.2020 um 07:53 hat John Snow geschrieben:
Quoted 14 lines
move python/qemu/*.py to python/qemu/lib/*.py.
To create a namespace package, the 'qemu' directory itself shouldn't have module files in it. Thus, these files will go under a 'lib' package directory instead.
Bolster the
lib/__init__.pyfile a little bit, Make the top-level classes and functions available directly inside theqemu.libnamespace, to facilitate a convenient shorthand:Quoted 1 line
from qemu.lib import QEMUQtestMachine, QEMUMonitorProtocol
Lastly, update all of the existing import directives.
Quoted 7 lines
(Note: these scripts were not necessarily tested to see if they still work. Some of these scripts are in obvious states of disrepair and it is beyond the scope of this patch to attempt to fix them.)
Signed-off-by John Snow <[email protected]>--- python/qemu/__init__.py | 11 -----
When trying to reproduce your mypy problem, I was wondering why mypy was complaining that it couldn't find qemu.lib. The reason is that removing __init__.py from qemu means it's not a valid module any more. If I recreate it locally, mypy stops complaining.
So I think we need to leave this file here.
Kevin
Re: [PATCH RFC 26/32] python//machine.py: use qmp.command
In reply to John SnowAm 29.05.2020 um 02:18 hat John Snow geschrieben:
Quoted 40 lines
[...]
Quoted 9 lines
(quoted diff) +2
-4- def qmp(self, cmd, conv_keys=True, **args): - """ - Invoke a QMP command and return the response dict - """ + @classmethod + def _qmp_args(cls, _conv_keys: bool = True, **args: Any) -> Dict[str, Any]: qmp_args = dict() for key, value in args.items():Review
Quoted 10 lines
(quoted diff) +5
-1- if conv_keys: + if _conv_keys: qmp_args[key.replace('_', '-')] = value else: qmp_args[key] = value + return qmp_args + def qmp(self, cmd: str, + conv_keys: bool = True, + **args: Any) -> QMPMessage:This creates an interesting problem with iotests 297:
(quoted diff) +1
-1-Success: no issues found in 1 source file +iotests.py:563: error: Argument 2 to "qmp" of "QEMUMachine" hasincompatible type "**Dict[str, str]"; expected "bool" +Found 1 error in 1 file (checked 1 source file)
def
hmp(self, command_line: str, use_log: bool = False)-> QMPResponse:
cmd = 'human-monitor-command'
kwargs = {'command-line':command_line}
ifuse_log:
returnself.qmp_log(cmd, **kwargs)
else:
returnself.qmp(cmd, **kwargs)
Quoted 5 lines
It seems like mypy is unable to understand that we are passing keyword arguments, and instead believes we're passing something to the
conv_keysparameter.(Is this a bug...?)
It took some experimenting, but in the end, I think the logic behind it is reasonable enough. If you have kwargs: Dict[str, T] and pass it as **kwargs, mypy checks that passing T to _every_ keyword argument is fine because generally speaking you have to expect that the name of any argument could be present in kwargs. (In the specific example, of course, we know all the keys, but mypy doesn't look at the content of the dict.)
With this in mind, getting rid of the error is very simple because you only need to make sure that T is Any instead of str:
tests/qemu-iotests/iotests.py +1 -1
@@ -556,7 +556,7 @@ class VM(qtest.QEMUQtestMachine): def hmp(self, command_line: str, use_log: bool = False) -> QMPResponse: cmd = 'human-monitor-command' - kwargs = {'command-line': command_line} + kwargs: Dict[str, Any] = {'command-line': command_line} if use_log: return self.qmp_log(cmd, **kwargs) else:
Kevin
Re: [PATCH RFC 26/32] python//machine.py: use qmp.command
Am 02.06.2020 um 12:18 hat Kevin Wolf geschrieben:
Quoted 63 lines
Am 29.05.2020 um 02:18 hat John Snow geschrieben:
Quoted 40 lines
[...]
Quoted 9 lines
(quoted diff) +2
-4- def qmp(self, cmd, conv_keys=True, **args): - """ - Invoke a QMP command and return the response dict - """ + @classmethod + def _qmp_args(cls, _conv_keys: bool = True, **args: Any) -> Dict[str, Any]: qmp_args = dict() for key, value in args.items():Review
Quoted 10 lines
(quoted diff) +5
-1- if conv_keys: + if _conv_keys: qmp_args[key.replace('_', '-')] = value else: qmp_args[key] = value + return qmp_args + def qmp(self, cmd: str, + conv_keys: bool = True, + **args: Any) -> QMPMessage:This creates an interesting problem with iotests 297:
(quoted diff) +1
-1-Success: no issues found in 1 source file +iotests.py:563: error: Argument 2 to "qmp" of "QEMUMachine" hasincompatible type "**Dict[str, str]"; expected "bool" +Found 1 error in 1 file (checked 1 source file)
def
hmp(self, command_line: str, use_log: bool = False)-> QMPResponse:
cmd = 'human-monitor-command'
kwargs = {'command-line':command_line}
ifuse_log:
returnself.qmp_log(cmd, **kwargs)
else:
returnself.qmp(cmd, **kwargs)Quoted 5 lines
It seems like mypy is unable to understand that we are passing keyword arguments, and instead believes we're passing something to the
conv_keysparameter.(Is this a bug...?)
It took some experimenting, but in the end, I think the logic behind it is reasonable enough. If you have kwargs: Dict[str, T] and pass it as **kwargs, mypy checks that passing T to _every_ keyword argument is fine because generally speaking you have to expect that the name of any argument could be present in kwargs. (In the specific example, of course, we know all the keys, but mypy doesn't look at the content of the dict.)
With this in mind, getting rid of the error is very simple because you only need to make sure that T is Any instead of str:
tests/qemu-iotests/iotests.py +0
-0
tests/qemu-iotests/iotests.py class VM(qtest.QEMUQtestMachine): · line 562
Quoted 8 lines
tests/qemu-iotests/iotests.py +1
-1@@ -556,7 +556,7 @@ class VM(qtest.QEMUQtestMachine): def hmp(self, command_line: str, use_log: bool = False) -> QMPResponse: cmd = 'human-monitor-command' - kwargs = {'command-line': command_line} + kwargs: Dict[str, Any] = {'command-line': command_line} if use_log: return self.qmp_log(cmd, **kwargs) else:
There is actually also a second way to get rid of the error, which is specifying conv_keys explicitly so that the value from kwargs wouldn't be used (if anything, we would get an exception because the value was specified twice). So the patch below makes the mypy error go away, too.
I think I prefer the explicit Dict[str, Any] because it doesn't end up duplicating the default value, but I just wanted to mention that this option exists, too.
Kevin
tests/qemu-iotests/iotests.py +1 -1
@@ -560,7 +560,7 @@ class VM(qtest.QEMUQtestMachine): if use_log: return self.qmp_log(cmd, **kwargs) else: - return self.qmp(cmd, **kwargs) + return self.qmp(cmd, True, **kwargs) def pause_drive(self, drive: str, event: Optional[str] = None) -> None: """Pause drive r/w operations"""
Re: [PATCH RFC 03/32] python//machine.py: remove bare except
In reply to John SnowAm 14.05.2020 um 07:53 hat John Snow geschrieben:
Quoted 4 lines
Catch only the timeout error; if there are other problems, allow the stack trace to be visible.
Signed-off-by John Snow <[email protected]>
Having a visible stack trace is nice, but don't we still want to kill the qemu process to be sure that it's gone even though we can't wait for it to be shut down cleanly?
Kevin
Re: [PATCH RFC 01/32] python/qemu: create qemu.lib module
In reply to Philippe Mathieu-DaudéQuoted 71 lines · Philippe Mathieu-Daudé
Quoted 60 lines · Vladimir Sementsov-Ogievskiy
Quoted 59 lines · John Snow
Quoted 56 lines · Vladimir Sementsov-Ogievskiy
Quoted 54 lines · John Snow
Quoted 23 lines · Vladimir Sementsov-Ogievskiy
Quoted 6 lines · John Snow
move python/qemu/*.py to python/qemu/lib/*.py.
To create a namespace package, the 'qemu' directory itself shouldn't have module files in it. Thus, these files will go under a 'lib' package
directory instead.Hmm..
On the first glance, it looks better to have
from qemu import QEMUMachine
than
from qemu.lib import QEMUMachinewhy do we need this extra ".lib" part?
Is it needed only for internal use?
Assume we have installed qemu package. Can we write
Quoted 9 lines
from qemu import QEMUMachine
? Or we still need qemu.lib ?
I don't remember any python package, which made me to write "import from
package_name.lib ..."It's a strategy to create "qemu" as a PEP420 namespace package; i.e. "qemu" forms a namespace, but you need a name for the actual package underneath it.
"qemu.lib" is one package, with qmp, qtest, and machine modules. "qemu" isn't really a package in this system, it's just a namespace.
The idea is that this allows us to create a more modular rollout of various python scripts and services as desired instead of monolithically
bundling them all inside of a "qemu" package.It also allows us to fork or split out the sub-packages to separate repos, if we wish. i.e., let's say we create a "qemu.sdk" subpackage, we
can eventually fork it off into its own repo with its own installer and so forth. These subpackages can be installed and managed separately.Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Okay, I understand.. No real objections than.
Still, maybe, everything should not go into lib, maybe something like
qemu/vm/ - qmp, QEMUMachine, etc
qemu/qtest/ - qtestI'm not sure this part is relevant now, as we have not good projection of what/who/how this package will be consumed.
Quoted 8 lines
I suppose by VM you mean VirtualMachine. I find it confusing. Maybe simply "machine"? We also have 'tools' and 'user-space processes'.
QMP is protocol, common to all. "qemu.core.qmp"?
We also have the gdb(stub) protocol, common to machine(system) & user.
The block layer has its classes, "qemu.block"?
Sounds good. But I see now that we shouldn't care too much about this, as we just don't know how it all will be used..
Quoted 22 lines
Quoted 12 lines
Quoted 12 lines
Quoted 5 lines
would be more user friendly? But I'm not sure. I just thought that "lib" is too generic.
lib is a very generic name, I agree.
Splitting accel, qmp and QEMUMachine in one package and keeping qtest in another is fine too. I'm not sure if I like "vm" for the name of that core package, though.
Quoted 7 lines
Quoted 5 lines
I want to avoid using "qemu/sdk" because I have some plans for trying to generate and package a "real" SDK using that namespace.
"devkit"? "testkit"? "core"? Naming things is always the worst part.
I think, "core" sounds good.
Agreed.
Signature · 2 lines
Best regards, Vladimir
Re: [PATCH RFC 01/32] python/qemu: create qemu.lib module
In reply to Kevin WolfQuoted 23 lines · Kevin Wolf
Am 14.05.2020 um 07:53 hat John Snow geschrieben:
Quoted 14 lines
move python/qemu/*.py to python/qemu/lib/*.py.
To create a namespace package, the 'qemu' directory itself shouldn't have module files in it. Thus, these files will go under a 'lib' package directory instead.
Bolster the
lib/__init__.pyfile a little bit, Make the top-level classes and functions available directly inside theqemu.libnamespace, to facilitate a convenient shorthand:Quoted 1 line
from qemu.lib import QEMUQtestMachine, QEMUMonitorProtocol
Lastly, update all of the existing import directives.
Quoted 7 lines
(Note: these scripts were not necessarily tested to see if they still work. Some of these scripts are in obvious states of disrepair and it is beyond the scope of this patch to attempt to fix them.)
Signed-off-by John Snow <[email protected]>--- python/qemu/__init__.py | 11 -----
Quoted 9 lines
When trying to reproduce your mypy problem, I was wondering why mypy was complaining that it couldn't find qemu.lib. The reason is that removing
__init__.pyfrom qemu means it's not a valid module any more. If I recreate it locally, mypy stops complaining.So I think we need to leave this file here.
Kevin
Depends. You'll want --namespace-packages to parse a PEP420 namespace.
(It's not a given we definitely want a PEP420 namespace, but that's what I created here.)
--js
Re: [PATCH RFC 03/32] python//machine.py: remove bare except
In reply to Kevin WolfQuoted 4 lines · Kevin Wolf
Am 14.05.2020 um 07:53 hat John Snow geschrieben:
Quoted 3 lines
Catch only the timeout error; if there are other problems, allow the stack trace to be visible.
Quoted 8 lines
Quoted 1 line
Signed-off-by John Snow <[email protected]>Having a visible stack trace is nice, but don't we still want to kill the qemu process to be sure that it's gone even though we can't wait for it to be shut down cleanly?
Kevin
Ah, yes. I should kill and re-raise. I think Vladimir checked in some code here recently that kept this patch from applying, so I'll have to re-tune it anyway.
Thanks for the spot.
--js
Re: [PATCH RFC 26/32] python//machine.py: use qmp.command
In reply to Kevin WolfQuoted 3 lines · Kevin Wolf
I think I prefer the explicit Dict[str, Any] because it doesn't end up duplicating the default value, but I just wanted to mention that this option exists, too.
Oh, I see what's going on here now! Thanks, that's very helpful! In this case, for now, I think I like your first solution, too.
Someday in the future, I want to look into strictly typed generated SDK objects where mypy *does* know the types of every field in a QMP command object.
That's not today, and it's probably not soon, (and it might be never,) but until then, explicitly hinting the kwargs to be Dict[str, Any] is reasonable enough.
Thank you so much!
--js
Re: [PATCH RFC 01/32] python/qemu: create qemu.lib module
In reply to John SnowAm 02.06.2020 um 18:44 hat John Snow geschrieben:
Quoted 33 lines · Kevin Wolf
Quoted 30 lines
Am 14.05.2020 um 07:53 hat John Snow geschrieben:
Quoted 14 lines
move python/qemu/*.py to python/qemu/lib/*.py.
To create a namespace package, the 'qemu' directory itself shouldn't have module files in it. Thus, these files will go under a 'lib' package directory instead.
Bolster the
lib/__init__.pyfile a little bit, Make the top-level classes and functions available directly inside theqemu.libnamespace, to facilitate a convenient shorthand:Quoted 1 line
from qemu.lib import QEMUQtestMachine, QEMUMonitorProtocol
Lastly, update all of the existing import directives.
Quoted 7 lines
(Note: these scripts were not necessarily tested to see if they still work. Some of these scripts are in obvious states of disrepair and it is beyond the scope of this patch to attempt to fix them.)
Signed-off-by John Snow <[email protected]>--- python/qemu/__init__.py | 11 -----When trying to reproduce your mypy problem, I was wondering why mypy was complaining that it couldn't find qemu.lib. The reason is that removing
__init__.pyfrom qemu means it's not a valid module any more. If I recreate it locally, mypy stops complaining.So I think we need to leave this file here.
Quoted 7 lines
Quoted 2 lines
Kevin
Depends. You'll want --namespace-packages to parse a PEP420 namespace.
(It's not a given we definitely want a PEP420 namespace, but that's what I created here.)
I'm just running 297, so if this is how mypy should be called now, the series needs to make a change to 297.
Kevin
Re: [PATCH RFC 01/32] python/qemu: create qemu.lib module
Quoted 39 lines · Kevin Wolf
Am 02.06.2020 um 18:44 hat John Snow geschrieben:
Quoted 38 lines · Kevin Wolf
Quoted 23 lines
Am 14.05.2020 um 07:53 hat John Snow geschrieben:
Quoted 14 lines
move python/qemu/*.py to python/qemu/lib/*.py.
To create a namespace package, the 'qemu' directory itself shouldn't have module files in it. Thus, these files will go under a 'lib' package directory instead.
Bolster the
lib/__init__.pyfile a little bit, Make the top-level classes and functions available directly inside theqemu.libnamespace, to facilitate a convenient shorthand:Quoted 1 line
from qemu.lib import QEMUQtestMachine, QEMUMonitorProtocol
Lastly, update all of the existing import directives.
Quoted 7 lines
(Note: these scripts were not necessarily tested to see if they still work. Some of these scripts are in obvious states of disrepair and it is beyond the scope of this patch to attempt to fix them.)
Signed-off-by John Snow <[email protected]>--- python/qemu/__init__.py | 11 -----Quoted 9 lines
When trying to reproduce your mypy problem, I was wondering why mypy was complaining that it couldn't find qemu.lib. The reason is that removing
__init__.pyfrom qemu means it's not a valid module any more. If I recreate it locally, mypy stops complaining.So I think we need to leave this file here.
Kevin
Depends. You'll want --namespace-packages to parse a PEP420 namespace.
Quoted 8 lines
Quoted 2 lines
(It's not a given we definitely want a PEP420 namespace, but that's what I created here.)
I'm just running 297, so if this is how mypy should be called now, the series needs to make a change to 297.
Kevin
Yes, sorry -- 297 wasn't merged yet when I wrote the 32 patch RFC, and my followup series to add mypy support is now factored to occur prior to the library split.
I have corrected this in the more recent postings, sorry for the inconvenience.