QEMU validator: A method to specify QEMU crash-test cases

index 5c8cf6a056..078ce65f3b 100644
@@ -244,7 +244,9 @@ class QEMUMonitorProtocol(object):
 
     def close(self):
         self.__sock.close()
-        self.__sockfile.close()
+        if self.__sockfile is not None:
+            self.__sockfile.close()
+            self.__sockfile = None
 
     def settimeout(self, timeout):
         self.__sock.settimeout(timeout)

index 078ce65f3b..e9cb6b2683 100644
@@ -108,7 +108,7 @@ class QEMUMonitorProtocol(object):
         try:
             self.__json_read()
         except socket.error as err:
-            if err[0] == errno.EAGAIN:
+            if err.args[0] == errno.EAGAIN:
                 # No data available
                 pass
         self.__sock.setblocking(1)

@@ -168,9 +168,9 @@ class QEMUMonitorProtocol(object):
         try:
             self.__sock.sendall(json.dumps(qmp_cmd).encode('utf-8'))
         except socket.error as err:
-            if err[0] == errno.EPIPE:
+            if err.args[0] == errno.EPIPE:
                 return
-            raise socket.error(err)
+            raise
         resp = self.__json_read()
         self.logger.debug("<<< %s", resp)
         return resp

index e9cb6b2683..91b537ea04 100644
@@ -73,6 +73,8 @@ class QEMUMonitorProtocol(object):
             raise QMPConnectError
         # Greeting seems ok, negotiate capabilities
         resp = self.cmd('qmp_capabilities')
+        if resp is None:
+            raise QMPConnectError("QMP connection unexpectedly closed")
         if "return" in resp:
             return greeting
         raise QMPCapabilitiesError

@@ -182,6 +184,8 @@ class QEMUMonitorProtocol(object):
         @param name: command name (string)
         @param args: command arguments (dict)
         @param cmd_id: command id (dict, list, string or int)
+        @return QMP response as a Python dict or None if the connection has
+                been closed
         """
         qmp_cmd = {'execute': name}
         if args:

@@ -195,6 +199,8 @@ class QEMUMonitorProtocol(object):
         Build and send a QMP command to the monitor, report errors if any
         """
         ret = self.cmd(cmd, kwds)
+        if ret is None:
+            raise QMPConnectError("QMP connection unexpectedly closed")
         if "error" in ret:
             raise Exception(ret['error']['desc'])
         return ret['return']

index 08a3e9af5a..c051c4525a 100644
@@ -76,7 +76,6 @@ class QEMUMachine(object):
             name = "qemu-%d" % os.getpid()
         self._name = name
         self._monitor_address = monitor_address
-        self._vm_monitor = None
         self._qemu_log_path = None
         self._qemu_log_file = None
         self._popen = None

@@ -168,29 +167,28 @@ class QEMUMachine(object):
             with open(self._qemu_log_path, "r") as iolog:
                 self._iolog = iolog.read()
 
+    def _vm_monitor(self):
+        if self._monitor_address is not None:
+            return self._monitor_address
+        else:
+            return os.path.join(self._temp_dir, self._name + "-monitor.sock")
+
     def _base_args(self):
-        if isinstance(self._monitor_address, tuple):
-            moncdev = "socket,id=mon,host=%s,port=%s" % (
-                self._monitor_address[0],
-                self._monitor_address[1])
+        addr = self._vm_monitor()
+        if isinstance(addr, tuple):
+            moncdev = "socket,id=mon,host=%s,port=%s" % (addr[0], addr[1])
         else:
-            moncdev = 'socket,id=mon,path=%s' % self._vm_monitor
+            moncdev = 'socket,id=mon,path=%s' % (addr)
         return ['-chardev', moncdev,
                 '-mon', 'chardev=mon,mode=control',
                 '-display', 'none', '-vga', 'none']
 
     def _pre_launch(self):
         self._temp_dir = tempfile.mkdtemp(dir=self._test_dir)
-        if self._monitor_address is not None:
-            self._vm_monitor = self._monitor_address
-        else:
-            self._vm_monitor = os.path.join(self._temp_dir,
-                                            self._name + "-monitor.sock")
         self._qemu_log_path = os.path.join(self._temp_dir, self._name + ".log")
         self._qemu_log_file = open(self._qemu_log_path, 'wb')
 
-        self._qmp = qmp.qmp.QEMUMonitorProtocol(self._vm_monitor,
-                                                server=True)
+        self._qmp = qmp.qmp.QEMUMonitorProtocol(self._vm_monitor(), server=True)
 
     def _post_launch(self):
         self._qmp.accept()

index c051c4525a..d9f85bb153 100644
@@ -173,15 +173,20 @@ class QEMUMachine(object):
         else:
             return os.path.join(self._temp_dir, self._name + "-monitor.sock")
 
-    def _base_args(self):
+    def _monitor_args(self):
         addr = self._vm_monitor()
         if isinstance(addr, tuple):
             moncdev = "socket,id=mon,host=%s,port=%s" % (addr[0], addr[1])
         else:
             moncdev = 'socket,id=mon,path=%s' % (addr)
         return ['-chardev', moncdev,
-                '-mon', 'chardev=mon,mode=control',
-                '-display', 'none', '-vga', 'none']
+                '-mon', 'chardev=mon,mode=control']
+
+    def _display_args(self):
+        return ['-display', 'none', '-vga', 'none']
+
+    def _base_args(self):
+        return self._monitor_args() + self._display_args()
 
     def _pre_launch(self):
         self._temp_dir = tempfile.mkdtemp(dir=self._test_dir)

index d9f85bb153..84bb3da613 100644
@@ -199,6 +199,7 @@ class QEMUMachine(object):
         self._qmp.accept()
 
     def _post_shutdown(self):
+        self._load_io_log()
         if self._qemu_log_file is not None:
             self._qemu_log_file.close()
             self._qemu_log_file = None

@@ -250,7 +251,6 @@ class QEMUMachine(object):
         '''Wait for the VM to power off'''
         self._popen.wait()
         self._qmp.close()
-        self._load_io_log()
         self._post_shutdown()
 
     def shutdown(self):

@@ -263,7 +263,6 @@ class QEMUMachine(object):
                 self._popen.kill()
             self._popen.wait()
 
-        self._load_io_log()
         self._post_shutdown()
 
         exitcode = self.exitcode()

index 84bb3da613..4a71542c6b 100644
@@ -258,12 +258,10 @@ class QEMUMachine(object):
         if self.is_running():
             try:
                 self._qmp.cmd('quit')
-                self._qmp.close()
             except:
                 self._popen.kill()
-            self._popen.wait()
 
-        self._post_shutdown()
+        self.wait()
 
         exitcode = self.exitcode()
         if exitcode is not None and exitcode < 0:

index 4a71542c6b..1241e7f10b 100644
@@ -199,6 +199,10 @@ class QEMUMachine(object):
         self._qmp.accept()
 
     def _post_shutdown(self):
+        if self._qmp is not None:
+            self._qmp.close()
+            self._qmp = None
+
         self._load_io_log()
         if self._qemu_log_file is not None:
             self._qemu_log_file.close()

@@ -250,7 +254,6 @@ class QEMUMachine(object):
     def wait(self):
         '''Wait for the VM to power off'''
         self._popen.wait()
-        self._qmp.close()
         self._post_shutdown()
 
     def shutdown(self):

index 1241e7f10b..e82540a235 100644
@@ -54,7 +54,7 @@ class QEMUMachine(object):
     '''
 
     def __init__(self, binary, args=None, wrapper=None, name=None,
-                 test_dir="/var/tmp", monitor_address=None,
+                 test_dir="/var/tmp", monitor_address=None, qmp=True,
                  socket_scm_helper=None):
         '''
         Initialize a QEMUMachine

@@ -65,6 +65,7 @@ class QEMUMachine(object):
         @param name: prefix for socket and log file names (default: qemu-PID)
         @param test_dir: where to create socket and log file
         @param monitor_address: address for QMP monitor
+        @param qmp: if False, disable QMP monitor
         @param socket_scm_helper: helper program, required for send_fd_scm()"
         @note: Qemu process is not started until launch() is used.
         '''

@@ -90,6 +91,7 @@ class QEMUMachine(object):
         self._test_dir = test_dir
         self._temp_dir = None
         self._launched = False
+        self._qmp_enabled = qmp
 
         # just in case logging wasn't configured by the main script:
         logging.basicConfig()

@@ -168,13 +170,18 @@ class QEMUMachine(object):
                 self._iolog = iolog.read()
 
     def _vm_monitor(self):
-        if self._monitor_address is not None:
+        if not self._qmp_enabled:
+            return None
+        elif self._monitor_address is not None:
             return self._monitor_address
         else:
             return os.path.join(self._temp_dir, self._name + "-monitor.sock")
 
     def _monitor_args(self):
         addr = self._vm_monitor()
+        if addr is None:
+            return []
+
         if isinstance(addr, tuple):
             moncdev = "socket,id=mon,host=%s,port=%s" % (addr[0], addr[1])
         else:

@@ -193,10 +200,14 @@ class QEMUMachine(object):
         self._qemu_log_path = os.path.join(self._temp_dir, self._name + ".log")
         self._qemu_log_file = open(self._qemu_log_path, 'wb')
 
-        self._qmp = qmp.qmp.QEMUMonitorProtocol(self._vm_monitor(), server=True)
+        self._qmp = \
+            qmp.qmp.QEMUMonitorProtocol(self._vm_monitor(), server=True) \
+            if self._qmp_enabled \
+            else None
 
     def _post_launch(self):
-        self._qmp.accept()
+        if self._qmp:
+            self._qmp.accept()
 
     def _post_shutdown(self):
         if self._qmp is not None:

index e82540a235..226d2c4d48 100644
@@ -225,6 +225,8 @@ class QEMUMachine(object):
             shutil.rmtree(self._temp_dir)
             self._temp_dir = None
 
+        self._launched = False
+
     def launch(self):
         """
         Launch the VM and make sure we cleanup and expose the

@@ -286,8 +288,6 @@ class QEMUMachine(object):
                 command = ''
             LOG.warn(msg, exitcode, command)
 
-        self._launched = False
-
     def qmp(self, cmd, conv_keys=True, **args):
         '''Invoke a QMP command and return the response dict'''
         qmp_args = dict()

index 226d2c4d48..e19e4b34d0 100644
@@ -225,6 +225,15 @@ class QEMUMachine(object):
             shutil.rmtree(self._temp_dir)
             self._temp_dir = None
 
+        exitcode = self.exitcode()
+        if exitcode is not None and exitcode < 0:
+            msg = 'qemu received signal %i: %s'
+            if self._qemu_full_args:
+                command = ' '.join(self._qemu_full_args)
+            else:
+                command = ''
+            LOG.warn(msg, exitcode, command)
+
         self._launched = False
 
     def launch(self):

@@ -279,15 +288,6 @@ class QEMUMachine(object):
 
         self.wait()
 
-        exitcode = self.exitcode()
-        if exitcode is not None and exitcode < 0:
-            msg = 'qemu received signal %i: %s'
-            if self._qemu_full_args:
-                command = ' '.join(self._qemu_full_args)
-            else:
-                command = ''
-            LOG.warn(msg, exitcode, command)
-
     def qmp(self, cmd, conv_keys=True, **args):
         '''Invoke a QMP command and return the response dict'''
         qmp_args = dict()

index e19e4b34d0..aaba04b3c1 100644
@@ -275,7 +275,8 @@ class QEMUMachine(object):
 
     def wait(self):
         '''Wait for the VM to power off'''
-        self._popen.wait()
+        if self.is_running():
+            self._popen.wait()
         self._post_shutdown()
 
     def shutdown(self):

index aaba04b3c1..045dca5d02 100644
@@ -279,13 +279,24 @@ class QEMUMachine(object):
             self._popen.wait()
         self._post_shutdown()
 
-    def shutdown(self):
-        '''Terminate the VM and clean up'''
-        if self.is_running():
+    def shutdown(self, force=None):
+        '''Terminate the VM and clean up
+
+        @param force: If True, terminate QEMU process, if False always try
+                      clean shutdown, if None terminate QEMU process if
+                      clean shutdown fails.
+        '''
+        if not force and self.is_running():
             try:
                 self._qmp.cmd('quit')
             except:
-                self._popen.kill()
+                if force is None:
+                    force = True
+                else:
+                    raise
+
+        if force and self.is_running():
+            self._popen.kill()
 
         self.wait()
 

index 045dca5d02..00b44ea0f4 100644
@@ -251,7 +251,7 @@ class QEMUMachine(object):
             self._launch()
             self._launched = True
         except:
-            self.shutdown()
+            self.shutdown(force=True)
 
             LOG.debug('Error launching VM')
             if self._qemu_full_args:

index 00b44ea0f4..d25fe030bb 100644
@@ -311,6 +311,9 @@ class QEMUMachine(object):
 
         return self._qmp.cmd(cmd, args=qmp_args)
 
+    def qmp_obj(self, *args, **kwargs):
+        return self._qmp.cmd_obj(*args, **kwargs)
+
     def command(self, cmd, conv_keys=True, **args):
         '''
         Invoke a QMP command.

index d25fe030bb..d32b923a15 100644
@@ -236,6 +236,9 @@ class QEMUMachine(object):
 
         self._launched = False
 
+    def is_launched(self):
+        return self._launched
+
     def launch(self):
         """
         Launch the VM and make sure we cleanup and expose the

new file mode 100755
index 0000000000..4312571feb
@@ -0,0 +1,724 @@
+#!/usr/bin/env python
+#
+#  Copyright (c) 2018 Red Hat Inc
+#
+# Author:
+#  Eduardo Habkost <[email protected]>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+"""
+QEMU validator script
+=====================
+
+This script will get test YAML test case specifications or Python
+modules as input, and generate/run test cases based on them.
+
+USAGE
+-----
+
+validator.py <specification-file>... -V VAR1=value1 VAR1=value2 VAR2=value3
+
+specification-file is a YAML file containing the test specification.
+
+Example::
+
+    # this test specification is equivalent to the
+    # "device/introspect/list" test case in device-introspect-test.c
+    command-line: '$QEMU -nodefaults -machine none'
+    monitor-commands:
+    - qmp:
+      - execute: qom-list-types
+        arguments:
+          implements: 'device'
+          abstract: true
+    - hmp: 'device_add help'
+
+
+VARIABLE EXPANSION
+------------------
+
+The test runner will try to run the test cases with all possible values
+for variables appearing in the test specification.
+
+Some built-in variables are automatically expanded:
+
+* `$MACHINE` - Expands to a machine-type name supported by $QEMU
+* `$ACCEL` - Expands to an accelerator name supported by $QEMU
+* `$DEVICE` - Expands to a (user-creatable) device type name supported by $QEMU
+* `$CPU` - Expands to a CPU model name supported by $QEMU
+
+Note that the $QEMU variable must be specified in th
+
+TEST SPECIFICATION FIELDS
+-------------------------
+
+command-line
+~~~~~~~~~~~~
+
+List or string, containing the QEMU command-line to be run.
+
+Default: '$QEMU'
+
+
+monitor-commands
+~~~~~~~~~~~~~~~~
+
+Mapping or list-of-mappings containing monitor commands to run.  The key on each
+item can be ``hmp`` or ``qmp``.  The value on each entry can be a string,
+mapping, or list.
+
+Default: None.
+
+TODO: not implemented yet.
+
+
+qmp
+~~~
+
+Boolean.  If true (the default), a QMP monitor is configured on the command-line
+automatically.
+
+If true, the test runner will issue a ``quit`` command automatically when
+testing is finished.  If false, the test runner will wait until QEMU exits by
+itself.
+
+Example::
+
+    # just run $QEMU -help and ensure it won't crash
+    command-line: ['$QEMU', '-help']
+    qmp: false
+
+
+TODO: whitelist
+TODO: validate output against reference output
+TODO: configure defaults for variables
+TODO: compatibility with Avocado multiplexer?
+"""
+
+import sys
+import os
+import string
+import argparse
+import pprint
+import yaml
+import logging
+import shlex
+import pipes
+import re
+import itertools
+import traceback
+import socket
+from collections import OrderedDict
+
+sys.path.append(os.path.join(os.path.dirname(__file__), '..', 'scripts'))
+from qemu import QEMUMachine
+from qmp.qmp import QMPError
+
+logger = logging.getLogger('qemu.tests.validator')
+dbg = logger.debug
+
+# Python 2.7 compatibility:
+shquote = getattr(shlex, 'quote', pipes.quote)
+
+class InvalidSpecification(Exception):
+    pass
+
+def qom_type_names(vm, **kwargs):
+    """Run qom-list-types QMP command, return type names"""
+    types = vm.command('qom-list-types', **kwargs)
+    return [t['name'] for t in types]
+
+
+def info_qdm(vm):
+    """Parse 'info qdm' output"""
+    args = {'command-line': 'info qdm'}
+    devhelp = vm.command('human-monitor-command', **args)
+    for l in devhelp.split('\n'):
+        l = l.strip()
+        if l == '' or l.endswith(':'):
+            continue
+        d = {'name': re.search(r'name "([^"]+)"', l).group(1),
+             'no-user': (re.search(', no-user', l) is not None)}
+        yield d
+
+
+class QemuBinaryInfo(object):
+    """Information for a specific QEMU binary"""
+    def __init__(self, binary):
+        """Don't instantiate this directly, use get_binary_info()"""
+        self.binary = binary
+
+        args = ['-S', '-machine', 'none,accel=kvm:tcg']
+        dbg("querying info for QEMU binary: %s", binary)
+        vm = QEMUMachine(binary=binary, args=args)
+        vm.launch()
+        try:
+            self.alldevs = qom_type_names(vm, implements='device', abstract=False)
+            # there's no way to query DeviceClass::user_creatable using QMP,
+            # so use 'info qdm':
+            self.no_user_devs = [d['name'] for d in info_qdm(vm, ) if d['no-user']]
+            self.machines = [m['name'] for m in vm.command('query-machines')]
+            self.user_devs = [dev for dev in self.alldevs if dev not in self.no_user_devs]
+            self.kvm_available = vm.command('query-kvm')['enabled']
+            self.cpu_models = [c['name'] for c in vm.command('query-cpu-definitions')]
+        finally:
+            vm.shutdown()
+
+    def available_accels(self):
+        if self.kvm_available:
+            yield 'kvm'
+        yield 'tcg'
+
+BINARY_INFO = {}
+
+def get_binary_info(binary):
+    """Lookup info for QEMU binary, caching data"""
+    if binary not in BINARY_INFO:
+        BINARY_INFO[binary] = QemuBinaryInfo(binary)
+    return BINARY_INFO[binary]
+
+
+# TEST CASE GENERATION LOGIC
+#TODO: the functions below are not used yet, refactor code to use
+#      the simpler variable expansion logic
+
+def updatedict(d1, d2):
+    """Return updated dictionary, after d1.update(d2)"""
+    d = d1.copy()
+    d.update(d2)
+    return d
+
+def newdict(d, k, v):
+    """Return new dictionary with d[k] = v
+
+    >>> a = {'a':1, 'b':2}
+    >>> b = newdict(a, 'a', 100)
+    >>> sorted(a.items())
+    [('a', 1), ('b', 2)]
+    >>> sorted(b.items())
+    [('a', 100), ('b', 2)]
+    """
+    d = d.copy()
+    d[k] = v
+    return d
+
+def mapchain(fn, l):
+    """map(fn, l) and then chain the results together
+
+    >>> multiples = lambda i: [i, i*2, i*3]
+    >>> list(mapchain(multiples, [2, 3, 5]))
+    [2, 4, 6, 3, 6, 9, 5, 10, 15]
+    """
+    return itertools.chain(*map(fn, l))
+
+def call_func(v):
+    """Call function, if v is callable"""
+    if callable(v):
+        return v()
+    return v
+
+def fnitem(tc, var, fn):
+    """Replace tc[var] with fn(tc[var])"""
+    return newdict(tc, var, fn(tc[var]))
+
+def call_var_func(tc, var):
+    """Update `var` with result of its enumeration func, if necessary
+
+    >>> retlist = lambda tc: [{'a':[1,2,3], 'c':['hi']}]
+    >>> tc = {'a':retlist, 'b':[100]}
+    >>> tc = call_var_func(tc, 'a')
+    >>> list(tc) == [{'a':[1,2,3], 'b':[100], 'c':['hi']}]
+    True
+    """
+    if callable(tc[var]):
+        r = tc[var](tc)
+        assert all(var in t for t in r)
+        return (updatedict(tc, t) for t in r)
+    else:
+        return [tc]
+
+#def simple_enum(fn):
+#    """Wrapper for simple enumeration functions that return a list of values
+#
+#    >>> fn = simple_enum(lambda: [1,2,3])
+#    >>> tc = {'a':fn}
+#    >>> tc = call_var_func(tc, 'a')
+#    >>> tc['a']
+#    [1, 2, 3]
+#    """
+#    def enum_func(tc):
+#        return {}
+
+def split_var(tc, var):
+    """Split test case into multiple items
+
+    >>> tc = {'a':[1,2,3], 'b':100}
+    >>> tc = split_var(tc, 'a')
+    >>> [r['a'] for r in tc]
+    [[1], [2], [3]]
+    """
+    return (newdict(tc, var, [v]) for v in tc[var])
+
+def split_vars(tc, vars):
+    """Split test case for multiple variables
+
+    >>> list1 = [1,2]
+    >>> list2 = [10, 20]
+    >>> tc = {'a':list1, 'b':list2, 'c':[100, 200, 300]}
+    >>> tc = split_vars(tc, ['a', 'b'])
+    >>> list(tc) == [{'a':[i], 'b':[j], 'c':[100, 200, 300]} for i in [1,2] for j in [10, 20]]
+    True
+    """
+    tcs = [tc]
+    for var in vars:
+        #TODO: call_var_func()
+        tcs = mapchain(lambda tc: split_var(tc, var), tcs)
+    return tcs
+
+class VariableDefinition(object):
+    # variable names that must be set before this variable
+    deps = []
+
+    def __init__(self):
+        self.values = None
+
+    @staticmethod
+    def enumerate(values):
+        """Enumerate possible values for the variable"""
+        raise NotImplementedError
+
+class BuiltinVars(object):
+    """Namespace for builtin variables"""
+    class QEMU(VariableDefinition):
+        """QEMU binary path, must be set explicitly by the user"""
+        def enumerate(self, values):
+            raise Exception("QEMU variable not set")
+
+    class MACHINE(VariableDefinition):
+        """Machine-type name.  Don't use with $MACHINE_OPT"""
+        deps = ['QEMU']
+
+        @staticmethod
+        def enumerate(values):
+            return get_binary_info(values['QEMU']).machines
+
+    class ACCEL(VariableDefinition):
+        deps = ['QEMU']
+
+        @staticmethod
+        def enumerate(values):
+            return get_binary_info(values['QEMU']).available_accels()
+
+    class DEVICE(VariableDefinition):
+        deps = ['QEMU']
+
+        @staticmethod
+        def enumerate(values):
+            return get_binary_info(values['QEMU']).user_devs
+
+    class CPU(VariableDefinition):
+        deps = ['QEMU']
+
+        @staticmethod
+        def enumerate(values):
+            return get_binary_info(values['QEMU']).cpu_models
+
+class VariableEnumerator(object):
+    """Helper class that will enumerate possible values for variables"""
+    def __init__(self):
+        # start with built-in variables:
+        self._vars = dict((v, getattr(BuiltinVars, v)()) for v in dir(BuiltinVars) if not v.startswith('_'))
+        self._defaults = {}
+
+    def _add_var(self, var):
+        return self._vars.setdefault(var, VariableDefinition())
+
+    def set_values(self, var, values):
+        """Set default values for variable `var`
+
+        `values` can be a list of values, or a single item
+        Default values override the values returned by VariableDefinition.enumerate()
+        """
+        if not isinstance(values, list):
+            values = [values]
+        self._add_var(var).values = values
+
+    def update_values(self, valuedict):
+        dbg("update_values: %r", valuedict)
+        for k,v in valuedict.items():
+            self.set_values(k, v)
+
+    def lookup_var(self, var):
+        """Lookup variable"""
+        return self._vars.get(var)
+
+    def var_deps(self, var):
+        """Return variables need to be set before `var`"""
+        return self.lookup_var(var).deps
+
+    def order_deps(self, vars):
+        """Return full list of variables, including dependencies in the right order
+        """
+        dbg("START order_deps %r", vars)
+        result = OrderedDict()
+        vars = OrderedDict(((v,1) for v in reversed(vars)))
+        while vars:
+            dbg("queue: %r", list(vars.keys()))
+            v,_ = vars.popitem(last=True)
+            dbg("next var: %s", v)
+            deps = self.var_deps(v)
+            dbg("deps: %r", deps)
+            if all(dep in result for dep in deps):
+                dbg("var %s is ready", v)
+                result[v] = 1
+                continue
+            # dependencies not met:
+            vars[v] = 1
+            for dep in deps:
+                if dep in vars:
+                    raise Exception("Variable dependency cycle: %s" % (' -> '.join(vars.keys())))
+                vars[dep] = 1
+        return list(result.keys())
+
+    def enumerate(self, env, var, values):
+        """Enumerate possible values for variable `var`
+
+        May be called only if all requirements for `var` are set in `values`.
+        """
+        v = self.lookup_var(var)
+        assert all(dep in values for dep in v.deps)
+        dbg("var: %r", v)
+        if v.values:
+            r = v.values
+        else:
+            r = v.enumerate(values)
+        logger.debug('Values for %s: %r', var, r)
+        return r
+
+# HELPER FUNCTIONS FOR TEMPLATE STRINGS:
+
+def apply_template(templ, values):
+    """Apply variables to a template, supporting strings and lists
+
+    >>> apply_template('$QEMU -machine X', {'QEMU':'qemu-system-x86_64'})
+    'qemu-system-x86_64 -machine X'
+    >>> apply_template({"$TEST": ["$FOO", "is $BAR"]}, \
+                       {'FOO':'XX', 'BAR':'YY', 'TEST':'TT'})
+    {'$TEST': ['XX', 'is YY']}
+    """
+    if isinstance(templ, str):
+        return string.Template(templ).substitute(values)
+    elif isinstance(templ, list):
+        return [apply_template(s, values) for s in templ]
+    elif isinstance(templ, dict):
+        return dict( (k, apply_template(v, values)) for (k, v) in templ.items())
+    else:
+        return templ
+
+def vars_for_template(templ):
+    """Return list of variables used by s when used as template string
+
+    >>> vars_for_template('abcde fgh')
+    []
+    >>> vars_for_template('$A is ${A}, not ${B} or $C')
+    ['A', 'B', 'C']
+    >>> vars_for_template(['$QEMU', '-machine' , '$MACHINE$MACHINE_OPT'])
+    ['QEMU', 'MACHINE', 'MACHINE_OPT']
+    """
+    usedKeys = OrderedDict()
+    class LoggingDict(object):
+        def __getitem__(self, k):
+            usedKeys[k] = 1
+            return 'X'
+    apply_template(templ, LoggingDict())
+    return list(usedKeys.keys())
+
+class TestSpecification(object):
+    def __init__(self, data):
+        self._data = data
+        self.normalize()
+
+    def normalize(self):
+        """Normalize test specification data
+
+        * ensure 'command-line' is a list of arguments
+        * 'monitor-commands' will be an array
+        """
+        # if command-line is omitted, just run QEMU with no arguments:
+        self._data.setdefault('command-line', ['$QEMU'])
+        self._data.setdefault('monitor-commands', [])
+
+        # 'monitor-commands' must be a list
+        if not isinstance(self.get('monitor-commands'), list):
+            self._data['monitor-commands'] = [self.get('monitor-commands')]
+
+    @classmethod
+    def load_file(cls, file):
+        data = yaml.load(open(file))
+        return cls(data)
+
+    def get(self, key, default=None):
+        return self._data.get(key, default)
+
+    def _gen_subtests(self, testcases, env, vars):
+        """Call _gen_test_cases() for each test case in `testcases`"""
+        #import pdb; pdb.set_trace()
+        for tc in testcases:
+            for st in self._gen_test_cases(env, vars, tc):
+                yield st
+
+    def _gen_var_testcases(self, env, var, values):
+        """Generate one test case for each possible value for `var`"""
+        for value in self.var_enum.enumerate(env, var, values):
+            newvalues = values.copy()
+            newvalues[var] = value
+            yield newvalues
+
+    def _gen_test_cases(self, env, vars, values):
+        """Generate list of test cases
+
+        :param vars: List of variable names that are not set yet
+        :param values: values of variables that are already set
+        """
+        if not vars:
+            # No unset variables -> only 1 test case
+            return [values.copy()]
+
+        # pick next unset variable, enumerate values, set it:
+        var = vars[0]
+        cases = self._gen_var_testcases(env, var, values)
+        cases = self._gen_subtests(cases, env, vars[1:])
+        return cases
+
+    def gen_test_cases(self, env):
+        """Generate all test cases for this test specification"""
+        # we generate combinations for the command-line and monitor commands
+        # in separate steps, so test cases using the same QEMU command-line
+        # are grouped together
+        vars = vars_for_template(self.get('command-line')) + vars_for_template(self.get('monitor-commands'))
+
+        self.var_enum = VariableEnumerator()
+        if not env.args.full:
+            self.var_enum.update_values(self.get('defaults', {}))
+        self.var_enum.update_values(env.var_values)
+
+        # put dependencies in right order:
+        vars = self.var_enum.order_deps(vars)
+        cases = self._gen_test_cases(env, vars, {})
+        return (TestCase(self, c) for c in cases)
+
+class TestCase(object):
+    def __init__(self, spec, values):
+        self.spec = spec
+        self.values = values
+
+    def __str__(self):
+        return ' '.join('%s=%s' % (k, shquote(v)) for k,v in self.values.items())
+
+    def is_expected_entry(self, expected_entry):
+        """Check if `expected_entry` matches the testcase/results"""
+        expected_vars = expected_entry.copy()
+        for var,value in expected_vars.items():
+            if self.values.get(var) != value:
+                return False
+        return True
+
+    def is_expected_failure(self):
+        for e in self.getField('expected-failures', []):
+            if self.is_expected_entry(e):
+                return True
+
+    def getField(self, var, default=None):
+        """Get value of test spec field, expanding variables"""
+        return apply_template(self.spec.get(var, default), self.values)
+
+    def qmp_cmd(self, vm, cmd):
+        if isinstance(cmd, list):
+            for c in cmd:
+                self.qmp_cmd(vm, c)
+        elif isinstance(cmd, dict):
+            return vm.qmp_obj(cmd)
+        else:
+            raise InvalidSpecification("QMP command must be dict: %r" % (cmd))
+
+    def hmp_cmd(self, vm, cmd):
+        return vm.command('human-monitor-command', command_line=cmd)
+
+    def monitor_cmd(self, vm, cmd):
+        dbg("monitor cmd: %r", cmd)
+        if isinstance(cmd, dict):
+            for k,v in cmd.items():
+                if k == 'qmp':
+                    self.qmp_cmd(vm, v)
+                elif k == 'hmp':
+                    self.hmp_cmd(vm, v)
+                else:
+                    raise InvalidSpecification("Invalid monitor command: %r: %r" % (k, v))
+
+    def run(self, env):
+        """Check one specific test case
+
+        Returns a dictionary containing failure information on error,
+        or None on success
+        """
+        result = {'success': True }
+        result['is-expected-failure'] = self.is_expected_failure()
+
+        cmdline = self.getField('command-line')
+        if not isinstance(cmdline, list):
+            cmdline = shlex.split(cmdline)
+
+        qmp = self.getField('qmp', True)
+        #TODO: use context manager to enter/exit borrowed VM from env
+        vm = env.get_vm(cmdline, qmp)
+        try:
+            if not vm.is_launched():
+                vm.launch()
+            #TODO: generate/enumerate variables inside monitor commands too
+            for cmd in self.getField('monitor-commands', []):
+                self.monitor_cmd(vm, cmd)
+            if not qmp:
+                vm.wait()
+                env.drop_vm()
+        except KeyboardInterrupt:
+            raise
+        except QMPError as err:
+            result['exception'] = repr(err)
+            result['success'] = False
+        except socket.error as err:
+            result['exception'] = repr(err)
+            result['success'] = False
+
+        dbg('vm is %r', vm)
+        ec = vm.exitcode()
+        dbg("exit code: %r", ec)
+        if ec is not None and ec != 0:
+            result['success'] = False
+        result['exitcode'] = ec
+        result['log'] = vm.get_log()
+
+        #TODO: use context manager to enter/exit borrowed VM from env
+        if not result['success']:
+            env.drop_vm()
+
+        return result
+
+
+class TestEnv(object):
+    def __init__(self, args):
+        self.args = args
+        self._last_vm_args = None
+        self._last_vm = None
+
+    def qemu_binaries(self):
+        return self.args.qemu_binaries
+
+    def drop_vm(self):
+        """Drop existing VM object"""
+        if self._last_vm:
+            #TODO: record failures here
+            self._last_vm.shutdown()
+            self._last_vm = None
+            self._last_vm_args = None
+
+    def get_vm(self, cmdline, qmp):
+        """Get VM object for test case"""
+        if self._last_vm_args == (cmdline, qmp) and self._last_vm.is_running():
+            dbg("Reusing VM object for cmdline %r", cmdline)
+            return self._last_vm
+
+        dbg("Starting new VM for cmdline %r", cmdline)
+        #FIXME: need to catch exitcode/segfaults here somehow  :(
+        self.drop_vm()
+        vm = QEMUMachine(binary=cmdline[0], args=cmdline[1:], qmp=qmp)
+        self._last_vm = vm
+        self._last_vm_args = (cmdline, qmp)
+        return vm
+
+def main():
+    parser = argparse.ArgumentParser(description="Generic QEMU validator")
+    parser.set_defaults(loglevel=logging.INFO)
+    parser.add_argument('-V', metavar='VAR=VALUE', nargs='*',
+                        help="Force variabie VAR to VALUE",
+                        action='append', dest='vars', default=[])
+    parser.add_argument('-d', '--debug',action='store_const',
+                        dest='loglevel', const=logging.DEBUG,
+                        help='debug output')
+    parser.add_argument('-v', '--verbose',action='store_const',
+                        dest='loglevel', const=logging.INFO,
+                        help='verbose output')
+    parser.add_argument('-q', '--quiet',action='store_const',
+                        dest='loglevel', const=logging.WARN,
+                        help='non-verbose output')
+    parser.add_argument("--dry-run", action="store_true",
+                        help="Don't run test cases")
+    parser.add_argument("--full", action="store_true",
+                        help="Run all test case combinations, not just the default for the test specification")
+    parser.add_argument("testfiles", nargs="+", metavar="FILE",
+                        help="Load test case specification from FILE")
+    args = parser.parse_args()
+
+    env = TestEnv(args)
+
+    vars = {}
+    if args.vars:
+        for varval in itertools.chain(*args.vars):
+            var,val = varval.split('=', 1)
+            vars.setdefault(var, []).append(val)
+    env.var_values = vars
+
+    logging.basicConfig(stream=sys.stdout, level=args.loglevel, format='%(levelname)s: %(message)s')
+    resultdict = {}
+    try:
+        for testfile in args.testfiles:
+            specname = os.path.basename(testfile)
+            #TODO: support test specifications pointing to Python modules
+            spec = TestSpecification.load_file(testfile)
+            logger.debug("Test specification:")
+            logger.debug(pprint.pformat(spec._data))
+            logger.debug('---')
+            for tc in spec.gen_test_cases(env):
+                if tc.is_expected_failure():
+                    logger.info("%s: Skipped: %s", specname, str(tc))
+                    continue
+                logger.info("%s: Running: %s", specname, str(tc))
+                if not args.dry_run:
+                    r = tc.run(env)
+                    logger.debug("Result:")
+                    logger.debug(pprint.pformat(r))
+                    if not r['success']:
+                        logger.error("%s: failed: %s", specname, tc)
+                    resultdict.setdefault(r['success'], []).append( (tc, r) )
+    except KeyboardInterrupt:
+        # Print partial test result summary on interrupt
+        logger.info("Interrupted. Partial test summary follows")
+        pass
+
+    env.drop_vm()
+
+    if not args.dry_run:
+        logger.info('%d successes', len(resultdict.get(True, [])))
+        failures = resultdict.get(False, [])
+        if failures:
+            logger.error('%d failures', len(failures))
+            for tc,r in failures:
+                logger.error("Failed: %s", tc)
+                logger.error("Result:")
+                pprint.pprint(r)
+                dbg("Result: %r", r)
+
+if __name__ == '__main__':
+    sys.exit(main())

new file mode 100644
index 0000000000..48032995ff
@@ -0,0 +1,5 @@
+# Simply check if QEMU will run without errors using a CPU model:
+command-line: '$QEMU -S -cpu $CPU'
+expected-failures:
+    # "host" doesn't always work
+    - CPU: 'host'

new file mode 100644
index 0000000000..b4c0daf70c
@@ -0,0 +1,17 @@
+# This test specification will try device_add with all device types
+command-line: '$QEMU -S -machine $MACHINE,accel=$ACCEL'
+monitor-commands:
+- qmp:
+  - execute: 'device_add'
+    arguments:
+      driver: '$DEVICE'
+      id: 'test-device-for-$DEVICE'
+  - execute: 'device_del'
+    arguments:
+      id: 'test-device-for-$DEVICE'
+defaults:
+  MACHINE: 'none'
+  ACCEL: 'kvm:tcg'
+expected-failures:
+  - MACHINE: 'xenpv'
+  - MACHINE: 'xenfv'

new file mode 100644
index 0000000000..edaab0ee8f
@@ -0,0 +1,7 @@
+# this is a very simple test case generator that will run QEMU
+# using all combinations of -machine and -device.
+# This replaces scripts/device-crash-test.
+# TODO: use a $MACHINE_OPT variable to make -machine optional
+command-line: '$QEMU -S -machine $MACHINE,accel=$ACCEL -device $DEVICE'
+
+#TODO: represent the whitelist from device-crash-test script somehow

new file mode 100644
index 0000000000..95aa5c9f5b
@@ -0,0 +1,3 @@
+# Just check if -device ...,help works:
+command-line: '$QEMU -device $DEVICE,help'
+qmp: false

new file mode 100644
index 0000000000..f5b10aff9a
@@ -0,0 +1,23 @@
+# This test specification is equivalent to "device/introspect/concrete" in
+# tests/device-introspect-test.c
+command-line: '$QEMU -S -machine $MACHINE,accel=$ACCEL'
+monitor-commands:
+- qmp:
+  - execute: 'device-list-properties'
+    arguments:
+      typename: '$DEVICE'
+- hmp: 'device_add $DEVICE,help'
+- hmp: 'info qom-tree'
+defaults:
+  MACHINE: 'none'
+  ACCEL: 'kvm:tcg'
+expected-failures:
+  - MACHINE: 'xenpv'
+  - MACHINE: 'xenfv'
+
+#TODO: the test runner could support something like:
+# extra-cases:
+#   # like "device/introspect/none":
+#   - DEVICE: 'nonexistent'
+#   # like "device/introspect/abstract":
+#   - DEVICE: 'device'

new file mode 100644
index 0000000000..ac0004c05d
@@ -0,0 +1,10 @@
+# this test specification is equivalent to the
+# "device/introspect/list" test case in device-introspect-test.c
+command-line: '$QEMU -nodefaults -machine none'
+monitor-commands:
+- qmp:
+  - execute: qom-list-types
+    arguments:
+      implements: 'device'
+      abstract: true
+- hmp: 'device_add help'

new file mode 100644
index 0000000000..84ec8d7090
@@ -0,0 +1,3 @@
+# just run $QEMU -help and ensure it won't crash
+command-line: '$QEMU -help'
+qmp: false