KVM test: replace 'env' dict with an Env object
cover KVM test: replace 'env' dict with an Env object
This is a respin of Eduardo's patchset with small modifications.
Differences from Eduardo's patches:
- These patches don't change the module structure of the KVM test (no kvm package is created). We can still do that in the future if we choose to.
- The class is named Env instead of KvmEnv.
- It's defined in kvm_utils.py instead of a module of its own.
- No unittests for Env. I personally don't think they're necessary, nor do I think anyone will bother to use them, but let me know if you disagree.
- The __init__() method takes care of the loading if a filename is provided.
- save() can do what _dump() did, so _dump() is unnecessary.
- The series is a lot shorter and will hopefully be easier to review.
Michael Goldish (4):
KVM test: introduce the Env class
KVM test: use kvm_utils.Env wherever appropriate
KVM test: use the new Env methods instead of the old env_* functions KVM test: get rid of the kvm_utils.env_* functions
client/tests/kvm/kvm.py | 12 +- client/tests/kvm/kvm_preprocessing.py | 12 +- client/tests/kvm/kvm_scheduler.py | 4 +- client/tests/kvm/kvm_test_utils.py | 4 +- client/tests/kvm/kvm_utils.py | 165 +++++++++++++++-------------- client/tests/kvm/tests/ksm_overcommit.py | 4 +- client/tests/kvm/tests/qemu_img.py | 6 +- client/tests/kvm/tests/stepmaker.py | 2 +- client/tests/kvm/tests/steps.py | 2 +- client/tests/kvm/tests/stress_boot.py | 2 +- client/tests/kvm/tests/unittest.py | 2 +- client/tests/kvm/tests/virtio_console.py | 2 +- 12 files changed, 111 insertions(+), 106 deletions(-)
Signature · 1 line
1.7.3.4
PATCH 1/4 KVM test: introduce the Env class
It's a wrapper to the 'env' object used by KVM tests. It behaves like a dictionary, but may implement additional common operations used by KVM tests.
--- client/tests/kvm/kvm_utils.py | 85 +++++++++++++++++++++++++++++++++++++++++ 1 files changed, 85 insertions(+), 0 deletions(-)
client/tests/kvm/kvm_utils.py +85 -0
@@ -62,6 +62,91 @@ def load_env(filename, version): return default +class Env(UserDict.IterableUserDict): + """ + A dict-like object containing global objects used by tests. + """ + def __init__(self, filename=None, version=0): + """ + Create an empty Env object or load an existing one from a file. + + If the version recorded in the file is lower than version, or if some + error occurs during unpickling, or if filename is not supplied, + create an empty Env object. + + @param filename: Path to an env file. + @param version: Required env version (int). + """ + UserDict.IterableUserDict.__init__(self) + empty = {"version": version} + if filename: + self._filename = filename + try: + f = open(filename, "r") + env = cPickle.load(f) + f.close() + if env.get("version", 0) >= version: + self.data = env + else: + logging.warn("Incompatible env file found. Not using it.") + self.data = empty + # Almost any exception can be raised during unpickling, so let's + # catch them all + except Exception, e: + logging.warn(e) + self.data = empty + else: + self.data = empty + + + def save(self, filename=None): + """ + Pickle the contents of the Env object into a file. + + @param filename: Filename to pickle the dict into. If not supplied, + use the filename from which the dict was loaded. + """ + filename = filename or self._filename + f = open(filename, "w") + cPickle.dump(self.data, f) + f.close() + + + def get_all_vms(self): + """ + Return a list of all VM objects in this Env object. + """ + return [o for o in self.values() if is_vm(o)] + + + def get_vm(self, name): + """ + Return a VM object by its name. + + @param name: VM name. + """ + return self.get("vm__%s" % name) + + + def register_vm(self, name, vm): + """ + Register a VM in this Env object. + + @param name: VM name. + @param vm: VM object. + """ + self["vm__%s" % name] = vm + + + def unregister_vm(self, name): + """ + Remove a given VM. + + @param name: VM name. + """ + del self["vm__%s" % name] + + class Params(UserDict.IterableUserDict): """ A dict-like object passed to every test.
Signature · 1 line
1.7.3.4
PATCH 2/4 KVM test: use kvm_utils.Env wherever appropriate
In reply to Michael GoldishThe new Env object is backward compatible with the regular env dict, so all the kvm_utils.env_*() functions should work as usual.
--- client/tests/kvm/kvm.py | 10 +++++----- client/tests/kvm/kvm_scheduler.py | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-)
client/tests/kvm/kvm.py +5 -5
@@ -43,7 +43,7 @@ class kvm(test.test): logging.info("Unpickling env. You may see some harmless error " "messages.") env_filename = os.path.join(self.bindir, params.get("env", "env")) - env = kvm_utils.load_env(env_filename, self.env_version) + env = kvm_utils.Env(env_filename, self.env_version) test_passed = False
@@ -68,13 +68,13 @@ class kvm(test.test): try: kvm_preprocessing.preprocess(self, params, env) finally: - kvm_utils.dump_env(env, env_filename) + env.save() # Run the test function run_func = getattr(test_module, "run_%s" % t_type) try: run_func(self, params, env) finally: - kvm_utils.dump_env(env, env_filename) + env.save() test_passed = True except Exception, e:
@@ -85,7 +85,7 @@ class kvm(test.test): kvm_preprocessing.postprocess_on_error( self, params, env) finally: - kvm_utils.dump_env(env, env_filename) + env.save() raise finally:
@@ -99,7 +99,7 @@ class kvm(test.test): logging.error("Exception raised during " "postprocessing: %s", e) finally: - kvm_utils.dump_env(env, env_filename) + env.save() except Exception, e: if params.get("abort_on_error") != "yes":
client/tests/kvm/kvm_scheduler.py +2 -2
@@ -74,13 +74,13 @@ class scheduler: # The scheduler wants this worker to free its used resources elif cmd[0] == "cleanup": env_filename = os.path.join(self.bindir, self_dict["env"]) - env = kvm_utils.load_env(env_filename, {}) + env = kvm_utils.Env(env_filename) for obj in env.values(): if isinstance(obj, kvm_vm.VM): obj.destroy() elif isinstance(obj, kvm_subprocess.Spawn): obj.close() - kvm_utils.dump_env(env, env_filename) + env.save() w.write("cleanup_done\n") w.write("ready\n")
Signature · 1 line
1.7.3.4
PATCH 3/4 KVM test: use the new Env methods instead of the old env_* functions
In reply to Michael Goldish--- client/tests/kvm/kvm.py | 2 +- client/tests/kvm/kvm_preprocessing.py | 12 ++++++------ client/tests/kvm/kvm_test_utils.py | 4 ++-- client/tests/kvm/tests/ksm_overcommit.py | 4 ++-- client/tests/kvm/tests/qemu_img.py | 6 +++--- client/tests/kvm/tests/stepmaker.py | 2 +- client/tests/kvm/tests/steps.py | 2 +- client/tests/kvm/tests/stress_boot.py | 2 +- client/tests/kvm/tests/unittest.py | 2 +- client/tests/kvm/tests/virtio_console.py | 2 +- 10 files changed, 19 insertions(+), 19 deletions(-)
client/tests/kvm/kvm.py +1 -1
@@ -106,7 +106,7 @@ class kvm(test.test): raise # Abort on error logging.info("Aborting job (%s)", e) - for vm in kvm_utils.env_get_all_vms(env): + for vm in env.get_all_vms(): if vm.is_dead(): continue logging.info("VM '%s' is alive.", vm.name)
client/tests/kvm/kvm_preprocessing.py +6 -6
@@ -50,13 +50,13 @@ def preprocess_vm(test, params, env, name): @param name: The name of the VM object. """ logging.debug("Preprocessing VM '%s'..." % name) - vm = kvm_utils.env_get_vm(env, name) + vm = env.get_vm(name) if vm: logging.debug("VM object found in environment") else: logging.debug("VM object does not exist; creating it") vm = kvm_vm.VM(name, params, test.bindir, env.get("address_cache")) - kvm_utils.env_register_vm(env, name, vm) + env.register_vm(name, vm) start_vm = False
@@ -122,7 +122,7 @@ def postprocess_vm(test, params, env, name): @param name: The name of the VM object. """ logging.debug("Postprocessing VM '%s'..." % name) - vm = kvm_utils.env_get_vm(env, name) + vm = env.get_vm(name) if vm: logging.debug("VM object found in environment") else:
@@ -338,7 +338,7 @@ def postprocess(test, params, env): if params.get("kill_unresponsive_vms") == "yes": logging.debug("'kill_unresponsive_vms' specified; killing all VMs " "that fail to respond to a remote login request...") - for vm in kvm_utils.env_get_all_vms(env): + for vm in env.get_all_vms(): if vm.is_alive(): session = vm.remote_login() if session:
@@ -350,7 +350,7 @@ def postprocess(test, params, env): kvm_subprocess.kill_tail_threads() # Terminate tcpdump if no VMs are alive - living_vms = [vm for vm in kvm_utils.env_get_all_vms(env) if vm.is_alive()] + living_vms = [vm for vm in env.get_all_vms() if vm.is_alive()] if not living_vms and "tcpdump" in env: env["tcpdump"].close() del env["tcpdump"]
@@ -408,7 +408,7 @@ def _take_screendumps(test, params, env): cache = {} while True: - for vm in kvm_utils.env_get_all_vms(env): + for vm in env.get_all_vms(): if not vm.is_alive(): continue try:
client/tests/kvm/kvm_test_utils.py +2 -2
@@ -35,7 +35,7 @@ def get_living_vm(env, vm_name): @param vm_name: Name of the desired VM object. @return: A VM object. """ - vm = kvm_utils.env_get_vm(env, vm_name) + vm = env.get_vm(vm_name) if not vm: raise error.TestError("VM '%s' not found in environment" % vm_name) if not vm.is_alive():
@@ -275,7 +275,7 @@ def migrate(vm, env=None, mig_timeout=3600, mig_protocol="tcp", # Replace the source VM with the new cloned VM if (dest_host == 'localhost') and (env is not None): - kvm_utils.env_register_vm(env, vm.name, dest_vm) + env.register_vm(vm.name, dest_vm) # Return the new cloned VM if dest_host == 'localhost':
client/tests/kvm/tests/ksm_overcommit.py +2 -2
@@ -544,7 +544,7 @@ def run_ksm_overcommit(test, params, env): # Creating the first guest kvm_preprocessing.preprocess_vm(test, params, env, vm_name) - lvms.append(kvm_utils.env_get_vm(env, vm_name)) + lvms.append(env.get_vm(vm_name)) if not lvms[0]: raise error.TestError("VM object not found in environment") if not lvms[0].is_alive():
@@ -575,7 +575,7 @@ def run_ksm_overcommit(test, params, env): # Last VM is later used to run more allocators simultaneously lvms.append(lvms[0].clone(vm_name, params)) - kvm_utils.env_register_vm(env, vm_name, lvms[i]) + env.register_vm(vm_name, lvms[i]) params['vms'] += " " + vm_name logging.debug("Booting guest %s" % lvms[i].name)
client/tests/kvm/tests/qemu_img.py +3 -3
@@ -289,7 +289,7 @@ def run_qemu_img(test, params, env): # Start a new VM, using backing file as its harddisk vm_name = params.get('main_vm') kvm_preprocessing.preprocess_vm(test, params, env, vm_name) - vm = kvm_utils.env_get_vm(env, vm_name) + vm = env.get_vm(vm_name) vm.create() timeout = int(params.get("login_timeout", 360)) session = kvm_test_utils.wait_for_login(vm, timeout=timeout)
@@ -317,7 +317,7 @@ def run_qemu_img(test, params, env): # Here, the commit_testfile should not exist vm_name = params.get('main_vm') kvm_preprocessing.preprocess_vm(test, params, env, vm_name) - vm = kvm_utils.env_get_vm(env, vm_name) + vm = env.get_vm(vm_name) vm.create() timeout = int(params.get("login_timeout", 360)) session = kvm_test_utils.wait_for_login(vm, timeout=timeout)
@@ -343,7 +343,7 @@ def run_qemu_img(test, params, env): # Start a new VM, using image_name as its harddisk vm_name = params.get('main_vm') kvm_preprocessing.preprocess_vm(test, params, env, vm_name) - vm = kvm_utils.env_get_vm(env, vm_name) + vm = env.get_vm(vm_name) vm.create() timeout = int(params.get("login_timeout", 360)) session = kvm_test_utils.wait_for_login(vm, timeout=timeout)
client/tests/kvm/tests/stepmaker.py +1 -1
@@ -337,7 +337,7 @@ class StepMaker(stepeditor.StepMakerWindow): def run_stepmaker(test, params, env): - vm = kvm_utils.env_get_vm(env, params.get("main_vm")) + vm = env.get_vm(params.get("main_vm")) if not vm: raise error.TestError("VM object not found in environment") if not vm.is_alive():
client/tests/kvm/tests/steps.py +1 -1
@@ -181,7 +181,7 @@ def barrier_2(vm, words, params, debug_dir, data_scrdump_filename, def run_steps(test, params, env): - vm = kvm_utils.env_get_vm(env, params.get("main_vm")) + vm = env.get_vm(params.get("main_vm")) if not vm: raise error.TestError("VM object not found in environment") if not vm.is_alive():
client/tests/kvm/tests/stress_boot.py +1 -1
@@ -36,7 +36,7 @@ def run_stress_boot(tests, params, env): vm_name = "vm" + str(num) vm_params = vm.get_params().copy() curr_vm = vm.clone(vm_name, vm_params) - kvm_utils.env_register_vm(env, vm_name, curr_vm) + env.register_vm(vm_name, curr_vm) logging.info("Booting guest #%d" % num) kvm_preprocessing.preprocess_vm(tests, vm_params, env, vm_name) params['vms'] += " " + vm_name
client/tests/kvm/tests/unittest.py +1 -1
@@ -88,7 +88,7 @@ def run_unittest(test, params, env): try: vm_name = params.get('main_vm') kvm_preprocessing.preprocess_vm(test, params, env, vm_name) - vm = kvm_utils.env_get_vm(env, vm_name) + vm = env.get_vm(vm_name) vm.create() vm.monitor.cmd("cont") logging.info("Waiting for unittest %s to complete, timeout %s, "
client/tests/kvm/tests/virtio_console.py +1 -1
@@ -583,7 +583,7 @@ def run_virtio_console(test, params, env): kvm_preprocessing.preprocess_vm(test, params, env, params.get("main_vm")) - vm = kvm_utils.env_get_vm(env, params.get("main_vm")) + vm = env.get_vm(params.get("main_vm")) session = kvm_test_utils.wait_for_login(vm, 0, float(params.get("boot_timeout", 240)),
Signature · 1 line
1.7.3.4
PATCH 4/4 KVM test: get rid of the kvm_utils.env_* functions
In reply to Michael GoldishAlso, move is_vm() closer to the definition of Env.
--- client/tests/kvm/kvm_utils.py | 88 ++--------------------------------------- 1 files changed, 4 insertions(+), 84 deletions(-)
client/tests/kvm/kvm_utils.py +4 -84
@@ -27,39 +27,13 @@ def _unlock_file(f): f.close() -def dump_env(obj, filename): - """ - Dump KVM test environment to a file. - - @param filename: Path to a file where the environment will be dumped to. - """ - file = open(filename, "w") - cPickle.dump(obj, file) - file.close() - - -def load_env(filename, version): +def is_vm(obj): """ - Load KVM test environment from an env file. - If the version recorded in the file is lower than version, return an empty - env. If some other error occurs during unpickling, return an empty env. + Tests whether a given object is a VM object. - @param filename: Path to an env file. + @param obj: Python object. """ - default = {"version": version} - try: - file = open(filename, "r") - env = cPickle.load(file) - file.close() - if env.get("version", 0) < version: - logging.warn("Incompatible env file found. Not using it.") - return default - return env - # Almost any exception can be raised during unpickling, so let's catch - # them all - except Exception, e: - logging.warn(e) - return default + return obj.__class__.__name__ == "VM" class Env(UserDict.IterableUserDict):
@@ -326,60 +300,6 @@ def verify_ip_address_ownership(ip, macs, timeout=10.0): return bool(regex.search(o)) -# Functions for working with the environment (a dict-like object) - -def is_vm(obj): - """ - Tests whether a given object is a VM object. - - @param obj: Python object (pretty much everything on python). - """ - return obj.__class__.__name__ == "VM" - - -def env_get_all_vms(env): - """ - Return a list of all VM objects on a given environment. - - @param env: Dictionary with environment items. - """ - vms = [] - for obj in env.values(): - if is_vm(obj): - vms.append(obj) - return vms - - -def env_get_vm(env, name): - """ - Return a VM object by its name. - - @param name: VM name. - """ - return env.get("vm__%s" % name) - - -def env_register_vm(env, name, vm): - """ - Register a given VM in a given env. - - @param env: Environment where we will register the VM. - @param name: VM name. - @param vm: VM object. - """ - env["vm__%s" % name] = vm - - -def env_unregister_vm(env, name): - """ - Remove a given VM from a given env. - - @param env: Environment where we will un-register the VM. - @param name: VM name. - """ - del env["vm__%s" % name] - - # Utility functions for dealing with external processes def find_command(cmd):
Signature · 1 line
1.7.3.4
Quoted 12 lines · Michael Goldish
This is a respin of Eduardo's patchset with small modifications.
Differences from Eduardo's patches:
- These patches don't change the module structure of the KVM test (no kvm package is created). We can still do that in the future if we choose to.
- The class is named Env instead of KvmEnv.
- It's defined inkvm_utils.pyinstead of a module of its own.
- No unittests for Env. I personally don't think they're necessary, nor do I think anyone will bother to use them, but let me know if you disagree.
- The__init__()method takes care of the loading if a filename is provided.
-save()can do what_dump()did, so_dump()is unnecessary.
- The series is a lot shorter and will hopefully be easier to review.
Hi Michael, thanks! The only thing I do not agree is with the unittest, it's useful, specially for me when I am applying and checking the patches, so I ask you to send the unittest Eduardo created on a future patch. Other than that, great work!
Quoted 20 lines
Michael Goldish (4):
KVM test: introduce the Env class
KVM test: use kvm_utils.Env wherever appropriate
KVM test: use the new Env methods instead of the old env_* functions KVM test: get rid of the kvm_utils.env_* functionsclient/tests/kvm/kvm.py | 12 +- client/tests/kvm/kvm_preprocessing.py | 12 +- client/tests/kvm/kvm_scheduler.py | 4 +- client/tests/kvm/kvm_test_utils.py | 4 +- client/tests/kvm/kvm_utils.py | 165 +++++++++++++++-------------- client/tests/kvm/tests/ksm_overcommit.py | 4 +- client/tests/kvm/tests/qemu_img.py | 6 +- client/tests/kvm/tests/stepmaker.py | 2 +- client/tests/kvm/tests/steps.py | 2 +- client/tests/kvm/tests/stress_boot.py | 2 +- client/tests/kvm/tests/unittest.py | 2 +- client/tests/kvm/tests/virtio_console.py | 2 +- 12 files changed, 111 insertions(+), 106 deletions(-)