KVM test: Include the preprocessing param migration_mode
PATCH 1/5 KVM test: Include the preprocessing param migration_mode
In order to include multi-host migration, add a pre-processor parameter that will make the test start a VM in incoming migration mode:
migration_mode = tcp
Will make the preprocessor start the VM in tcp -incoming mode.
--- client/tests/kvm/kvm_preprocessing.py | 16 +++++++++++++--- 1 files changed, 13 insertions(+), 3 deletions(-)
client/tests/kvm/kvm_preprocessing.py +13 -3
@@ -60,6 +60,8 @@ def preprocess_vm(test, params, env, name): start_vm = False + migration_mode = params.get("migration_mode", None) + if params.get("restart_vm") == "yes": logging.debug("'restart_vm' specified; (re)starting VM...") start_vm = True
@@ -72,11 +74,19 @@ def preprocess_vm(test, params, env, name): logging.debug("VM's qemu command differs from requested one; " "restarting it...") start_vm = True + elif migration_mode is not None: + logging.debug("Starting VM on migration incoming mode...") + start_vm = True if start_vm: - # Start the VM (or restart it if it's already up) - if not vm.create(name, params, test.bindir): - raise error.TestError("Could not start VM") + if migration_mode is not None: + if not vm.create(name, params, test.bindir, + migration_mode=migration_mode): + raise error.TestError("Could not start VM for migration") + else: + # Start the VM (or restart it if it's already up) + if not vm.create(name, params, test.bindir): + raise error.TestError("Could not start VM") else: # Don't start the VM, just update its params vm.params = params
Signature · 1 line
1.7.3.4
PATCH 2/5 KVM test: Introduce migration_multi_host test
This test will carry out the migration between guests in two different hosts. Depending on the test 'role', different actions will be taken, and those actions will be synchronized through the autotest barrier mechanism.
--- client/tests/kvm/tests/migration_multi_host.py | 108 ++++++++++++++++++++++++ client/tests/kvm/tests_base.cfg.sample | 13 +++ 2 files changed, 121 insertions(+), 0 deletions(-) create mode 100644 client/tests/kvm/tests/migration_multi_host.py
client/tests/kvm/tests/migration_multi_host.py new file +108 -0
@@ -0,0 +1,108 @@ +import logging, time, socket +from autotest_lib.client.common_lib import error +import kvm_subprocess, kvm_test_utils, kvm_utils + + +def run_migration_multi_host(test, params, env): + """ + KVM multi-host migration test: + + Migration execution progress: + + source host dest host + ---------------------------------------------------------------------------- + log into guest + ---------------------------------------------------------------------------- + start socket server + + wait 30 secs -------------------- wait login_timeout+30 secs --------------- + + accept connection connect to socket server,send mig_port + ---------------------------------------------------------------------------- + start migration + + wait 30 secs -------------------- wait mig_timeout+30 secs ----------------- + + try to log into migrated guest check VM's status via monitor cmd + ---------------------------------------------------------------------------- + + @param test: kvm test object. + @param params: Dictionary with test parameters. + @param env: Dictionary with the test environment. + """ + def guest_active(vm): + o = vm.monitor.info("status") + if isinstance(o, str): + return "status: running" in o + else: + return o.get("status") == "running" + + vm = kvm_test_utils.get_living_vm(env, params.get("main_vm")) + login_timeout = int(params.get("login_timeout", 360)) + role = params.get("role") + srchost = params.get("srchost") + dsthost = params.get("dsthost") + mig_timeout = int(params.get("mig_timeout")) + # Port used to communicate info between source and destination + comm_port = int(params.get("comm_port", 12324)) + regain_ip_cmd = params.get("regain_ip_cmd", "dhclient") + if role == 'source': + session = kvm_test_utils.wait_for_login(vm, timeout=login_timeout) + + # Listen on a port to get the migration port received from + # dest machine + s_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s_socket.bind(('', comm_port)) + s_socket.listen(1) + + # Wait 30 seconds for source and dest to reach this point + test.job.barrier(srchost,'socket_started', 30).rendezvous(srchost, + dsthost) + + c_socket, addr = s_socket.accept() + mig_port = int(c_socket.recv(6)) + logging.info("Received from destination the migration port %s", + mig_port) + c_socket.close() + + logging.info("Start migrating now...") + kvm_test_utils.migrate(vm=vm, dest_host=dsthost, mig_port=mig_port, + env=env) + + # Wait up to 30 seconds for dest to reach this point + test.job.barrier(srchost, 'mig_finished', 30).rendezvous(srchost, + dsthost) + + elif role == 'destination': + # Wait up to login_timeout + 30 seconds for the source to + # reach this point + test.job.barrier(dsthost, 'socket_started', + login_timeout + 30).rendezvous(srchost, + dsthost) + + c_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + c_socket.connect((srchost, comm_port)) + logging.info("Communicating to source migration port %s", + vm.migration_port) + c_socket.send("%d" % vm.migration_port) + c_socket.close() + + # Wait up to mig_timeout + 30 seconds for the source to + # reach this point: migration finished + test.job.barrier(dsthost, 'mig_finished', + mig_timeout + 30).rendezvous(srchost, + dsthost) + + if not guest_active(vm): + raise error.TestFail("Guest not active after migration") + + logging.info("Migrated guest appears to be running") + + # Log into the guest again + logging.info("Logging into migrated guest after migration...") + session_serial = kvm_test_utils.wait_for_login(vm, timeout=login_timeout, serial=True) + session_serial.cmd(regain_ip_cmd) + session = kvm_test_utils.wait_for_login(vm, timeout=login_timeout) + + else: + raise error.TestError('Invalid role specified')
client/tests/kvm/tests_base.cfg.sample +13 -0
@@ -159,6 +159,19 @@ variants: iterations = 1 type = migration_with_file_transfer + - migrate_multi_host: install setup unattended_install.cdrom + type = migration_multi_host + migration_test_command = help + migration_bg_command = "cd /tmp; nohup tcpdump -q -t ip host localhost" + migration_bg_check_command = pgrep tcpdump + migration_bg_kill_command = pkill tcpdump + kill_vm_on_error = yes + iterations = 2 + used_mem = 1024 + mig_timeout = 3600 + comm_port = 13234 + regain_ip_cmd = dhclient + - boot_savevm: install setup unattended_install.cdrom type = boot_savevm savevm_delay = 0.3
Signature · 1 line
1.7.3.4
PATCH 3/5 KVM test: Add migration server control file
In reply to Lucas Meneghel RodriguesIntroduce the server migration control file, that makes possible to run the multi host migration. It parses the base config file, restricts it with appropriate parameters, generates the test dicts, modify the test_dicts so there's a distinction between the migration roles ('destination' or 'source').
--- client/tests/kvm/migration_control.srv | 122 ++++++++++++++++++++++++++++++++ 1 files changed, 122 insertions(+), 0 deletions(-) create mode 100644 client/tests/kvm/migration_control.srv
client/tests/kvm/migration_control.srv new file +122 -0
@@ -0,0 +1,122 @@ +AUTHOR = "Yolkfull Chow <[email protected]>" +TIME = "SHORT" +NAME = "Migration across multiple hosts" +TEST_CATEGORY = "Functional" +TEST_CLASS = "Virtualization" +TEST_TYPE = "Server" +DOC = """ +Migrate KVM guest between two hosts. It parses the base config file, restricts +it with appropriate parameters, generates the test dicts, modify the test_dicts +so there's a distinction between the migration roles ('dest' or 'source'). +""" + +import sys, os, commands, glob, shutil, logging, random +from autotest_lib.server import utils + +# Specify the directory of autotest before you start this test +AUTOTEST_DIR = '/usr/local/autotest' + +# Specify the root directory that on client machines +rootdir = '/tmp/kvm_autotest_root' + +# Make possible to import the KVM test APIs +KVM_DIR = os.path.join(AUTOTEST_DIR, 'client/tests/kvm') +sys.path.append(KVM_DIR) + +import common, kvm_config + +def generate_mac_address(): + r = random.SystemRandom() + mac = "9a:%02x:%02x:%02x:%02x:%02x" % (r.randint(0x00, 0xff), + r.randint(0x00, 0xff), + r.randint(0x00, 0xff), + r.randint(0x00, 0xff), + r.randint(0x00, 0xff)) + return mac + + +def run(pair): + logging.info("KVM migration running on source host [%s] and destination " + "host [%s]\n", pair[0], pair[1]) + + source = hosts.create_host(pair[0]) + dest = hosts.create_host(pair[1]) + source_at = autotest.Autotest(source) + dest_at = autotest.Autotest(dest) + + cfg_file = os.path.join(KVM_DIR, "tests_base.cfg") + + if not os.path.exists(cfg_file): + raise error.JobError("Config file %s was not found", cfg_file) + + # Get test set (dictionary list) from the configuration file + cfg = kvm_config.config() + test_variants = """ +image_name(_.*)? ?<= /tmp/kvm_autotest_root/images/ +cdrom(_.*)? ?<= /tmp/kvm_autotest_root/ +floppy ?<= /tmp/kvm_autotest_root/ +Linux: + unattended_install: + kernel ?<= /tmp/kvm_autotest_root/ + initrd ?<= /tmp/kvm_autotest_root/ +qemu_binary = /usr/libexec/qemu-kvm +qemu_img_binary = /usr/bin/qemu-img +only qcow2 +only virtio_net +only virtio_blk +only smp2 +only no_pci_assignable +only smallpages +only Fedora.13.64 +only migrate_multi_host +nic_mode = tap +nic_mac_nic1 = %s +""" % (generate_mac_address()) + cfg.fork_and_parse(cfg_file, test_variants) + test_dicts = cfg.get_list() + + source_control_file = dest_control_file = """ +kvm_test_dir = os.path.join(os.environ['AUTODIR'],'tests/kvm') +sys.path.append(kvm_test_dir)\n +""" + for params in test_dicts: + params['srchost'] = source.ip + params['dsthost'] = dest.ip + params['rootdir'] = rootdir + + source_params = params.copy() + source_params['role'] = "source" + + dest_params = params.copy() + dest_params['role'] = "destination" + dest_params['migration_mode'] = "tcp" + + # Report the parameters we've received + print "Test parameters:" + keys = params.keys() + keys.sort() + for key in keys: + logging.debug(" %s = %s", key, params[key]) + + source_control_file += "job.run_test('kvm', tag='%s', params=%s)" % (source_params['shortname'], source_params) + dest_control_file += "job.run_test('kvm', tag='%s', params=%s)" % (dest_params['shortname'], dest_params) + + logging.info('Source control file:\n%s', source_control_file) + logging.info('Destination control file:\n%s', dest_control_file) + dest_command = subcommand(dest_at.run, + [dest_control_file, dest.hostname]) + + source_command = subcommand(source_at.run, + [source_control_file, source.hostname]) + + parallel([dest_command, source_command]) + +# Grab the pairs (and failures) +(pairs, failures) = utils.form_ntuples_from_machines(machines, 2) + +# Log the failures +for failure in failures: + job.record("FAIL", failure[0], "kvm", failure[1]) + +# Now run through each pair and run +job.parallel_simple(run, pairs, log=False)
Signature · 1 line
1.7.3.4
PATCH 4/5 KVM test: Modifications on the migrate utility function
In reply to Lucas Meneghel RodriguesIn order to accomodate multi-host migration, make migrate accept 2 additional parameters, dest_host and mig_port. Also, handle properly the case where we are doing migration inside the same host.
--- client/tests/kvm/kvm_test_utils.py | 50 +++++++++++++++++++++++------------ 1 files changed, 33 insertions(+), 17 deletions(-)
client/tests/kvm/kvm_test_utils.py +33 -17
@@ -134,7 +134,7 @@ def reboot(vm, session, method="shell", sleep_before_reset=10, nic_index=0, def migrate(vm, env=None, mig_timeout=3600, mig_protocol="tcp", mig_cancel=False, offline=False, stable_check=False, - clean=False, save_path=None): + clean=False, save_path=None, dest_host='localhost', mig_port=None): """ Migrate a VM locally and re-register it in the environment.
@@ -144,7 +144,10 @@ def migrate(vm, env=None, mig_timeout=3600, mig_protocol="tcp", @param mig_timeout: timeout value for migration. @param mig_protocol: migration protocol @param mig_cancel: Test migrate_cancel or not when protocol is tcp. - @return: The post-migration VM. + @param dest_host: Destination host (defaults to 'localhost'). + @param mig_port: Port that will be used for migration. + @return: The post-migration VM, in case of same host migration, True in + case of multi-host migration. """ def mig_finished(): o = vm.monitor.info("migrate")
@@ -182,18 +185,25 @@ def migrate(vm, env=None, mig_timeout=3600, mig_protocol="tcp", raise error.TestFail("Timeout expired while waiting for migration " "to finish") - dest_vm = vm.clone() - if stable_check: + if dest_host == 'localhost': + dest_vm = vm.clone() + + if (dest_host == 'localhost') and stable_check: # Pause the dest vm after creation dest_vm.params['extra_params'] = (dest_vm.params.get('extra_params','') + ' -S') - if not dest_vm.create(migration_mode=mig_protocol, mac_source=vm): - raise error.TestError("Could not create dest VM") + if dest_host == 'localhost': + if not dest_vm.create(migration_mode=mig_protocol, mac_source=vm): + raise error.TestError("Could not create dest VM") + try: try: if mig_protocol == "tcp": - uri = "tcp:localhost:%d" % dest_vm.migration_port + if dest_host == 'localhost': + uri = "tcp:localhost:%d" % dest_vm.migration_port + else: + uri = 'tcp:%s:%d' % (dest_host, mig_port) elif mig_protocol == "unix": uri = "unix:%s" % dest_vm.migration_file elif mig_protocol == "exec":
@@ -212,11 +222,12 @@ def migrate(vm, env=None, mig_timeout=3600, mig_protocol="tcp", raise error.TestFail("Failed to cancel migration") if offline: vm.monitor.cmd("cont") - dest_vm.destroy(gracefully=False) + if dest_host == 'localhost': + dest_vm.destroy(gracefully=False) return vm else: wait_for_migration() - if stable_check: + if (dest_host == 'localhost') and stable_check: save_path = None or "/tmp" save1 = os.path.join(save_path, "src") save2 = os.path.join(save_path, "dst")
@@ -231,14 +242,15 @@ def migrate(vm, env=None, mig_timeout=3600, mig_protocol="tcp", raise error.TestFail("Mismatch of VM state before " "and after migration") - if offline: + if (dest_host == 'localhost') and offline: dest_vm.monitor.cmd("cont") except: - dest_vm.destroy() + if dest_host == 'localhost': + dest_vm.destroy() raise finally: - if stable_check and clean: + if (dest_host == 'localhost') and stable_check and clean: logging.debug("Cleaning the state files") if os.path.isfile(save1): os.remove(save1)
@@ -253,19 +265,23 @@ def migrate(vm, env=None, mig_timeout=3600, mig_protocol="tcp", else: raise error.TestFail("Migration ended with unknown status") - if "paused" in dest_vm.monitor.info("status"): - logging.debug("Destination VM is paused, resuming it...") - dest_vm.monitor.cmd("cont") + if dest_host == 'localhost': + if "paused" in dest_vm.monitor.info("status"): + logging.debug("Destination VM is paused, resuming it...") + dest_vm.monitor.cmd("cont") # Kill the source VM vm.destroy(gracefully=False) # Replace the source VM with the new cloned VM - if env is not None: + if (dest_host == 'localhost') and (env is not None): kvm_utils.env_register_vm(env, vm.name, dest_vm) # Return the new cloned VM - return dest_vm + if dest_host == 'localhost': + return dest_vm + else: + return vm def stop_windows_service(session, service, timeout=120):
Signature · 1 line
1.7.3.4
PATCH 5/5 KVM test: kvm_vm: Allow NIC MACs to be defined on config file
In reply to Lucas Meneghel RodriguesIn order to allow the multi-host migration test to start VMs with the same MAC address, allow nic addresses to be defined on the config file, such as:
nic_mac_nic1 = AA:BB:CC:DD:EE
This way we can generate a MAC on the server control file and distribute this to the new VMs that are going to be created.
--- client/tests/kvm/kvm_vm.py | 14 +++++++++++--- 1 files changed, 11 insertions(+), 3 deletions(-)
client/tests/kvm/kvm_vm.py +11 -3
@@ -598,14 +598,22 @@ class VM: self.uuid = f.read().strip() f.close() + + # Generate or copy MAC addresses for all NICs num_nics = len(kvm_utils.get_sub_dict_names(params, "nics")) for vlan in range(num_nics): - mac = mac_source and mac_source.get_mac_address(vlan) - if mac: + nic_name = kvm_utils.get_sub_dict_names(params, "nics")[vlan] + nic_params = kvm_utils.get_sub_dict(params, nic_name) + if nic_params.get("nic_mac", None): + mac = nic_params.get("nic_mac") kvm_utils.set_mac_address(self.instance, vlan, mac) else: - kvm_utils.generate_mac_address(self.instance, vlan) + mac = mac_source and mac_source.get_mac_address(vlan) + if mac: + kvm_utils.set_mac_address(self.instance, vlan, mac) + else: + kvm_utils.generate_mac_address(self.instance, vlan) # Assign a PCI assignable device self.pci_assignable = None
Signature · 1 line
1.7.3.4