kvm_subprocess, guestwizard improvements, timedrift and other small things
cover kvm_subprocess, guestwizard improvements, timedrift and other small things
The following patch series includes all the patches I sent that have not been applied yet. They are all rebased against the latest HEAD (including the recent UUID patch).
Some new things are included too, such as a simple time drift test for Windows.
Sorry for posting so many commits at once.
PATCH 1/17 Add new module kvm_subprocess
This module is intended to be used for controlling all child processes in KVM
tests: both QEMU processes and SSH/SCP/Telnet processes. Processes started with this module keep running and can be interacted with even after the parent process exits.
The current run_bg() utility tracks a child process as long as the parent process is running. When the parent process exits, the tracking thread terminates and cannot resume when needed.
Currently SSH/SCP/Telnet communication is handled by kvm_utils.kvm_spawn, which does not allow the child process to run after the parent process exits. Thus, open SSH/SCP/Telnet sessions cannot be reused by tests following the one in which they are opened.
The new module provides a solution to these two problems, and also saves some code by reusing common code required both for QEMU processes and SSH/SCP/Telnet processes.
--- client/tests/kvm/kvm_subprocess.py | 1146 ++++++++++++++++++++++++++++++++++++ 1 files changed, 1146 insertions(+), 0 deletions(-) create mode 100644 client/tests/kvm/kvm_subprocess.py
client/tests/kvm/kvm_subprocess.py new file +1146 -0
@@ -0,0 +1,1146 @@ +#!/usr/bin/python +import sys, subprocess, pty, select, os, time, signal, re, termios, fcntl +import threading, logging, commands +import common, kvm_utils + +""" +A class and functions used for running and controlling child processes. + +@copyright: 2008-2009 Red Hat Inc. +""" + + +def run_bg(command, termination_func=None, output_func=None, output_prefix="", + timeout=1.0): + """ + Run command as a subprocess. Call output_func with each line of output + from the subprocess (prefixed by output_prefix). Call termination_func + when the subprocess terminates. Return when timeout expires or when the + subprocess exits -- whichever occurs first. + + @brief: Run a subprocess in the background and collect its output and + exit status. + + @param command: The shell command to execute + @param termination_func: A function to call when the process terminates + (should take an integer exit status parameter) + @param output_func: A function to call with each line of output from + the subprocess (should take a string parameter) + @param output_prefix: A string to pre-pend to each line of the output, + before passing it to stdout_func + @param timeout: Time duration (in seconds) to wait for the subprocess to + terminate before returning + + @return: A kvm_tail object. + """ + process = kvm_tail(command=command, + termination_func=termination_func, + output_func=output_func, + output_prefix=output_prefix) + + end_time = time.time() + timeout + while time.time() < end_time and process.is_alive(): + time.sleep(0.1) + + return process + + +def run_fg(command, output_func=None, output_prefix="", timeout=1.0): + """ + Run command as a subprocess. Call output_func with each line of output + from the subprocess (prefixed by prefix). Return when timeout expires or + when the subprocess exits -- whichever occurs first. If timeout expires + and the subprocess is still running, kill it before returning. + + @brief: Run a subprocess in the foreground and collect its output and + exit status. + + @param command: The shell command to execute + @param output_func: A function to call with each line of output from + the subprocess (should take a string parameter) + @param output_prefix: A string to pre-pend to each line of the output, + before passing it to stdout_func + @param timeout: Time duration (in seconds) to wait for the subprocess to + terminate before killing it and returning + + @return: A 2-tuple containing the exit status of the process and its + STDOUT/STDERR output. If timeout expires before the process + terminates, the returned status is None. + """ + process = run_bg(command, None, output_func, output_prefix, timeout) + output = process.get_output() + if process.is_alive(): + status = None + else: + status = process.get_status() + process.close() + return (status, output) + + +def _lock(filename): + if not os.path.exists(filename): + open(filename, "w").close() + fd = os.open(filename, os.O_RDWR) + fcntl.lockf(fd, fcntl.LOCK_EX) + return fd + + +def _unlock(fd): + fcntl.lockf(fd, fcntl.LOCK_UN) + os.close(fd) + + +def _locked(filename): + try: + fd = os.open(filename, os.O_RDWR) + except: + return False + try: + fcntl.lockf(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + except: + os.close(fd) + return True + fcntl.lockf(fd, fcntl.LOCK_UN) + os.close(fd) + return False + + +def _wait(filename): + fd = _lock(filename) + _unlock(fd) + + +def _get_filenames(base_dir, id): + return [os.path.join(base_dir, s + id) for s in + "shell-pid-", "status-", "output-", "inpipe-", + "lock-server-running-", "lock-client-starting-"] + + +def _get_reader_filename(base_dir, id, reader): + return os.path.join(base_dir, "outpipe-%s-%s" % (reader, id)) + + +class kvm_spawn: + """ + This class is used for spawning and controlling a child process. + + A new instance of this class can either run a new server (a small Python + program that reads output from the child process and reports it to the + client and to a text file) or attach to an already running server. + When a server is started it runs the child process. + The server writes output from the child's STDOUT and STDERR to a text file. + The text file can be accessed at any time using get_output(). + In addition, the server opens as many pipes as requested by the client and + writes the output to them. + The pipes are requested and accessed by classes derived from kvm_spawn. + These pipes are referred to as "readers". + The server also receives input from the client and sends it to the child + process. + An instance of this class can be pickled. Every derived class is + responsible for restoring its own state by properly defining + __getinitargs__(). + + The first named pipe is used by _tail(), a function that runs in the + background and reports new output from the child as it is produced. + The second named pipe is used by a set of functions that read and parse + output as requested by the user in an interactive manner, similar to + pexpect. + When unpickled it automatically + resumes _tail() if needed. + """ + + def __init__(self, command=None, id=None, echo=False, linesep="\n"): + """ + Initialize the class and run command as a child process. + + @param command: Command to run, or None if accessing an already running + server. + @param id: ID of an already running server, if accessing a running + server, or None if starting a new one. + @param echo: Boolean indicating whether echo should be initially + enabled for the pseudo terminal running the subprocess. This + parameter has an effect only when starting a new server. + @param linesep: Line separator to be appended to strings sent to the + child process by sendline(). + """ + self.id = id or kvm_utils.generate_random_string(8) + + # Define filenames for communication with server + base_dir = "/tmp/kvm_spawn" + try: + os.makedirs(base_dir) + except: + pass + (self.shell_pid_filename, + self.status_filename, + self.output_filename, + self.inpipe_filename, + self.lock_server_running_filename, + self.lock_client_starting_filename) = _get_filenames(base_dir, + self.id) + + # Remember some attributes + self.echo = echo + self.linesep = linesep + + # Make sure the 'readers' and 'close_hooks' attributes exist + if not hasattr(self, "readers"): + self.readers = [] + if not hasattr(self, "close_hooks"): + self.close_hooks = [] + + # Define the reader filenames + self.reader_filenames = dict( + (reader, _get_reader_filename(base_dir, self.id, reader)) + for reader in self.readers) + + # Let the server know a client intends to open some pipes; + # if the executed command terminates quickly, the server will wait for + # the client to release the lock before exiting + lock_client_starting = _lock(self.lock_client_starting_filename) + + # Start the server (which runs the command) + if command: + sub = subprocess.Popen("python %s" % __file__, + shell=True, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + # Send parameters to the server + sub.stdin.write("%s\n" % self.id) + sub.stdin.write("%s\n" % echo) + sub.stdin.write("%s\n" % ",".join(self.readers)) + sub.stdin.write("%s\n" % command) + # Wait for the server to complete its initialization + sub.stdout.readline() + + # Open the reading pipes + self.reader_fds = {} + try: + assert(_locked(self.lock_server_running_filename)) + for reader, filename in self.reader_filenames.items(): + self.reader_fds[reader] = os.open(filename, os.O_RDONLY) + except: + pass + + # Allow the server to continue + _unlock(lock_client_starting) + + + # The following two functions are defined to make sure the state is set + # exclusively by the constructor call as specified in __getinitargs__(). + + def __getstate__(self): + pass + + + def __setstate__(self, state): + pass + + + def __getinitargs__(self): + # Save some information when pickling -- will be passed to the + # constructor upon unpickling + return (None, self.id, self.echo, self.linesep) + + + def _add_reader(self, reader): + """ + Add a reader whose file descriptor can be obtained with _get_fd(). + Should be called before __init__(). Intended for use by derived + classes. + + @param reader: The name of the reader. + """ + if not hasattr(self, "readers"): + self.readers = [] + self.readers.append(reader) + + + def _add_close_hook(self, hook): + """ + Add a close hook function to be called when close() is called. + The function will be called after the process terminates but before + final cleanup. Intended for use by derived classes. + + @param hook: The hook function. + """ + if not hasattr(self, "close_hooks"): + self.close_hooks = [] + self.close_hooks.append(hook) + + + def _get_fd(self, reader): + """ + Return an open file descriptor corresponding to the specified reader + pipe. If no such reader exists, or the pipe could not be opened, + return None. Intended for use by derived classes. + + @param reader: The name of the reader. + """ + return self.reader_fds.get(reader) + + + def get_id(self): + """ + Return the instance's id attribute, which may be used to access the + process in the future. + """ + return self.id + + + def get_shell_pid(self): + """ + Return the PID of the subshell process, or None if not available. + The subshell is the shell that runs the command. + """ + try: + file = open(self.shell_pid_filename, "r") + pid = int(file.read()) + file.close() + return pid + except: + return None + + + def get_pid(self, index=0): + """ + Try to get and return the PID of a child process of the subshell. + This is usually the PID of the process executed in the subshell. + There are 3 exceptions: + - If the subshell couldn't start the process for some reason, no + PID can be returned. + - If the subshell is running several processes in parallel, + multiple PIDs can be returned. Use the index parameter in this + case. + - Before starting the process, after the process has terminated, + or while running shell code that doesn't start any processes -- + no PID can be returned. + + @param index: The index of the child process whose PID is requested. + Normally this should remain 0. + @return: The PID of the child process, or None if none could be found. + """ + parent_pid = self.get_shell_pid() + if not parent_pid: + return None + pids = commands.getoutput("ps --ppid %d -o pid=" % parent_pid).split() + try: + return int(pids[index]) + except: + return None + + + def get_status(self): + """ + Wait for the process to exit and return its exit status, or None + if the exit status is not available. + """ + _wait(self.lock_server_running_filename) + try: + file = open(self.status_filename, "r") + status = int(file.read()) + file.close() + return status + except: + return None + + + def get_output(self): + """ + Return the STDOUT and STDERR output of the process so far. + """ + try: + file = open(self.output_filename, "r") + output = file.read() + file.close() + return output + except: + return "" + + + def is_alive(self): + """ + Return True if the process is running. + """ + pid = self.get_shell_pid() + # See if the PID exists + try: + os.kill(pid, 0) + except: + return False + # Make sure the PID belongs to the original process + filename = "/proc/%d/cmdline" % pid + try: + file = open(filename, "r") + cmdline = file.read() + file.close() + except: + # If we couldn't find the file for some reason, skip the check + return True + if self.id in cmdline: + return True + return False + + + def close(self, sig=signal.SIGTERM): + """ + Kill the child process if it's alive and remove temporary files. + + @param sig: The signal to send the process when attempting to kill it. + """ + # Kill it if it's alive + if self.is_alive(): + try: + os.kill(self.get_shell_pid(), sig) + except: + pass + # Wait for the server to exit + _wait(self.lock_server_running_filename) + # Call all cleanup routines + for hook in self.close_hooks: + hook() + # Close reader file descriptors + for fd in self.reader_fds.values(): + try: + os.close(fd) + except: + pass + # Remove all used files + for filename in (_get_filenames("/tmp/kvm_spawn", self.id) + + self.reader_filenames.values()): + try: + os.unlink(filename) + except OSError: + pass + + + def set_linesep(self, linesep): + """ + Sets the line separator string (usually "\\n"). + + @param linesep: Line separator string. + """ + self.linesep = linesep + + + def send(self, str=""): + """ + Send a string to the child process. + + @param str: String to send to the child process. + """ + try: + fd = os.open(self.inpipe_filename, os.O_RDWR) + os.write(fd, str) + os.close(fd) + except: + pass + + + def sendline(self, str=""): + """ + Send a string followed by a line separator to the child process. + + @param str: String to send to the child process. + """ + self.send(str + self.linesep) + + +class kvm_tail(kvm_spawn): + """ + This class runs a child process in the background and sends its output in + real time, line-by-line, to a callback function. + + See kvm_spawn's docstring. + + This class uses a single pipe reader to read data in real time from the + child process and report it to a given callback function. + When the child process exits, its exit status is reported to an additional + callback function. + + When this class is unpickled, it automatically resumes reporting output. + """ + + def __init__(self, command=None, id=None, echo=False, linesep="\n", + termination_func=None, output_func=None, output_prefix=""): + """ + Initialize the class and run command as a child process. + + @param command: Command to run, or None if accessing an already running + server. + @param id: ID of an already running server, if accessing a running + server, or None if starting a new one. + @param echo: Boolean indicating whether echo should be initially + enabled for the pseudo terminal running the subprocess. This + parameter has an effect only when starting a new server. + @param linesep: Line separator to be appended to strings sent to the + child process by sendline(). + @param termination_func: Function to call when the process exits. The + function must accept a single exit status parameter. + @param output_func: Function to call whenever a line of output is + available from the STDOUT or STDERR streams of the process. + The function must accept a single string parameter. The string + does not include the final newline. + @param output_prefix: String to prepend to lines sent to output_func. + """ + # Add a reader and a close hook + self._add_reader("tail") + self._add_close_hook(self._join_thread) + + # Init the superclass + kvm_spawn.__init__(self, command, id, echo, linesep) + + # Remember some attributes + self.termination_func = termination_func + self.output_func = output_func + self.output_prefix = output_prefix + + # Start the thread in the background + self.tail_thread = threading.Thread(None, self._tail) + self.tail_thread.start() + + + def __getinitargs__(self): + return kvm_spawn.__getinitargs__(self) + (self.termination_func, + self.output_func, + self.output_prefix) + + + def set_termination_func(self, termination_func): + """ + Set the termination_func attribute. See __init__() for details. + + @param termination_func: Function to call when the process terminates. + Must take a single parameter -- the exit status. + """ + self.termination_func = termination_func + + + def set_output_func(self, output_func): + """ + Set the output_func attribute. See __init__() for details. + + @param output_func: Function to call for each line of STDOUT/STDERR + output from the process. Must take a single string parameter. + """ + self.output_func = output_func + + + def set_output_prefix(self, output_prefix): + """ + Set the output_prefix attribute. See __init__() for details. + + @param output_prefix: String to pre-pend to each line sent to + output_func (see set_output_callback()). + """ + self.output_prefix = output_prefix + + + def _tail(self): + def print_line(text): + # Pre-pend prefix and remove trailing whitespace + text = self.output_prefix + text.rstrip() + # Sanitize text + text = text.decode("utf-8", "replace") + # Pass it to output_func + try: + self.output_func(text) + except TypeError: + pass + + fd = self._get_fd("tail") + buffer = "" + while True: + try: + # See if there's any data to read from the pipe + r, w, x = select.select([fd], [], [], 0.05) + except: + break + if fd in r: + # Some data is available; read it + new_data = os.read(fd, 1024) + if not new_data: + break + buffer += new_data + # Send the output to output_func line by line + # (except for the last line) + if self.output_func: + lines = buffer.split("\n") + for line in lines[:-1]: + print_line(line) + # Leave only the last line + last_newline_index = buffer.rfind("\n") + buffer = buffer[last_newline_index+1:] + else: + # No output is available right now; flush the buffer + if buffer: + print_line(buffer) + buffer = "" + # The process terminated; print any remaining output + if buffer: + print_line(buffer) + # Get the exit status, print it and send it to termination_func + status = self.get_status() + if status is None: + return + print_line("(Process terminated with status %s)" % status) + try: + self.termination_func(status) + except TypeError: + pass + + + def _join_thread(self): + # Wait for the tail thread to exit + if self.tail_thread: + self.tail_thread.join() + + +class kvm_expect(kvm_tail): + """ + This class runs a child process in the background and provides expect-like + services. + + It also provides all of kvm_tail's functionality. + """ + + def __init__(self, command=None, id=None, echo=False, linesep="\n", + termination_func=None, output_func=None, output_prefix=""): + """ + Initialize the class and run command as a child process. + + @param command: Command to run, or None if accessing an already running + server. + @param id: ID of an already running server, if accessing a running + server, or None if starting a new one. + @param echo: Boolean indicating whether echo should be initially + enabled for the pseudo terminal running the subprocess. This + parameter has an effect only when starting a new server. + @param linesep: Line separator to be appended to strings sent to the + child process by sendline(). + @param termination_func: Function to call when the process exits. The + function must accept a single exit status parameter. + @param output_func: Function to call whenever a line of output is + available from the STDOUT or STDERR streams of the process. + The function must accept a single string parameter. The string + does not include the final newline. + @param output_prefix: String to prepend to lines sent to output_func. + """ + # Add a reader + self._add_reader("expect") + + # Init the superclass + kvm_tail.__init__(self, command, id, echo, linesep, + termination_func, output_func, output_prefix) + + + def __getinitargs__(self): + return kvm_tail.__getinitargs__(self) + + + def read_nonblocking(self, timeout=None): + """ + Read from child until there is nothing to read for timeout seconds. + + @param timeout: Time (seconds) to wait before we give up reading from + the child process, or None to use the default value. + """ + if timeout is None: + timeout = 0.1 + fd = self._get_fd("expect") + data = "" + while True: + try: + r, w, x = select.select([fd], [], [], timeout) + except: + return data + if fd in r: + new_data = os.read(fd, 1024) + if not new_data: + return data + data += new_data + else: + return data + + + def match_patterns(self, str, patterns): + """ + Match str against a list of patterns. + + Return the index of the first pattern that matches a substring of str. + None and empty strings in patterns are ignored. + If no match is found, return None. + + @param patterns: List of strings (regular expression patterns). + """ + for i in range(len(patterns)): + if not patterns[i]: + continue + if re.search(patterns[i], str): + return i + + + def read_until_output_matches(self, patterns, filter=lambda x: x, + timeout=30.0, internal_timeout=None, + print_func=None): + """ + Read using read_nonblocking until a match is found using match_patterns, + or until timeout expires. Before attempting to search for a match, the + data is filtered using the filter function provided. + + @brief: Read from child using read_nonblocking until a pattern + matches. + @param patterns: List of strings (regular expression patterns) + @param filter: Function to apply to the data read from the child before + attempting to match it against the patterns (should take and + return a string) + @param timeout: The duration (in seconds) to wait until a match is + found + @param internal_timeout: The timeout to pass to read_nonblocking + @param print_func: A function to be used to print the data being read + (should take a string parameter) + @return: Tuple containing the match index (or None if no match was + found) and the data read so far. + """ + match = None + data = "" + + end_time = time.time() + timeout + while time.time() < end_time: + # Read data from child + newdata = self.read_nonblocking(internal_timeout) + # Print it if necessary + if print_func and newdata: + str = newdata + if str.endswith("\n"): + str = str[:-1] + for line in str.split("\n"): + print_func(line.decode("utf-8", "replace")) + data += newdata + + done = False + # Look for patterns + match = self.match_patterns(filter(data), patterns) + if match is not None: + done = True + # Check if child has died + if not self.is_alive(): + logging.debug("Process terminated with status %s" % self.get_status()) + done = True + # Are we done? + if done: break + + # Print some debugging info + if match is None and (self.is_alive() or self.get_status() != 0): + logging.debug("Timeout elapsed or process terminated. Output:" + + kvm_utils.format_str_for_message(data.strip())) + + return (match, data) + + + def read_until_last_word_matches(self, patterns, timeout=30.0, + internal_timeout=None, print_func=None): + """ + Read using read_nonblocking until the last word of the output matches + one of the patterns (using match_patterns), or until timeout expires. + + @param patterns: A list of strings (regular expression patterns) + @param timeout: The duration (in seconds) to wait until a match is + found + @param internal_timeout: The timeout to pass to read_nonblocking + @param print_func: A function to be used to print the data being read + (should take a string parameter) + @return: A tuple containing the match index (or None if no match was + found) and the data read so far. + """ + def get_last_word(str): + if str: + return str.split()[-1] + else: + return "" + + return self.read_until_output_matches(patterns, get_last_word, + timeout, internal_timeout, + print_func) + + + def read_until_last_line_matches(self, patterns, timeout=30.0, + internal_timeout=None, print_func=None): + """ + Read using read_nonblocking until the last non-empty line of the output + matches one of the patterns (using match_patterns), or until timeout + expires. Return a tuple containing the match index (or None if no match + was found) and the data read so far. + + @brief: Read using read_nonblocking until the last non-empty line + matches a pattern. + + @param patterns: A list of strings (regular expression patterns) + @param timeout: The duration (in seconds) to wait until a match is + found + @param internal_timeout: The timeout to pass to read_nonblocking + @param print_func: A function to be used to print the data being read + (should take a string parameter) + """ + def get_last_nonempty_line(str): + nonempty_lines = [l for l in str.splitlines() if l.strip()] + if nonempty_lines: + return nonempty_lines[-1] + else: + return "" + + return self.read_until_output_matches(patterns, get_last_nonempty_line, + timeout, internal_timeout, + print_func) + + +class kvm_shell_session(kvm_expect): + """ + This class runs a child process in the background. It it suited for + processes that provide an interactive shell, such as SSH and Telnet. + + It provides all services of kvm_expect and kvm_tail. In addition, it + provides command running services, and a utility function to test the + process for responsiveness. + """ + + def __init__(self, command=None, id=None, echo=False, linesep="\n", + termination_func=None, output_func=None, output_prefix="", + prompt=r"[\#\$]\s*$", status_test_command="echo $?"): + """ + Initialize the class and run command as a child process. + + @param command: Command to run, or None if accessing an already running + server. + @param id: ID of an already running server, if accessing a running + server, or None if starting a new one. + @param echo: Boolean indicating whether echo should be initially + enabled for the pseudo terminal running the subprocess. This + parameter has an effect only when starting a new server. + @param linesep: Line separator to be appended to strings sent to the + child process by sendline(). + @param termination_func: Function to call when the process exits. The + function must accept a single exit status parameter. + @param output_func: Function to call whenever a line of output is + available from the STDOUT or STDERR streams of the process. + The function must accept a single string parameter. The string + does not include the final newline. + @param output_prefix: String to prepend to lines sent to output_func. + @param prompt: Regular expression describing the shell's prompt line. + @param status_test_command: Command to be used for getting the last + exit status of commands run inside the shell (used by + get_command_status_output() and friends). + """ + # Init the superclass + kvm_expect.__init__(self, command, id, echo, linesep, + termination_func, output_func, output_prefix) + + # Remember some attributes + self.prompt = prompt + self.status_test_command = status_test_command + + + def __getinitargs__(self): + return kvm_expect.__getinitargs__(self) + (self.prompt, + self.status_test_command) + + + def set_prompt(self, prompt): + """ + Set the prompt attribute for later use by read_up_to_prompt. + + @param: String that describes the prompt contents. + """ + self.prompt = prompt + + + def set_status_test_command(self, status_test_command): + """ + Set the command to be sent in order to get the last exit status. + + @param status_test_command: Command that will be sent to get the last + exit status. + """ + self.status_test_command = status_test_command + + + def is_responsive(self, timeout=5.0): + """ + Return True if the process responds to STDIN/terminal input. + + Send a newline to the child process (e.g. SSH or Telnet) and read some + output using read_nonblocking(). + If all is OK, some output should be available (e.g. the shell prompt). + In that case return True. Otherwise return False. + + @param timeout: Time duration to wait before the process is considered + unresponsive. + """ + # Read all output that's waiting to be read, to make sure the output + # we read next is in response to the newline sent + self.read_nonblocking(timeout=0.1) + # Send a newline + self.sendline() + # Wait up to timeout seconds for some output from the child + end_time = time.time() + timeout + while time.time() < end_time: + time.sleep(0.5) + if self.read_nonblocking(timeout=0).strip(): + return True + # No output -- report unresponsive + return False + + + def read_up_to_prompt(self, timeout=30.0, internal_timeout=None, + print_func=None): + """ + Read using read_nonblocking until the last non-empty line of the output + matches the prompt regular expression set by set_prompt, or until + timeout expires. + + @brief: Read using read_nonblocking until the last non-empty line + matches the prompt. + + @param timeout: The duration (in seconds) to wait until a match is + found + @param internal_timeout: The timeout to pass to read_nonblocking + @param print_func: A function to be used to print the data being + read (should take a string parameter) + + @return: A tuple containing True/False indicating whether the prompt + was found, and the data read so far. + """ + (match, output) = self.read_until_last_line_matches([self.prompt], + timeout, + internal_timeout, + print_func) + return (match is not None, output) + + + def get_command_status_output(self, command, timeout=30.0, + internal_timeout=None, print_func=None): + """ + Send a command and return its exit status and output. + + @param command: Command to send (must not contain newline characters) + @param timeout: The duration (in seconds) to wait until a match is + found + @param internal_timeout: The timeout to pass to read_nonblocking + @param print_func: A function to be used to print the data being read + (should take a string parameter) + + @return: A tuple (status, output) where status is the exit status or + None if no exit status is available (e.g. timeout elapsed), and + output is the output of command. + """ + def remove_command_echo(str, cmd): + if str and str.splitlines()[0] == cmd: + str = "".join(str.splitlines(True)[1:]) + return str + + def remove_last_nonempty_line(str): + return "".join(str.rstrip().splitlines(True)[:-1]) + + # Print some debugging info + logging.debug("Sending command: %s" % command) + + # Read everything that's waiting to be read + self.read_nonblocking(0.1) + + # Send the command and get its output + self.sendline(command) + (match, output) = self.read_up_to_prompt(timeout, internal_timeout, + print_func) + # Remove the echoed command from the output + output = remove_command_echo(output, command) + # If the prompt was not found, return the output so far + if not match: + return (None, output) + # Remove the final shell prompt from the output + output = remove_last_nonempty_line(output) + + # Send the 'echo ...' command to get the last exit status + self.sendline(self.status_test_command) + (match, status) = self.read_up_to_prompt(10.0, internal_timeout) + if not match: + return (None, output) + status = remove_command_echo(status, self.status_test_command) + status = remove_last_nonempty_line(status) + # Get the first line consisting of digits only + digit_lines = [l for l in status.splitlines() if l.strip().isdigit()] + if not digit_lines: + return (None, output) + status = int(digit_lines[0].strip()) + + # Print some debugging info + if status != 0: + logging.debug("Command failed; status: %d, output:%s", status, + kvm_utils.format_str_for_message(output.strip())) + + return (status, output) + + + def get_command_status(self, command, timeout=30.0, internal_timeout=None, + print_func=None): + """ + Send a command and return its exit status. + + @param command: Command to send + @param timeout: The duration (in seconds) to wait until a match is + found + @param internal_timeout: The timeout to pass to read_nonblocking + @param print_func: A function to be used to print the data being read + (should take a string parameter) + + @return: Exit status or None if no exit status is available (e.g. + timeout elapsed). + """ + (status, output) = self.get_command_status_output(command, timeout, + internal_timeout, + print_func) + return status + + + def get_command_output(self, command, timeout=30.0, internal_timeout=None, + print_func=None): + """ + Send a command and return its output. + + @param command: Command to send + @param timeout: The duration (in seconds) to wait until a match is + found + @param internal_timeout: The timeout to pass to read_nonblocking + @param print_func: A function to be used to print the data being read + (should take a string parameter) + """ + (status, output) = self.get_command_status_output(command, timeout, + internal_timeout, + print_func) + return output + + +# The following is the server part of the module. + +def _server_main(): + id = sys.stdin.readline().strip() + echo = sys.stdin.readline().strip() == "True" + readers = sys.stdin.readline().strip().split(",") + command = sys.stdin.readline().strip() + " && echo %s > /dev/null" % id + + # Define filenames to be used for communication + base_dir = "/tmp/kvm_spawn" + (shell_pid_filename, + status_filename, + output_filename, + inpipe_filename, + lock_server_running_filename, + lock_client_starting_filename) = _get_filenames(base_dir, id) + + # Populate the reader filenames list + reader_filenames = [_get_reader_filename(base_dir, id, reader) + for reader in readers] + + # Set $TERM = dumb + os.putenv("TERM", "dumb") + + (shell_pid, shell_fd) = pty.fork() + if shell_pid == 0: + # Child process: run the command in a subshell + os.execv("/bin/sh", ["/bin/sh", "-c", command]) + else: + # Parent process + lock_server_running = _lock(lock_server_running_filename) + + # Set terminal echo on/off and disable pre- and post-processing + attr = termios.tcgetattr(shell_fd) + attr[0] &= ~termios.INLCR + attr[0] &= ~termios.ICRNL + attr[0] &= ~termios.IGNCR + attr[1] &= ~termios.OPOST + if echo: + attr[3] |= termios.ECHO + else: + attr[3] &= ~termios.ECHO + termios.tcsetattr(shell_fd, termios.TCSANOW, attr) + + # Open output file + output_file = open(output_filename, "w") + # Open input pipe + os.mkfifo(inpipe_filename) + inpipe_fd = os.open(inpipe_filename, os.O_RDWR) + # Open output pipes (readers) + reader_fds = [] + for filename in reader_filenames: + os.mkfifo(filename) + reader_fds.append(os.open(filename, os.O_RDWR)) + + # Write shell PID to file + file = open(shell_pid_filename, "w") + file.write(str(shell_pid)) + file.close() + + # Print something to stdout so the client can start working + print "hello" + sys.stdout.flush() + + # Initialize buffers + buffers = ["" for reader in readers] + + # Read from child and write to files/pipes + while True: + # Make a list of reader pipes whose buffers are not empty + fds = [fd for (i, fd) in enumerate(reader_fds) if buffers[i]] + # Wait until there's something to do + r, w, x = select.select([shell_fd, inpipe_fd], fds, []) + # If a reader pipe is ready for writing -- + for (i, fd) in enumerate(reader_fds): + if fd in w: + bytes_written = os.write(fd, buffers[i]) + buffers[i] = buffers[i][bytes_written:] + # If there's data to read from the child process -- + if shell_fd in r: + try: + data = os.read(shell_fd, 16384) + except OSError: + break + # Remove carriage returns from the data -- they often cause + # trouble and are normally not needed + data = data.replace("\r", "") + output_file.write(data) + output_file.flush() + for i in range(len(readers)): + buffers[i] += data + # If there's data to read from the client -- + if inpipe_fd in r: + data = os.read(inpipe_fd, 1024) + os.write(shell_fd, data) + + # Wait for the shell process to exit and get its exit status + status = os.waitpid(shell_pid, 0)[1] + status = os.WEXITSTATUS(status) + file = open(status_filename, "w") + file.write(str(status)) + file.close() + + # Wait for the client to finish initializing + _wait(lock_client_starting_filename) + + # Delete FIFOs + for filename in reader_filenames + [inpipe_filename]: + try: + os.unlink(filename) + except OSError: + pass + + # Close all files and pipes + output_file.close() + os.close(inpipe_fd) + for fd in reader_fds: + os.close(fd) + + _unlock(lock_server_running) + + +if __name__ == "__main__": + _server_main()
Signature · 1 line
1.5.4.1
PATCH 2/17 Modify kvm_vm and kvm_preprocessing to use the new kvm_subprocess module
--- client/tests/kvm/kvm_preprocessing.py | 27 ++------ client/tests/kvm/kvm_vm.py | 111 +++++++++++++++----------------- 2 files changed, 59 insertions(+), 79 deletions(-)
client/tests/kvm/kvm_preprocessing.py +7 -20
@@ -1,7 +1,7 @@ import sys, os, time, commands, re, logging, signal from autotest_lib.client.bin import test from autotest_lib.client.common_lib import error -import kvm_vm, kvm_utils +import kvm_vm, kvm_utils, kvm_subprocess def preprocess_image(test, params):
@@ -75,7 +75,7 @@ def preprocess_vm(test, params, env, name): qemu_path, image_dir, iso_dir): - logging.debug("VM's qemu command differs from requested one;" + logging.debug("VM's qemu command differs from requested one; " "restarting it...") start_vm = True
@@ -158,13 +158,11 @@ def process_command(test, params, env, command, command_timeout, # execute command logging.info("Executing command '%s'..." % command) timeout = int(command_timeout) - (status, pid, output) = kvm_utils.run_bg("cd %s; %s" % (test.bindir, + (status, output) = kvm_subprocess.run_fg("cd %s; %s" % (test.bindir, command), - None, logging.debug, - "(command) ", - timeout = timeout) + logging.debug, "(command) ", + timeout=timeout) if status != 0: - kvm_utils.safe_kill(pid, signal.SIGTERM) logging.warn("Custom processing command failed: '%s'..." % command) if command_noncritical != "yes": raise error.TestError("Custom processing command failed")
@@ -204,17 +202,6 @@ def preprocess(test, params, env): @param params: A dict containing all VM and image parameters. @param env: The environment (a dict-like object). """ - # Verify the identities of all living VMs - for vm in env.values(): - if not kvm_utils.is_vm(vm): - continue - if vm.is_dead(): - continue - if not vm.verify_process_identity(): - logging.debug("VM '%s' seems to have been replaced by another" - " process" % vm.name) - vm.pid = None - # Destroy and remove VMs that are no longer needed in the environment requested_vms = kvm_utils.get_sub_dict_names(params, "vms") for key in env.keys():
@@ -282,8 +269,8 @@ def postprocess(test, params, env): # Remove them all logging.debug("'keep_ppm_files' not specified; removing all PPM files" " from results dir...") - kvm_utils.run_bg("rm -vf %s" % os.path.join(test.debugdir, "*.ppm"), - None, logging.debug, "(rm) ", timeout=5.0) + rm_cmd = "rm -vf %s" % os.path.join(test.debugdir, "*.ppm") + kvm_subprocess.run_fg(rm_cmd, logging.debug, "(rm) ", timeout=5.0) #execute any post_commands if params.get("post_command"):
client/tests/kvm/kvm_vm.py +52 -59
199 lines not on this page · raw message
Signature · 1 line
1.5.4.1
PATCH 3/17 Modify remote_login and remote_scp in kvm_utils to use kvm_subprocess
Also remove a reference to kvm_log that was left behind.
--- client/tests/kvm/kvm_utils.py | 13 ++++++++----- 1 files changed, 8 insertions(+), 5 deletions(-)
client/tests/kvm/kvm_utils.py +8 -5
@@ -2,6 +2,7 @@ import md5, thread, subprocess, time, string, random, socket, os, signal, pty import select, re, logging from autotest_lib.client.bin import utils from autotest_lib.client.common_lib import error +import kvm_subprocess """ KVM test utility functions.
@@ -631,8 +632,9 @@ def remote_login(command, password, prompt, linesep="\n", timeout=10): @return Return the kvm_spawn object on success and None on failure. """ - sub = kvm_spawn(command, linesep) - sub.set_prompt(prompt) + sub = kvm_subprocess.kvm_shell_session(command, + linesep=linesep, + prompt=prompt) password_prompt_count = 0
@@ -698,7 +700,7 @@ def remote_scp(command, password, timeout=300, login_timeout=10): @return: True if the transfer succeeds and False on failure. """ - sub = kvm_spawn(command) + sub = kvm_subprocess.kvm_expect(command) password_prompt_count = 0 _timeout = login_timeout
@@ -729,9 +731,10 @@ def remote_scp(command, password, timeout=300, login_timeout=10): sub.close() return False else: # match == None - logging.debug("Timeout or process terminated") + logging.debug("Timeout elapsed or process terminated") + status = sub.get_status() sub.close() - return sub.poll() == 0 + return status == 0 def scp_to_remote(host, port, username, password, local_path, remote_path,
Signature · 1 line
1.5.4.1
PATCH 4/17 Modify run_autotest() in kvm_tests.py to use the new kvm_subprocess module.
--- client/tests/kvm/kvm_tests.py | 9 +++++---- 1 files changed, 5 insertions(+), 4 deletions(-)
client/tests/kvm/kvm_tests.py +5 -4
@@ -1,6 +1,6 @@ import time, os, logging from autotest_lib.client.common_lib import utils, error -import kvm_utils, ppm_utils, scan_results +import kvm_utils, kvm_subprocess, ppm_utils, scan_results """ KVM test definitions.
@@ -274,15 +274,16 @@ def run_autotest(test, params, env): cmd += " --exclude=*.pyc" cmd += " --exclude=*.svn" cmd += " --exclude=*.git" - kvm_utils.run_bg(cmd % (test.bindir, tarred_autotest_path), timeout=30) + kvm_subprocess.run_fg(cmd % (test.bindir, tarred_autotest_path), timeout=30) # tar the contents of bindir/autotest/tests/<test_name> cmd = "cd %s; tar cvjf %s %s/*" cmd += " --exclude=*.pyc" cmd += " --exclude=*.svn" cmd += " --exclude=*.git" - kvm_utils.run_bg(cmd % (os.path.join(test.bindir, "autotest", "tests"), - tarred_test_path, test_name), timeout=30) + kvm_subprocess.run_fg(cmd % + (os.path.join(test.bindir, "autotest", "tests"), + tarred_test_path, test_name), timeout=30) # Check if we need to copy autotest.tar.bz2 copy = False
Signature · 1 line
1.5.4.1
PATCH 5/17 Remove kvm_spawn and run_bg() from kvm_utils.py.
These are now provided by kvm_subprocess.py.
--- client/tests/kvm/kvm_utils.py | 477 +---------------------------------------- 1 files changed, 2 insertions(+), 475 deletions(-)
client/tests/kvm/kvm_utils.py +2 -475
@@ -227,390 +227,8 @@ def check_kvm_source_dir(source_dir): raise error.TestError("Unknown source dir layout, cannot proceed.") -# The following are a class and functions used for SSH, SCP and Telnet -# communication with guests. - -class kvm_spawn: - """ - This class is used for spawning and controlling a child process. - """ - - def __init__(self, command, linesep="\n"): - """ - Initialize the class and run command as a child process. - - @param command: Command that will be run. - @param linesep: Line separator for the given platform. - """ - self.exitstatus = None - self.linesep = linesep - (pid, fd) = pty.fork() - if pid == 0: - os.execv("/bin/sh", ["/bin/sh", "-c", command]) - else: - self.pid = pid - self.fd = fd - - - def set_linesep(self, linesep): - """ - Sets the line separator string (usually "\\n"). - - @param linesep: Line separator character. - """ - self.linesep = linesep - - - def is_responsive(self, timeout=5.0): - """ - Return True if the session is responsive. - - Send a newline to the child process (e.g. SSH or Telnet) and read some - output using read_nonblocking. - If all is OK, some output should be available (e.g. the shell prompt). - In that case return True. Otherwise return False. - - @param timeout: Timeout that will happen before we consider the - process unresponsive - """ - self.read_nonblocking(timeout=0.1) - self.sendline() - output = self.read_nonblocking(timeout=timeout) - if output.strip(): - return True - return False - - - def poll(self): - """ - If the process exited, return its exit status. Otherwise return None. - The exit status is stored for use in subsequent calls. - """ - if self.exitstatus != None: - return self.exitstatus - pid, status = os.waitpid(self.pid, os.WNOHANG) - if pid: - self.exitstatus = os.WEXITSTATUS(status) - return self.exitstatus - else: - return None - - - def close(self): - """ - Close the session (close the process filedescriptors and kills the - process ID), and return the exit status. - """ - try: - os.close(self.fd) - os.kill(self.pid, signal.SIGTERM) - except OSError: - pass - return self.poll() - - - def sendline(self, str=""): - """ - Sends a string followed by a line separator to the child process. - - @param str: String that will be sent to the child process. - """ - try: - os.write(self.fd, str + self.linesep) - except OSError: - pass - - - def read_nonblocking(self, timeout=1.0): - """ - Read from child until there is nothing to read for timeout seconds. - - @param timeout: Time (seconds) of wait before we give up reading from - the child process. - """ - data = "" - while True: - r, w, x = select.select([self.fd], [], [], timeout) - if self.fd in r: - try: - data += os.read(self.fd, 1024) - except OSError: - return data - else: - return data - - - def match_patterns(self, str, patterns): - """ - Match str against a list of patterns. - - Return the index of the first pattern that matches a substring of str. - None and empty strings in patterns are ignored. - If no match is found, return None. - - @param patterns: List of strings (regular expression patterns). - """ - for i in range(len(patterns)): - if not patterns[i]: - continue - exp = re.compile(patterns[i]) - if exp.search(str): - return i - - - def read_until_output_matches(self, patterns, filter=lambda(x):x, - timeout=30.0, internal_timeout=1.0, - print_func=None): - """ - Read using read_nonblocking until a match is found using match_patterns, - or until timeout expires. Before attempting to search for a match, the - data is filtered using the filter function provided. - - @brief: Read from child using read_nonblocking until a pattern - matches. - @param patterns: List of strings (regular expression patterns) - @param filter: Function to apply to the data read from the child before - attempting to match it against the patterns (should take and - return a string) - @param timeout: The duration (in seconds) to wait until a match is - found - @param internal_timeout: The timeout to pass to read_nonblocking - @param print_func: A function to be used to print the data being read - (should take a string parameter) - @return: Tuple containing the match index (or None if no match was - found) and the data read so far. - """ - match = None - data = "" - - end_time = time.time() + timeout - while time.time() < end_time: - # Read data from child - newdata = self.read_nonblocking(internal_timeout) - # Print it if necessary - if print_func and newdata: - map(print_func, newdata.splitlines()) - data += newdata - - done = False - # Look for patterns - match = self.match_patterns(filter(data), patterns) - if match != None: - done = True - # Check if child has died - if self.poll() != None: - logging.debug("Process terminated with status %d", self.poll()) - done = True - # Are we done? - if done: break - - # Print some debugging info - if match == None and self.poll() != 0: - logging.debug("Timeout elapsed or process terminated. Output: %s", - format_str_for_message(data.strip())) - - return (match, data) - - - def get_last_word(self, str): - """ - Return the last word in str. - - @param str: String that will be analyzed. - """ - if str: - return str.split()[-1] - else: - return "" - - - def get_last_line(self, str): - """ - Return the last non-empty line in str. - - @param str: String that will be analyzed. - """ - last_line = "" - for line in str.splitlines(): - if line != "": - last_line = line - return last_line - - - def read_until_last_word_matches(self, patterns, timeout=30.0, - internal_timeout=1.0, print_func=None): - """ - Read using read_nonblocking until the last word of the output matches - one of the patterns (using match_patterns), or until timeout expires. - - @param patterns: A list of strings (regular expression patterns) - @param timeout: The duration (in seconds) to wait until a match is - found - @param internal_timeout: The timeout to pass to read_nonblocking - @param print_func: A function to be used to print the data being read - (should take a string parameter) - @return: A tuple containing the match index (or None if no match was - found) and the data read so far. - """ - return self.read_until_output_matches(patterns, self.get_last_word, - timeout, internal_timeout, - print_func) - - - def read_until_last_line_matches(self, patterns, timeout=30.0, - internal_timeout=1.0, print_func=None): - """ - Read using read_nonblocking until the last non-empty line of the output - matches one of the patterns (using match_patterns), or until timeout - expires. Return a tuple containing the match index (or None if no match - was found) and the data read so far. - - @brief: Read using read_nonblocking until the last non-empty line - matches a pattern. - - @param patterns: A list of strings (regular expression patterns) - @param timeout: The duration (in seconds) to wait until a match is - found - @param internal_timeout: The timeout to pass to read_nonblocking - @param print_func: A function to be used to print the data being read - (should take a string parameter) - """ - return self.read_until_output_matches(patterns, self.get_last_line, - timeout, internal_timeout, - print_func) - - - def set_prompt(self, prompt): - """ - Set the prompt attribute for later use by read_up_to_prompt. - - @param: String that describes the prompt contents. - """ - self.prompt = prompt - - - def read_up_to_prompt(self, timeout=30.0, internal_timeout=1.0, - print_func=None): - """ - Read using read_nonblocking until the last non-empty line of the output - matches the prompt regular expression set by set_prompt, or until - timeout expires. - - @brief: Read using read_nonblocking until the last non-empty line - matches the prompt. - - @param timeout: The duration (in seconds) to wait until a match is - found - @param internal_timeout: The timeout to pass to read_nonblocking - @param print_func: A function to be used to print the data being - read (should take a string parameter) - - @return: A tuple containing True/False indicating whether the prompt - was found, and the data read so far. - """ - (match, output) = self.read_until_last_line_matches([self.prompt], - timeout, - internal_timeout, - print_func) - if match == None: - return (False, output) - else: - return (True, output) - - - def set_status_test_command(self, status_test_command): - """ - Set the command to be sent in order to get the last exit status. - - @param status_test_command: Command that will be sent to get the last - exit status. - """ - self.status_test_command = status_test_command - - - def get_command_status_output(self, command, timeout=30.0, - internal_timeout=1.0, print_func=None): - """ - Send a command and return its exit status and output. - - @param command: Command to send - @param timeout: The duration (in seconds) to wait until a match is - found - @param internal_timeout: The timeout to pass to read_nonblocking - @param print_func: A function to be used to print the data being read - (should take a string parameter) - - @return: A tuple (status, output) where status is the exit status or - None if no exit status is available (e.g. timeout elapsed), and - output is the output of command. - """ - # Print some debugging info - logging.debug("Sending command: %s" % command) - - # Read everything that's waiting to be read - self.read_nonblocking(0.1) - - # Send the command and get its output - self.sendline(command) - (match, output) = self.read_up_to_prompt(timeout, internal_timeout, - print_func) - if not match: - return (None, "\n".join(output.splitlines()[1:])) - output = "\n".join(output.splitlines()[1:-1]) - - # Send the 'echo ...' command to get the last exit status - self.sendline(self.status_test_command) - (match, status) = self.read_up_to_prompt(10.0, internal_timeout) - if not match: - return (None, output) - status = int("\n".join(status.splitlines()[1:-1]).strip()) - - # Print some debugging info - if status != 0: - logging.debug("Command failed; status: %d, output:%s", status, - format_str_for_message(output.strip())) - - return (status, output) - - - def get_command_status(self, command, timeout=30.0, internal_timeout=1.0, - print_func=None): - """ - Send a command and return its exit status. - - @param command: Command to send - @param timeout: The duration (in seconds) to wait until a match is - found - @param internal_timeout: The timeout to pass to read_nonblocking - @param print_func: A function to be used to print the data being read - (should take a string parameter) - - @return: Exit status or None if no exit status is available (e.g. - timeout elapsed). - """ - (status, output) = self.get_command_status_output(command, timeout, - internal_timeout, - print_func) - return status - - - def get_command_output(self, command, timeout=30.0, internal_timeout=1.0, - print_func=None): - """ - Send a command and return its output. - - @param command: Command to send - @param timeout: The duration (in seconds) to wait until a match is - found - @param internal_timeout: The timeout to pass to read_nonblocking - @param print_func: A function to be used to print the data being read - (should take a string parameter) - """ - (status, output) = self.get_command_status_output(command, timeout, - internal_timeout, - print_func) - return output - +# The following are functions used for SSH, SCP and Telnet communication with +# guests. def remote_login(command, password, prompt, linesep="\n", timeout=10): """
@@ -810,97 +428,6 @@ def telnet(host, port, username, password, prompt, timeout=10): return remote_login(command, password, prompt, "\r\n", timeout) -# The following are functions used for running commands in the background. - -def track_process(sub, status_output=None, term_func=None, stdout_func=None, - prefix=""): - """ - Read lines from the stdout pipe of the subprocess. Pass each line to - stdout_func prefixed by prefix. Place the lines in status_output[1]. - When the subprocess exits, call term_func. Place the exit status in - status_output[0]. - - @brief Track a subprocess and report its output and termination. - - @param sub: An object returned by subprocess.Popen - @param status_output: A list in which the exit status and output are to be - stored. - @param term_func: A function to call when the process terminates - (should take no parameters) - @param stdout_func: A function to call with each line of output from the - subprocess (should take a string parameter) - - @param prefix -- a string to pre-pend to each line of the output, before - passing it to stdout_func - """ - while True: - # Read a single line from stdout - text = sub.stdout.readline() - # If the subprocess exited... - if text == "": - # Get exit code - status = sub.wait() - # Report it - if status_output: - status_output[0] = status - # Call term_func - if term_func: - term_func() - return - # Report the text - if status_output: - status_output[1] += text - # Call stdout_func with the returned text - if stdout_func: - text = prefix + text.strip() - # We need to sanitize the text before passing it to the logging - # system - text = text.decode('utf-8', 'replace') - stdout_func(text) - - -def run_bg(command, term_func=None, stdout_func=None, prefix="", timeout=1.0): - """ - Run command as a subprocess. Call stdout_func with each line of output from - the subprocess (prefixed by prefix). Call term_func when the subprocess - terminates. If timeout expires and the subprocess is still running, return. - - @brief: Run a subprocess in the background and collect its output and - exit status. - - @param command: The shell command to execute - @param term_func: A function to call when the process terminates - (should take no parameters) - @param stdout_func: A function to call with each line of output from - the subprocess (should take a string parameter) - @param prefix: A string to pre-pend to each line of the output, before - passing it to stdout_func - @param timeout: Time duration (in seconds) to wait for the subprocess to - terminate before returning - - @return: A 3-tuple containing the exit status (None if the subprocess is - still running), the PID of the subprocess (None if the subprocess - terminated), and the output collected so far. - """ - # Start the process - sub = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, - stderr=subprocess.STDOUT) - # Start the tracking thread - status_output = [None, ""] - thread.start_new_thread(track_process, (sub, status_output, term_func, - stdout_func, prefix)) - # Wait up to timeout secs for the process to exit - end_time = time.time() + timeout - while time.time() < end_time: - # If the process exited, return - if status_output[0] != None: - return (status_output[0], None, status_output[1]) - # Otherwise, sleep for a while - time.sleep(0.1) - # Report the PID and the output collected so far - return (None, sub.pid, status_output[1]) - - # The following are utility functions related to ports. def is_sshd_running(host, port, timeout=10.0):
Signature · 1 line
1.5.4.1
PATCH 6/17 kvm_guest_wizard: rename output_dir to debug_dir in barrier_2()
The name 'debug_dir' makes it clearer that it corresponds to test.debugdir.
--- client/tests/kvm/kvm_guest_wizard.py | 20 ++++++++++---------- 1 files changed, 10 insertions(+), 10 deletions(-)
client/tests/kvm/kvm_guest_wizard.py +10 -10
@@ -18,7 +18,7 @@ def handle_var(vm, params, varname): def barrier_2(vm, words, fail_if_stuck_for, stuck_detection_history, - output_dir, data_scrdump_filename, current_step_num): + debug_dir, data_scrdump_filename, current_step_num): if len(words) < 7: logging.error("Bad barrier_2 command line") return False
@@ -34,12 +34,12 @@ def barrier_2(vm, words, fail_if_stuck_for, stuck_detection_history, if sleep_duration < 1.0: sleep_duration = 1.0 if sleep_duration > 10.0: sleep_duration = 10.0 - scrdump_filename = os.path.join(output_dir, "scrdump.ppm") - cropped_scrdump_filename = os.path.join(output_dir, "cropped_scrdump.ppm") - expected_scrdump_filename = os.path.join(output_dir, "scrdump_expected.ppm") - expected_cropped_scrdump_filename = os.path.join(output_dir, + scrdump_filename = os.path.join(debug_dir, "scrdump.ppm") + cropped_scrdump_filename = os.path.join(debug_dir, "cropped_scrdump.ppm") + expected_scrdump_filename = os.path.join(debug_dir, "scrdump_expected.ppm") + expected_cropped_scrdump_filename = os.path.join(debug_dir, "cropped_scrdump_expected.ppm") - comparison_filename = os.path.join(output_dir, "comparison.ppm") + comparison_filename = os.path.join(debug_dir, "comparison.ppm") end_time = time.time() + timeout end_time_stuck = time.time() + fail_if_stuck_for
@@ -99,7 +99,7 @@ def barrier_2(vm, words, fail_if_stuck_for, stuck_detection_history, prev_whole_image_md5sums.insert(0, whole_image_md5sum) # Limit queue length to stuck_detection_history prev_whole_image_md5sums = \ - prev_whole_image_md5sums[:stuck_detection_history] + prev_whole_image_md5sums[:stuck_detection_history] # Sleep for a while time.sleep(sleep_duration)
@@ -113,12 +113,12 @@ def barrier_2(vm, words, fail_if_stuck_for, stuck_detection_history, logging.info(message) return False else: - # Collect information and put it in output_dir + # Collect information and put it in debug_dir if data_scrdump_filename and os.path.exists(data_scrdump_filename): # Read expected screendump image (ew, eh, edata) = \ ppm_utils.image_read_from_ppm_file(data_scrdump_filename) - # Write it in output_dir + # Write it in debug_dir ppm_utils.image_write_to_ppm_file(expected_scrdump_filename, ew, eh, edata) # Write the cropped version as well
@@ -131,7 +131,7 @@ def barrier_2(vm, words, fail_if_stuck_for, stuck_detection_history, ppm_utils.image_write_to_ppm_file(comparison_filename, w, h, data) # Print error messages and fail the test - long_message = message + "\n(see analysis at %s)" % output_dir + long_message = message + "\n(see analysis at %s)" % debug_dir logging.error(long_message) raise error.TestFail, message
Signature · 1 line
1.5.4.1
PATCH 7/17 kvm_guest_wizard: pass 'params' directly to barrier_2()
Currently parameters for barrier_2() are extracted from 'params' in the main run_steps() test routine, and then passed to barrier_2(). Instead, let barrier_2() extract parameters from 'params' as it sees fit. This will make adding new parameters slightly easier and cleaner.
--- client/tests/kvm/kvm_guest_wizard.py | 37 ++++++++++++++++----------------- 1 files changed, 18 insertions(+), 19 deletions(-)
client/tests/kvm/kvm_guest_wizard.py +18 -19
@@ -17,8 +17,8 @@ def handle_var(vm, params, varname): return True -def barrier_2(vm, words, fail_if_stuck_for, stuck_detection_history, - debug_dir, data_scrdump_filename, current_step_num): +def barrier_2(vm, words, params, debug_dir, data_scrdump_filename, + current_step_num): if len(words) < 7: logging.error("Bad barrier_2 command line") return False
@@ -41,6 +41,18 @@ def barrier_2(vm, words, fail_if_stuck_for, stuck_detection_history, "cropped_scrdump_expected.ppm") comparison_filename = os.path.join(debug_dir, "comparison.ppm") + fail_if_stuck_for = params.get("fail_if_stuck_for") + if fail_if_stuck_for: + fail_if_stuck_for = float(fail_if_stuck_for) + else: + fail_if_stuck_for = 1e308 + + stuck_detection_history = params.get("stuck_detection_history") + if stuck_detection_history: + stuck_detection_history = int(stuck_detection_history) + else: + stuck_detection_history = 2 + end_time = time.time() + timeout end_time_stuck = time.time() + fail_if_stuck_for start_time = time.time()
@@ -151,18 +163,6 @@ def run_steps(test, params, env): if not os.path.exists(steps_filename): raise error.TestError("Steps file not found: %s" % steps_filename) - fail_if_stuck_for = params.get("fail_if_stuck_for") - if fail_if_stuck_for: - fail_if_stuck_for = float(fail_if_stuck_for) - else: - fail_if_stuck_for = 1e308 - - stuck_detection_history = params.get("stuck_detection_history") - if stuck_detection_history: - stuck_detection_history = int(stuck_detection_history) - else: - stuck_detection_history = 2 - sf = open(steps_filename, "r") lines = sf.readlines() sf.close()
@@ -201,13 +201,12 @@ def run_steps(test, params, env): logging.error("Variable not defined: %s" % words[1]) elif words[0] == "barrier_2": if current_screendump: - scrdump_filename = ( - os.path.join(ppm_utils.get_data_dir(steps_filename), - current_screendump)) + scrdump_filename = os.path.join( + ppm_utils.get_data_dir(steps_filename), + current_screendump) else: scrdump_filename = None - if not barrier_2(vm, words, fail_if_stuck_for, - stuck_detection_history, test.debugdir, + if not barrier_2(vm, words, params, test.debugdir, scrdump_filename, current_step_num): skip_current_step = True else:
Signature · 1 line
1.5.4.1
PATCH 8/17 kvm_guest_wizard: allow keeping screendump history for debugging purposes
Add two new step file test parameters:
- keep_screendump_history: if equals 'yes', screendump history is saved in test.debugdir/barrier_history in JPG format. Each screendump taken by the test is saved if it differs from the previous screendump. By default, when a barrier succeeds all history is removed, so eventually the remaining files are only those of the (last) failed barrier, if any.
- keep_all_history: if equals 'yes', screendump history is not removed upon barrier success. The test leaves behind all the collected screendump history.
--- client/tests/kvm/kvm_guest_wizard.py | 38 ++++++++++++++++++++++++++++----- 1 files changed, 32 insertions(+), 6 deletions(-)
client/tests/kvm/kvm_guest_wizard.py +32 -6
@@ -1,6 +1,6 @@ import os, time, md5, re, shutil, logging from autotest_lib.client.common_lib import utils, error -import kvm_utils, ppm_utils +import kvm_utils, ppm_utils, kvm_subprocess """ Utilities to perform automatic guest installation using step files.
@@ -53,6 +53,11 @@ def barrier_2(vm, words, params, debug_dir, data_scrdump_filename, else: stuck_detection_history = 2 + keep_screendump_history = params.get("keep_screendump_history") == "yes" + if keep_screendump_history: + keep_all_history = params.get("keep_all_history") == "yes" + history_dir = os.path.join(debug_dir, "barrier_history") + end_time = time.time() + timeout end_time_stuck = time.time() + fail_if_stuck_for start_time = time.time()
@@ -91,21 +96,42 @@ def barrier_2(vm, words, params, debug_dir, data_scrdump_filename, # Read image file (w, h, data) = ppm_utils.image_read_from_ppm_file(scrdump_filename) + # Compute md5sum of whole image + whole_image_md5sum = ppm_utils.image_md5sum(w, h, data) + + # Write screendump to history_dir (as JPG) if requested + # and if the screendump differs from the previous one + if (keep_screendump_history and + whole_image_md5sum not in prev_whole_image_md5sums[:1]): + try: + os.makedirs(history_dir) + except: + pass + history_scrdump_filename = os.path.join(history_dir, + "scrdump-step_%s-%s.jpg" % (current_step_num, + time.strftime("%Y%m%d-%H%M%S"))) + kvm_subprocess.run_fg("convert -quality 30 %s %s" % + (scrdump_filename, history_scrdump_filename), + logging.debug, "(convert) ", timeout=30) + # Compare md5sum of barrier region with the expected md5sum calced_md5sum = ppm_utils.get_region_md5sum(w, h, data, x1, y1, dx, dy, cropped_scrdump_filename) if calced_md5sum == md5sum: + # Success -- remove screendump history unless requested not to + if keep_screendump_history and not keep_all_history: + kvm_subprocess.run_fg("rm -rvf %s" % history_dir, + logging.debug, "(rm) ", timeout=30) + # Report success return True - # Compute md5sum of whole image in order to compare it with - # previous ones - whole_image_md5sum = ppm_utils.image_md5sum(w, h, data) + # Insert image md5sum into queue of last seen images: # If md5sum is already in queue... if whole_image_md5sum in prev_whole_image_md5sums: # Remove md5sum from queue prev_whole_image_md5sums.remove(whole_image_md5sum) else: - # Extend 'stuck' timeout + # Otherwise extend 'stuck' timeout end_time_stuck = time.time() + fail_if_stuck_for # Insert md5sum at beginning of queue prev_whole_image_md5sums.insert(0, whole_image_md5sum)
@@ -129,7 +155,7 @@ def barrier_2(vm, words, params, debug_dir, data_scrdump_filename, if data_scrdump_filename and os.path.exists(data_scrdump_filename): # Read expected screendump image (ew, eh, edata) = \ - ppm_utils.image_read_from_ppm_file(data_scrdump_filename) + ppm_utils.image_read_from_ppm_file(data_scrdump_filename) # Write it in debug_dir ppm_utils.image_write_to_ppm_file(expected_scrdump_filename, ew, eh, edata)
Signature · 1 line
1.5.4.1
PATCH 9/17 kvm_tests.cfg.sample: add 'keep_screendump_history = yes' to step file tests
This should be rather harmless because the history does not take up much space, and is only kept for the failed barriers in failed tests, by default. It should significantly ease debugging of failed step file tests.
--- client/tests/kvm/kvm_tests.cfg.sample | 2 ++ 1 files changed, 2 insertions(+), 0 deletions(-)
client/tests/kvm/kvm_tests.cfg.sample +2 -0
@@ -30,6 +30,7 @@ variants: type = steps fail_if_stuck_for = 300 stuck_detection_history = 2 + keep_screendump_history = yes force_create_image = yes kill_vm = yes kill_vm_timeout = 60
@@ -40,6 +41,7 @@ variants: fail_if_stuck_for = 300 stuck_detection_history = 2 kill_vm_on_error = yes + keep_screendump_history = yes - boot: install setup type = boot
Signature · 1 line
1.5.4.1
PATCH 10/17 KVM test: optionally convert PPM files to PNG format after test
This is intended to save disk space. Requires ImageMagick (uses mogrify).
To enable:convert_ppm_files_to_png = yes
To enable only for failed tests:convert_ppm_files_to_png_on_error = yes
Reminder: by default PPM files are removed after the test (and after the conversion, if requested). To keep them specify:keep_ppm_files = yes
To keep them only for failed tests:keep_ppm_files_on_error = yes
A reasonable choice would be to keep PNG files for failed tests, and never keep PPM files. To do this specify convert_ppm_files_to_png_on_error = yes
without specifying 'keep_ppm_files = yes' or 'keep_ppm_files_on_error = yes' (or explicitly set to 'no', if it was set to 'yes' on a higher level in the config hierarchy).
This patch also makes small cosmetic changes to the 'keep_ppm_files' feature.
--- client/tests/kvm/kvm_preprocessing.py | 16 ++++++++++++---- 1 files changed, 12 insertions(+), 4 deletions(-)
client/tests/kvm/kvm_preprocessing.py +12 -4
@@ -264,11 +264,19 @@ def postprocess(test, params, env): """ process(test, params, env, postprocess_image, postprocess_vm) - # See if we should get rid of all PPM files - if not params.get("keep_ppm_files") == "yes": - # Remove them all + # Should we convert PPM files to PNG format? + if params.get("convert_ppm_files_to_png") == "yes": + logging.debug("'convert_ppm_files_to_png' specified; converting PPM" + " files to PNG format...") + mogrify_cmd = ("mogrify -format png %s" % + os.path.join(test.debugdir, "*.ppm")) + kvm_subprocess.run_fg(mogrify_cmd, logging.debug, "(mogrify) ", + timeout=30.0) + + # Should we keep the PPM files? + if params.get("keep_ppm_files") != "yes": logging.debug("'keep_ppm_files' not specified; removing all PPM files" - " from results dir...") + " from debug dir...") rm_cmd = "rm -vf %s" % os.path.join(test.debugdir, "*.ppm") kvm_subprocess.run_fg(rm_cmd, logging.debug, "(rm) ", timeout=5.0)
Signature · 1 line
1.5.4.1
PATCH 11/17 KVM test: kvm_tests.cfg.sample: convert PPM files to PNG by default
By default, always remove PPM files, and keep PNG files only for failed tests. This shouldn't do much harm, because while PPMs can be incorporated directly into step files, PNGs can be converted back to PPMs easily, and take less disk space.
(PNG is a lossless compression format.)
The 'keep_ppm_files' and 'keep_ppm_files_on_error' settings are commented out so the user can easily enable them if desired.
--- client/tests/kvm/kvm_tests.cfg.sample | 5 +++-- 1 files changed, 3 insertions(+), 2 deletions(-)
client/tests/kvm/kvm_tests.cfg.sample +3 -2
@@ -8,8 +8,9 @@ main_vm = vm1 # Some preprocessor/postprocessor params start_vm = yes -keep_ppm_files = no -keep_ppm_files_on_error = yes +convert_ppm_files_to_png_on_error = yes +#keep_ppm_files = yes +#keep_ppm_files_on_error = yes kill_vm = no kill_vm_gracefully = yes
Signature · 1 line
1.5.4.1
PATCH 12/17 KVM test: add simple timedrift test (mainly for Windows)
1) Log into a guest.
2) Take a time reading from the guest and host.
3) Run load on the guest and host.
4) Take a second time reading.
5) Stop the load and rest for a while.
6) Take a third time reading.
7) If the drift immediately after load is higher than a user- specified value (in %), fail.
If the drift after the rest period is higher than a user-specified value, fail.
--- client/tests/kvm/kvm.py | 1 + client/tests/kvm/kvm_tests.py | 161 ++++++++++++++++++++++++++++++++++++++++- 2 files changed, 160 insertions(+), 2 deletions(-)
client/tests/kvm/kvm.py +1 -0
@@ -55,6 +55,7 @@ class kvm(test.test): "kvm_install": test_routine("kvm_install", "run_kvm_install"), "linux_s3": test_routine("kvm_tests", "run_linux_s3"), "stress_boot": test_routine("kvm_tests", "run_stress_boot"), + "timedrift": test_routine("kvm_tests", "run_timedrift"), } # Make it possible to import modules from the test's bindir
client/tests/kvm/kvm_tests.py +159 -2
173 lines not on this page · raw message
Signature · 1 line
1.5.4.1
PATCH 13/17 KVM test: fix a parsing problem in kvm_config.py
Allow kvm_config to parse weird lines that seem to contain several operators, such as:time_filter_re = "(?<=TIME: ...)"
The '?<=' is recognized as the operator instead of the '='. To fix this, select the operator closest to the beginning of the line.
--- client/tests/kvm/kvm_config.py | 6 ++++-- 1 files changed, 4 insertions(+), 2 deletions(-)
client/tests/kvm/kvm_config.py +4 -2
@@ -294,10 +294,12 @@ class config: # Look for a known operator in the line operators = ["?+=", "?<=", "?=", "+=", "<=", "="] op_found = None + op_pos = len(line) for op in operators: - if op in line: + pos = line.find(op) + if pos >= 0 and pos < op_pos: op_found = op - break + op_pos = pos # Found an operator? if op_found:
Signature · 1 line
1.5.4.1
PATCH 14/17 KVM test: fix string and docstring indentation in kvm_config.py
--- client/tests/kvm/kvm_config.py | 99 +++++++++++++++++++++------------------- 1 files changed, 52 insertions(+), 47 deletions(-)
client/tests/kvm/kvm_config.py +52 -47
@@ -168,8 +168,8 @@ class config: """ Return the indent level of the next non-empty, non-comment line in file. - @param file: File like object. - @return: If no line is available, return -1. + @param file: File like object. + @return: If no line is available, return -1. """ pos = file.tell() line = self.get_next_line(file)
@@ -188,10 +188,10 @@ class config: """ Add name to str with a separator dot and return the result. - @param str: String that will be processed - @param name: name that will be appended to the string. - @return: If append is True, append name to str. - Otherwise, pre-pend name to str. + @param str: String that will be processed + @param name: name that will be appended to the string. + @return: If append is True, append name to str. + Otherwise, pre-pend name to str. """ if str == "": return name
@@ -208,14 +208,14 @@ class config: Read and parse lines from file like object until a line with an indent level lower than or equal to prev_indent is encountered. - @brief: Parse a 'variants' or 'subvariants' block from a file-like - object. - @param file: File-like object that will be parsed - @param list: List of dicts to operate on - @param subvariants: If True, parse in 'subvariants' mode; - otherwise parse in 'variants' mode - @param prev_indent: The indent level of the "parent" block - @return: The resulting list of dicts. + @brief: Parse a 'variants' or 'subvariants' block from a file-like + object. + @param file: File-like object that will be parsed + @param list: List of dicts to operate on + @param subvariants: If True, parse in 'subvariants' mode; + otherwise parse in 'variants' mode + @param prev_indent: The indent level of the "parent" block + @return: The resulting list of dicts. """ new_list = []
@@ -270,16 +270,16 @@ class config: Read and parse lines from file until a line with an indent level lower than or equal to prev_indent is encountered. - @brief: Parse a file-like object. - @param file: A file-like object - @param list: A list of dicts to operate on (list is modified in - place and should not be used after the call) - @param restricted: if True, operate in restricted mode - (prohibit 'variants') - @param prev_indent: the indent level of the "parent" block - @return: Return the resulting list of dicts. - @note: List is destroyed and should not be used after the call. - Only the returned list should be used. + @brief: Parse a file-like object. + @param file: A file-like object + @param list: A list of dicts to operate on (list is modified in + place and should not be used after the call) + @param restricted: if True, operate in restricted mode + (prohibit 'variants') + @param prev_indent: the indent level of the "parent" block + @return: Return the resulting list of dicts. + @note: List is destroyed and should not be used after the call. + Only the returned list should be used. """ while True: indent = self.get_next_line_indent(file)
@@ -305,8 +305,8 @@ class config: if op_found: if self.debug and not restricted: self.__debug_print(indented_line, - "Parsing operator (%d dicts in current " - "context)" % len_list) + "Parsing operator (%d dicts in current " + "context)" % len_list) (left, value) = self.split_and_strip(line, op_found) filters_and_key = self.split_and_strip(left, ":") filters = filters_and_key[:-1]
@@ -352,9 +352,9 @@ class config: list = filtered_list if self.debug and not restricted: self.__debug_print(indented_line, - "Parsing no/only (%d dicts in current " - "context, %d remain)" % - (len_list, len(list))) + "Parsing no/only (%d dicts in current " + "context, %d remain)" % + (len_list, len(list))) # Parse 'variants' elif line == "variants:":
@@ -365,8 +365,8 @@ class config: raise error.AutotestError(e_msg) if self.debug and not restricted: self.__debug_print(indented_line, - "Entering variants block (%d dicts in" - "current context)" % len_list) + "Entering variants block (%d dicts in " + "current context)" % len_list) list = self.parse_variants(file, list, subvariants=False, prev_indent=indent)
@@ -375,8 +375,8 @@ class config: elif line == "subvariants:": if self.debug and not restricted: self.__debug_print(indented_line, - "Entering subvariants block (%d dicts in " - "current context)" % len_list) + "Entering subvariants block (%d dicts in " + "current context)" % len_list) new_list = [] # Remember current file position pos = file.tell()
@@ -422,9 +422,9 @@ class config: elif line.endswith(":"): if self.debug and not restricted: self.__debug_print(indented_line, - "Entering multi-line exception block" - "(%d dicts in current context outside " - "exception)" % len_list) + "Entering multi-line exception block " + "(%d dicts in current context outside " + "exception)" % len_list) line = line.strip(":") new_list = [] # Remember current file position
@@ -452,8 +452,8 @@ class config: """ Nicely print two strings and an arrow. - @param str1: First string - @param str2: Second string + @param str1: First string + @param str2: Second string """ if str2: str = "%-50s ---> %s" % (str1, str2)
@@ -466,7 +466,12 @@ class config: """ Make some modifications to list, as part of parsing a 'variants' block. - @param list + @param list: List to be processed + @param name: Name to be prepended to the dictionary's 'name' key + @param dep_list: List of dependencies to be added to the dictionary's + 'depend' key + @param add_to_shortname: Boolean indicating whether name should be + prepended to the dictionary's 'shortname' key as well """ for dict in list: # Prepend name to the dict's 'name' field
@@ -483,15 +488,15 @@ class config: def __modify_list_subvariants(self, list, name, dep_list, add_to_shortname): """ - Make some modifications to list, as part of parsing a - 'subvariants' block. + Make some modifications to list, as part of parsing a 'subvariants' + block. - @param list: List that will be processed - @param name: Name that will be prepended to the dictionary name - @param dep_list: List of dependencies to be added to the list - dictionaries - @param add_to_shortname: Whether we'll add a shortname parameter to - the dictionaries. + @param list: List to be processed + @param name: Name to be appended to the dictionary's 'name' key + @param dep_list: List of dependencies to be added to the dictionary's + 'depend' key + @param add_to_shortname: Boolean indicating whether name should be + appended to the dictionary's 'shortname' as well """ for dict in list: # Add new dependencies
Signature · 1 line
1.5.4.1
PATCH 15/17 KVM test: add timedrift test to kvm_tests.cfg.sample
Currently the test will only run on Windows.
It should be able to run on Linux just as well, but if I understand correctly, testing time drift on Linux is less interesting.
Also make some tiny cosmetic changes (spacing), and move the stress_boot test before the shutdown test (shutdown should be last).
--- client/tests/kvm/kvm_tests.cfg.sample | 46 ++++++++++++++++++++++++++------ 1 files changed, 37 insertions(+), 9 deletions(-)
client/tests/kvm/kvm_tests.cfg.sample +37 -9
@@ -92,20 +92,33 @@ variants: test_name = disktest test_control_file = disktest.control - - linux_s3: install setup + - linux_s3: install setup type = linux_s3 - - shutdown: install setup + - timedrift: install setup + type = timedrift + extra_params += " -rtc-td-hack" + # Pin the VM and host load to CPU #0 + cpu_mask = 0x1 + # Set the load and rest durations + load_duration = 20 + rest_duration = 20 + # Fail if the drift after load is higher than 50% + drift_threshold = 50 + # Fail if the drift after the rest period is higher than 10% + drift_threshold_after_rest = 10 + + - stress_boot: install setup + type = stress_boot + max_vms = 5 + alive_test_cmd = ps aux + + - shutdown: install setup type = shutdown kill_vm = yes kill_vm_gracefully = no - - stress_boot: - type = stress_boot - max_vms = 5 - alive_test_cmd = ps aux - # NICs variants: - @rtl8139:
@@ -121,6 +134,7 @@ variants: variants: # Linux section - @Linux: + no timedrift cmd_shutdown = shutdown -h now cmd_reboot = shutdown -r now ssh_status_test_command = echo $?
@@ -303,8 +317,6 @@ variants: md5sum=bf4635e4a4bd3b43838e72bc8c329d55 md5sum_1m=18ecd37b639109f1b2af05cfb57dfeaf - - # Windows section - @Windows: no autotest
@@ -318,6 +330,21 @@ variants: migration_test_command = ver && vol stress_boot: alive_test_cmd = systeminfo + timedrift: + # For this to work, the ISO should contain vlc (vlc.exe) and a video (ED_1024.avi) + cdrom = windows/vlc.iso + time_command = "echo TIME: %date% %time%" + time_filter_re = "(?<=TIME: \w\w\w ).{19}(?=\.\d\d)" + time_format = "%m/%d/%Y %H:%M:%S" + guest_load_command = 'cmd /c "d:\vlc -f --loop --no-qt-privacy-ask --no-qt-system-tray d:\ED_1024.avi"' + # Alternative guest load: + #guest_load_command = "(dir /s && dir /s && dir /s && dir /s) > nul" + guest_load_stop_command = "taskkill /F /IM vlc.exe" + guest_load_instances = 2 + host_load_command = "bzip2 -c --best /dev/urandom > /dev/null" + # Alternative host load: + #host_load_command = "dd if=/dev/urandom of=/dev/null" + host_load_instances = 8 variants: - Win2000:
@@ -582,5 +609,6 @@ variants: only qcow2.*ide.*default.*up.*Ubuntu-8.10-server.*(autotest.sleeptest) only rtl8139 + # Choose your test list only fc8_quick
Signature · 1 line
1.5.4.1
PATCH 16/17 KVM test: initialize some VM attributes in __init__() to prevent trouble
'redirs' and 'vnc_port' might be used before they're defined, if make_qemu_command() is called before create(). To make sure this doesn't happen, define them in the VM constructor.
--- client/tests/kvm/kvm_vm.py | 2 ++ 1 files changed, 2 insertions(+), 0 deletions(-)
client/tests/kvm/kvm_vm.py +2 -0
@@ -112,6 +112,8 @@ class VM: @param iso_dir: The directory where ISOs reside """ self.process = None + self.redirs = {} + self.vnc_port = 5900 self.uuid = None self.name = name
Signature · 1 line
1.5.4.1
PATCH 17/17 KVM test: make some style changes in kvm_preprocessing.py
Make some small style changes to handling of pre- and post-commands. These changes are not required, but they make the code slightly shorter and more consistent with the rest of the code (IMO).
Also, do not print "Adding ... to environment" for each parameter in the dict because in some cases there are too many parameters and this generates a lot of output.
--- client/tests/kvm/kvm_preprocessing.py | 38 +++++++++++++-------------------- 1 files changed, 15 insertions(+), 23 deletions(-)
client/tests/kvm/kvm_preprocessing.py +15 -23
@@ -141,30 +141,22 @@ def process_command(test, params, env, command, command_timeout, @param test: An Autotest test object. @param params: A dict containing all VM and image parameters. @param env: The environment (a dict-like object). - @param command: Script containing the command to be run. - @param commmand_timeout: Timeout for command execution. - @param command_noncritical: if 'yes' test will not fail if command fails. + @param command: Command to be run. + @param command_timeout: Timeout for command execution. + @param command_noncritical: If True test will not fail if command fails. """ - if command_timeout is None: - command_timeout = "600" - - if command_noncritical is None: - command_noncritical = "no" - - # export environment vars + # Export environment vars for k in params.keys(): - logging.info("Adding KVM_TEST_%s to Environment" % (k)) - os.putenv("KVM_TEST_%s" % (k), str(params[k])) - # execute command + os.putenv("KVM_TEST_%s" % k, str(params[k])) + # Execute command logging.info("Executing command '%s'..." % command) - timeout = int(command_timeout) (status, output) = kvm_subprocess.run_fg("cd %s; %s" % (test.bindir, command), logging.debug, "(command) ", - timeout=timeout) + timeout=command_timeout) if status != 0: - logging.warn("Custom processing command failed: '%s'..." % command) - if command_noncritical != "yes": + logging.warn("Custom processing command failed: '%s'" % command) + if not command_noncritical: raise error.TestError("Custom processing command failed")
@@ -214,11 +206,11 @@ def preprocess(test, params, env): vm.destroy() del env[key] - #execute any pre_commands + # Execute any pre_commands if params.get("pre_command"): process_command(test, params, env, params.get("pre_command"), - params.get("pre_command_timeout"), - params.get("pre_command_noncritical")) + int(params.get("pre_command_timeout", "600")), + params.get("pre_command_noncritical") == "yes") # Preprocess all VMs and images process(test, params, env, preprocess_image, preprocess_vm)
@@ -280,11 +272,11 @@ def postprocess(test, params, env): rm_cmd = "rm -vf %s" % os.path.join(test.debugdir, "*.ppm") kvm_subprocess.run_fg(rm_cmd, logging.debug, "(rm) ", timeout=5.0) - #execute any post_commands + # Execute any post_commands if params.get("post_command"): process_command(test, params, env, params.get("post_command"), - params.get("post_command_timeout"), - params.get("post_command_noncritical")) + int(params.get("post_command_timeout", "600")), + params.get("post_command_noncritical") == "yes") def postprocess_on_error(test, params, env):
Signature · 1 line
1.5.4.1
Re: [Autotest] [KVM-AUTOTEST PATCH 17/17] KVM test: make some style changes in kvm_preprocessing.py
Quoted 6 lines · Michael Goldish
Make some small style changes to handling of pre- and post-commands. These changes are not required, but they make the code slightly shorter and more consistent with the rest of the code (IMO).
Also, do not print "Adding ... to environment" for each parameter in the dict because in some cases there are too many parameters and this generates a lot of output.
Nice cleanup, thanks. Applied.
Quoted 87 lines
Signed-off-by Michael Goldish <[email protected]>---
client/tests/kvm/kvm_preprocessing.py | 38 +++++++++++++--------------------1 files changed, 15 insertions(+), 23 deletions(-)client/tests/kvm/kvm_preprocessing.py +0
-0@@ -141,30 +141,22 @@ def process_command(test, params, env, command, command_timeout,@param test: An Autotest test object.
@param params: A dict containing all VM and image parameters. @param env: The environment (a dict-like object).
- @param command: Script containing the command to be run.
- @paramcommmand_timeout: Timeout for command execution.
- @paramcommand_noncritical: if 'yes' test will not fail if command fails.
+ @param command: Command to be run.
+ @paramcommand_timeout: Timeout for command execution.
+ @paramcommand_noncritical: If True test will not fail if command fails. """
- ifcommand_timeoutis None:
- command_timeout = "600"client/tests/kvm/kvm_preprocessing.py +1
-5- - if command_noncritical is None: - command_noncritical = "no" - - # export environment vars + # Export environment varsfor k in
params.keys():
- logging.info("Adding KVM_TEST_%s to Environment" % (k))
- os.putenv("KVM_TEST_%s" % (k),str(params[k]))
- # execute command
+ os.putenv("KVM_TEST_%s" % k,str(params[k]))
+ # Execute command
logging.info("Executing command '%s'..." % command)
- timeout =int(command_timeout)
(status, output) = kvm_subprocess.run_fg("cd %s; %s" % (test.bindir, command), logging.debug, "(command) ",
- timeout=timeout)
+ timeout=command_timeout) if status != 0:
- logging.warn("Custom processing command failed: '%s'..." % command)
- ifcommand_noncritical!= "yes":
+ logging.warn("Custom processing command failed: '%s'" % command)
+ if notcommand_noncritical:
raiseerror.TestError("Custom processing command failed")client/tests/kvm/kvm_preprocessing.py +0
-0@@ -214,11 +206,11 @@ def preprocess(test, params, env): vm.destroy()del env[key]
- #execute any
pre_commands
+ # Execute anypre_commands
ifparams.get("pre_command"):
process_command(test, params, env,params.get("pre_command"),
- params.get("pre_command_timeout"),
- params.get("pre_command_noncritical"))
+ int(params.get("pre_command_timeout", "600")),
+ params.get("pre_command_noncritical") == "yes")# Preprocess all VMs and images
process(test, params, env, preprocess_image, preprocess_vm)client/tests/kvm/kvm_preprocessing.py +1
-1@@ -280,11 +272,11 @@ def postprocess(test, params, env): rm_cmd = "rm -vf %s" % os.path.join(test.debugdir, "*.ppm") kvm_subprocess.run_fg(rm_cmd, logging.debug, "(rm) ", timeout=5.0) - #execute any post_commands + # Execute any post_commandsif
params.get("post_command"):
process_command(test, params, env,params.get("post_command"),
- params.get("post_command_timeout"),
- params.get("post_command_noncritical"))
+ int(params.get("post_command_timeout", "600")),
+ params.get("post_command_noncritical") == "yes")def
postprocess_on_error(test, params, env):
--
1.5.4.1_______________________________________________
Autotest mailing list
[email protected]
http://test.kernel.org/cgi-bin/mailman/listinfo/autotest
Signature · 1 line
Lucas Meneghel
Lucas Meneghel Rodrigues
AppliedRe: [Autotest] [KVM-AUTOTEST PATCH 16/17] KVM test: initialize some VM attributes in __init__() to prevent trouble
In reply to Michael GoldishQuoted 3 lines · Michael Goldish
'redirs' and '
vnc_port' might be used before they're defined, ifmake_qemu_command()is called beforecreate(). To make sure this doesn't happen, define them in the VM constructor.
Applied.
Quoted 26 lines
Signed-off-by Michael Goldish <[email protected]>---
client/tests/kvm/kvm_vm.py | 2 ++1 files changed, 2 insertions(+), 0 deletions(-)client/tests/kvm/kvm_vm.py +0
-0@@ -112,6 +112,8 @@ class VM:@param
iso_dir: The directory where ISOs reside """
self.process = None
+ self.redirs = {}
+ self.vnc_port = 5900
self.uuid = Noneself.name = name
--
1.5.4.1_______________________________________________
Autotest mailing list
[email protected]
http://test.kernel.org/cgi-bin/mailman/listinfo/autotest
Signature · 1 line
Lucas Meneghel
Re: [Autotest] [KVM-AUTOTEST PATCH 15/17] KVM test: add timedrift test to kvm_tests.cfg.sample
In reply to Michael GoldishQuoted 3 lines · Michael Goldish
Currently the test will only run on Windows.
It should be able to run on Linux just as well, but if I understand correctly, testing time drift on Linux is less interesting.
Linux is interesting too. The problem is more visible on windows since
it uses 1000hz frequency when it plays multimedia. It makes timer irq injection harder.
Does the test fail without the rtc-td-hack?
Quoted 100 lines
Also make some tiny cosmetic changes (spacing), and move the
stress_boottest before the shutdown test (shutdown should be last).Signed-off-by Michael Goldish<[email protected]>--- client/tests/kvm/kvm_tests.cfg.sample | 46 ++++++++++++++++++++++++++------ 1 files changed, 37 insertions(+), 9 deletions(-)client/tests/kvm/kvm_tests.cfg.sample +37
-9@@ -92,20 +92,33 @@ variants: test_name = disktest test_control_file = disktest.control - - linux_s3: install setup + - linux_s3: install setup type = linux_s3 - - shutdown: install setup + - timedrift: install setup + type = timedrift + extra_params += " -rtc-td-hack" + # Pin the VM and host load to CPU #0 + cpu_mask = 0x1 + # Set the load and rest durations + load_duration = 20 + rest_duration = 20 + # Fail if the drift after load is higher than 50% + drift_threshold = 50 + # Fail if the drift after the rest period is higher than 10% + drift_threshold_after_rest = 10 + + - stress_boot: install setup + type = stress_boot + max_vms = 5 + alive_test_cmd = ps aux + + - shutdown: install setup type = shutdown kill_vm = yes kill_vm_gracefully = no - - stress_boot: - type = stress_boot - max_vms = 5 - alive_test_cmd = ps aux - # NICs variants: - @rtl8139:@@ -121,6 +134,7 @@ variants: variants: # Linux section - @Linux: + no timedrift cmd_shutdown = shutdown -h now cmd_reboot = shutdown -r now ssh_status_test_command = echo $?@@ -303,8 +317,6 @@ variants: md5sum=bf4635e4a4bd3b43838e72bc8c329d55 md5sum_1m=18ecd37b639109f1b2af05cfb57dfeaf - - # Windows section - @Windows: no autotest@@ -318,6 +330,21 @@ variants: migration_test_command = ver&& vol stress_boot: alive_test_cmd = systeminfo + timedrift: + # For this to work, the ISO should contain vlc (vlc.exe) and a video (ED_1024.avi) + cdrom = windows/vlc.iso + time_command = "echo TIME: %date% %time%" + time_filter_re = "(?<=TIME: \w\w\w ).{19}(?=\.\d\d)" + time_format = "%m/%d/%Y %H:%M:%S" + guest_load_command = 'cmd /c "d:\vlc -f --loop --no-qt-privacy-ask --no-qt-system-tray d:\ED_1024.avi"' + # Alternative guest load: + #guest_load_command = "(dir /s&& dir /s&& dir /s&& dir /s)> nul" + guest_load_stop_command = "taskkill /F /IM vlc.exe" + guest_load_instances = 2 + host_load_command = "bzip2 -c --best /dev/urandom> /dev/null" + # Alternative host load: + #host_load_command = "dd if=/dev/urandom of=/dev/null" + host_load_instances = 8 variants: - Win2000:@@ -582,5 +609,6 @@ variants: only qcow2.*ide.*default.*up.*Ubuntu-8.10-server.*(autotest.sleeptest) only rtl8139 + # Choose your test list only fc8_quick
Re: [Autotest] [KVM-AUTOTEST PATCH 14/17] KVM test: fix string and docstring indentation in kvm_config.py
In reply to Michael GoldishQuoted 4 lines · Michael Goldish
Signed-off-by Michael Goldish <[email protected]>---
client/tests/kvm/kvm_config.py | 99 +++++++++++++++++++++-------------------1 files changed, 52 insertions(+), 47 deletions(-)
Nice cleanup, thanks, applied.
Quoted 196 lines
client/tests/kvm/kvm_config.py +0
-0@@ -168,8 +168,8 @@ class config:"""
Return the indent level of the next non-empty, non-comment line in file.- @param file: File like object.
- @return: If no line is available, return -1.
+ @param file: File like object.
+ @return: If no line is available, return -1. """
pos =file.tell()
line =self.get_next_line(file)client/tests/kvm/kvm_config.py +0
-0@@ -188,10 +188,10 @@ class config:"""
Add name to str with a separator dot and return the result.- @param str: String that will be processed
- @param name: name that will be appended to the string.
- @return: If append is True, append name to str.
- Otherwise, pre-pend name to str.
+ @param str: String that will be processed
+ @param name: name that will be appended to the string.
+ @return: If append is True, append name to str.
+ Otherwise, pre-pend name to str.
"""
if str == "":
return nameclient/tests/kvm/kvm_config.py +0
-0@@ -208,14 +208,14 @@ class config:Read and parse lines from file like object until a line with an indent level lower than or equal to
prev_indentis encountered.- @brief: Parse a 'variants' or 'subvariants' block from a file-like
- object.
- @param file: File-like object that will be parsed
- @param list: List of dicts to operate on
- @param subvariants: If True, parse in 'subvariants' mode;
- otherwise parse in 'variants' mode
- @paramprev_indent: The indent level of the "parent" block
- @return: The resulting list of dicts.
+ @brief: Parse a 'variants' or 'subvariants' block from a file-like
+ object.
+ @param file: File-like object that will be parsed
+ @param list: List of dicts to operate on
+ @param subvariants: If True, parse in 'subvariants' mode;
+ otherwise parse in 'variants' mode
+ @paramprev_indent: The indent level of the "parent" block
+ @return: The resulting list of dicts.
"""
new_list= []client/tests/kvm/kvm_config.py +0
-0@@ -270,16 +270,16 @@ class config:Read and parse lines from file until a line with an indent level lower than or equal to
prev_indentis encountered.- @brief: Parse a file-like object.
- @param file: A file-like object
- @param list: A list of dicts to operate on (list is modified in
- place and should not be used after the call)
- @param restricted: if True, operate in restricted mode
- (prohibit 'variants')
- @paramprev_indent: the indent level of the "parent" block
- @return: Return the resulting list of dicts.
- @note: List is destroyed and should not be used after the call.
- Only the returned list should be used.
+ @brief: Parse a file-like object.
+ @param file: A file-like object
+ @param list: A list of dicts to operate on (list is modified in
+ place and should not be used after the call)
+ @param restricted: if True, operate in restricted mode
+ (prohibit 'variants')
+ @paramprev_indent: the indent level of the "parent" block
+ @return: Return the resulting list of dicts.
+ @note: List is destroyed and should not be used after the call.
+ Only the returned list should be used.
"""
while True:
indent =self.get_next_line_indent(file)client/tests/kvm/kvm_config.py +0
-0@@ -305,8 +305,8 @@ class config:if
op_found:
if self.debug and not restricted:
self.__debug_print(indented_line,
- "Parsing operator (%d dicts in current "
- "context)" %len_list)
+ "Parsing operator (%d dicts in current "
+ "context)" %len_list) (left, value) =self.split_and_strip(line, op_found)filters_and_key=self.split_and_strip(left, ":")filters = filters_and_key[:-1]client/tests/kvm/kvm_config.py +0
-0@@ -352,9 +352,9 @@ class config:list =
filtered_list
if self.debug and not restricted:
self.__debug_print(indented_line,
- "Parsing no/only (%d dicts in current "
- "context, %d remain)" %
- (len_list, len(list)))
+ "Parsing no/only (%d dicts in current "
+ "context, %d remain)" %
+ (len_list, len(list)))# Parse 'variants'
elif line == "variants:":client/tests/kvm/kvm_config.py +0
-0@@ -365,8 +365,8 @@ class config: raise error.AutotestError(e_msg)if self.debug and not restricted:
self.__debug_print(indented_line,
- "Entering variants block (%d dicts in"
- "current context)" %len_list)
+ "Entering variants block (%d dicts in "
+ "current context)" %len_list) list =self.parse_variants(file, list, subvariants=False, prev_indent=indent)client/tests/kvm/kvm_config.py +0
-0@@ -375,8 +375,8 @@ class config:elif line == "subvariants:":
if self.debug and not restricted:
self.__debug_print(indented_line,
- "Entering subvariants block (%d dicts in "
- "current context)" %len_list)
+ "Entering subvariants block (%d dicts in "
+ "current context)" %len_list)new_list= []
# Remember current file position
pos =file.tell()client/tests/kvm/kvm_config.py +0
-0@@ -422,9 +422,9 @@ class config:elif
line.endswith(":"):
if self.debug and not restricted:
self.__debug_print(indented_line,
- "Entering multi-line exception block"
- "(%d dicts in current context outside "
- "exception)" %len_list)
+ "Entering multi-line exception block "
+ "(%d dicts in current context outside "
+ "exception)" %len_list) line =line.strip(":")
new_list= []
# Remember current file positionclient/tests/kvm/kvm_config.py +0
-0@@ -452,8 +452,8 @@ class config:"""
Nicely print two strings and an arrow.- @param str1: First string
- @param str2: Second string
+ @param str1: First string
+ @param str2: Second string
"""
if str2:
str = "%-50s ---> %s" % (str1, str2)client/tests/kvm/kvm_config.py +0
-0@@ -466,7 +466,12 @@ class config:"""
Make some modifications to list, as part of parsing a 'variants' block.- @param list
+ @param list: List to be processed
+ @param name: Name to be prepended to the dictionary's 'name' key
+ @paramdep_list: List of dependencies to be added to the dictionary's
+ 'depend' key
+ @paramadd_to_shortname: Boolean indicating whether name should be
+ prepended to the dictionary's 'shortname' key as well """
for dict in list:
# Prepend name to the dict's 'name' fieldclient/tests/kvm/kvm_config.py +0
-0@@ -483,15 +488,15 @@ class config:def
__modify_list_subvariants(self, list, name, dep_list, add_to_shortname):
"""
- Make some modifications to list, as part of parsing a
- 'subvariants' block.
+ Make some modifications to list, as part of parsing a 'subvariants'
+ block.- @param list: List that will be processed
- @param name: Name that will be prepended to the dictionary name
- @paramdep_list: List of dependencies to be added to the list
- dictionaries
- @paramadd_to_shortname: Whether we'll add a shortname parameter to
- the dictionaries.
+ @param list: List to be processed
+ @param name: Name to be appended to the dictionary's 'name' key
+ @paramdep_list: List of dependencies to be added to the dictionary's
+ 'depend' key
+ @paramadd_to_shortname: Boolean indicating whether name should be
+ appended to the dictionary's 'shortname' as well """
for dict in list:
# Add new dependencies
--
1.5.4.1_______________________________________________
Autotest mailing list
[email protected]
http://test.kernel.org/cgi-bin/mailman/listinfo/autotest
Signature · 1 line
Lucas Meneghel
Lucas Meneghel Rodrigues
AppliedRe: [Autotest] [KVM-AUTOTEST PATCH 13/17] KVM test: fix a parsing problem in kvm_config.py
In reply to Michael GoldishQuoted 6 lines · Michael Goldish
Allow
kvm_configto parse weird lines that seem to contain several operators, such as:time_filter_re= "(?<=TIME: ...)"The '?<=' is recognized as the operator instead of the '='. To fix this, select the operator closest to the beginning of the line.
Applied.
Quoted 32 lines
Signed-off-by Michael Goldish <[email protected]>---
client/tests/kvm/kvm_config.py | 6 ++++--1 files changed, 4 insertions(+), 2 deletions(-)client/tests/kvm/kvm_config.py +0
-0@@ -294,10 +294,12 @@ class config: # Look for a known operator in the lineoperators = ["?+=", "?<=", "?=", "+=", "<=", "="]
op_found= None
+ op_pos = len(line)
for op in operators:
- if op in line:
+ pos =line.find(op)
+ if pos >= 0 and pos <op_pos:
op_found= op
- break
+ op_pos = pos# Found an operator?
ifop_found:
--
1.5.4.1_______________________________________________
Autotest mailing list
[email protected]
http://test.kernel.org/cgi-bin/mailman/listinfo/autotest
Signature · 1 line
Lucas Meneghel
Re: [Autotest] [KVM-AUTOTEST PATCH 12/17] KVM test: add simple timedrift test (mainly for Windows)
In reply to Michael GoldishQuoted 104 lines · Michael Goldish
1) Log into a guest.
2) Take a time reading from the guest and host.
3) Run load on the guest and host.
4) Take a second time reading.
5) Stop the load and rest for a while.
6) Take a third time reading.
7) If the drift immediately after load is higher than a user- specified value (in %), fail.
If the drift after the rest period is higher than a user-specified value, fail.Signed-off-by Michael Goldish<[email protected]>--- client/tests/kvm/kvm.py | 1 + client/tests/kvm/kvm_tests.py | 161 ++++++++++++++++++++++++++++++++++++++++- 2 files changed, 160 insertions(+), 2 deletions(-)client/tests/kvm/kvm.py +1
-0@@ -55,6 +55,7 @@ class kvm(test.test): "kvm_install": test_routine("kvm_install", "run_kvm_install"), "linux_s3": test_routine("kvm_tests", "run_linux_s3"), "stress_boot": test_routine("kvm_tests", "run_stress_boot"), + "timedrift": test_routine("kvm_tests", "run_timedrift"), } # Make it possible to import modules from the test's bindirclient/tests/kvm/kvm_tests.py +54
-2@@ -1,4 +1,4 @@ -import time, os, logging +import time, os, logging, re, commands from autotest_lib.client.common_lib import utils, error import kvm_utils, kvm_subprocess, ppm_utils, scan_results@@ -529,7 +529,6 @@ def run_stress_boot(tests, params, env): """ # boot the first vm vm = kvm_utils.env_get_vm(env, params.get("main_vm")) - if not vm: raise error.TestError("VM object not found in environment") if not vm.is_alive():@@ -586,3 +585,161 @@ def run_stress_boot(tests, params, env): for se in sessions: se.close() logging.info("Total number booted: %d" % (num -1)) + + +def run_timedrift(test, params, env): + """ + Time drift test (mainly for Windows guests): + + 1) Log into a guest. + 2) Take a time reading from the guest and host. + 3) Run load on the guest and host. + 4) Take a second time reading. + 5) Stop the load and rest for a while. + 6) Take a third time reading. + 7) If the drift immediately after load is higher than a user- + specified value (in %), fail. + If the drift after the rest period is higher than a user-specified value, + fail. + + @param test: KVM test object. + @param params: Dictionary with test parameters. + @param env: Dictionary with the test environment. + """ + vm = kvm_utils.env_get_vm(env, params.get("main_vm")) + if not vm: + raise error.TestError("VM object not found in environment") + if not vm.is_alive(): + raise error.TestError("VM seems to be dead; Test requires a living VM") + + logging.info("Waiting for guest to be up...") + + session = kvm_utils.wait_for(vm.ssh_login, 240, 0, 2) + if not session: + raise error.TestFail("Could not log into guest") + + logging.info("Logged in") + + # Collect test parameters: + # Command to run to get the current time + time_command = params.get("time_command") + # Filter which should match a string to be passed to time.strptime() + time_filter_re = params.get("time_filter_re") + # Time format for time.strptime() + time_format = params.get("time_format") + guest_load_command = params.get("guest_load_command") + guest_load_stop_command = params.get("guest_load_stop_command") + host_load_command = params.get("host_load_command") + guest_load_instances = int(params.get("guest_load_instances", "1")) + host_load_instances = int(params.get("host_load_instances", "0")) + # CPU affinity mask for taskset + cpu_mask = params.get("cpu_mask", "0xFF") + load_duration = float(params.get("load_duration", "30")) + rest_duration = float(params.get("rest_duration", "10")) + drift_threshold = float(params.get("drift_threshold", "200")) + drift_threshold_after_rest = float(params.get("drift_threshold_after_rest",
client/tests/kvm/kvm_tests.py def run_stress_boot(tests, params, env): · line 650
Quoted 10 lines
client/tests/kvm/kvm_tests.py +10
-0+ "200")) + + guest_load_sessions = [] + host_load_sessions = [] + + # Remember the VM's previous CPU affinity + prev_cpu_mask = commands.getoutput("taskset -p %s" % vm.get_pid()) + prev_cpu_mask = prev_cpu_mask.split()[-1] + # Set the VM's CPU affinity + commands.getoutput("taskset -p %s %s" % (cpu_mask, vm.get_pid()))
Need to handle guest smp case where we want to pin the guest to several cpus.
Cheers for the test!
Quoted 95 lines
client/tests/kvm/kvm_tests.py +95
-0+ + try: + # Get time before load + host_time_0 = time.time() + session.sendline(time_command) + (match, s) = session.read_up_to_prompt() + s = re.findall(time_filter_re, s)[0] + guest_time_0 = time.mktime(time.strptime(s, time_format)) + + # Run some load on the guest + logging.info("Starting load on guest...") + for i in range(guest_load_instances): + load_session = vm.ssh_login() + if not load_session: + raise error.TestFail("Could not log into guest") + load_session.set_output_prefix("(guest load %d) " % i) + load_session.set_output_func(logging.debug) + load_session.sendline(guest_load_command) + guest_load_sessions.append(load_session) + + # Run some load on the host + logging.info("Starting load on host...") + for i in range(host_load_instances): + host_load_sessions.append( + kvm_subprocess.run_bg(host_load_command, + output_func=logging.debug, + output_prefix="(host load %d) " % i, + timeout=0.5)) + # Set the CPU affinity of the shell running the load process + pid = host_load_sessions[-1].get_shell_pid() + commands.getoutput("taskset -p %s %s" % (cpu_mask, pid)) + # Try setting the CPU affinity of the load process itself + pid = host_load_sessions[-1].get_pid() + if pid: + commands.getoutput("taskset -p %s %s" % (cpu_mask, pid)) + + # Sleep for a while (during load) + logging.info("Sleeping for %s seconds..." % load_duration) + time.sleep(load_duration) + + # Get time delta after load + host_time_1 = time.time() + session.sendline(time_command) + (match, s) = session.read_up_to_prompt() + s = re.findall(time_filter_re, s)[0] + guest_time_1 = time.mktime(time.strptime(s, time_format)) + + # Report results + host_delta = host_time_1 - host_time_0 + guest_delta = guest_time_1 - guest_time_0 + drift = 100.0 * (host_delta - guest_delta) / host_delta + logging.info("Host duration: %.2f" % host_delta) + logging.info("Guest duration: %.2f" % guest_delta) + logging.info("Drift: %.2f%%" % drift) + + finally: + logging.info("Cleaning up...") + # Restore the VM's CPU affinity + commands.getoutput("taskset -p %s %s" % (prev_cpu_mask, vm.get_pid())) + # Stop the guest load + if guest_load_stop_command: + session.get_command_output(guest_load_stop_command) + # Close all load shell sessions + for load_session in guest_load_sessions: + load_session.close() + for load_session in host_load_sessions: + load_session.close() + + # Sleep again (rest) + logging.info("Sleeping for %s seconds..." % rest_duration) + time.sleep(rest_duration) + + # Get time after rest + host_time_2 = time.time() + session.sendline(time_command) + (match, s) = session.read_up_to_prompt() + s = re.findall(time_filter_re, s)[0] + guest_time_2 = time.mktime(time.strptime(s, time_format)) + + # Report results + host_delta_total = host_time_2 - host_time_0 + guest_delta_total = guest_time_2 - guest_time_0 + drift_total = 100.0 * (host_delta_total - guest_delta_total) / host_delta + logging.info("Total host duration including rest: %.2f" % host_delta_total) + logging.info("Total guest duration including rest: %.2f" % guest_delta_total) + logging.info("Total drift after rest: %.2f%%" % drift_total) + + # Fail the test if necessary + if drift> drift_threshold: + raise error.TestFail("Time drift too large: %.2f%%" % drift) + if drift> drift_threshold_after_rest: + raise error.TestFail("Time drift too large after rest period: %.2f%%" + % drift_total) + + session.close()
Re: [Autotest] [KVM-AUTOTEST PATCH 12/17] KVM test: add simple timedrift test (mainly for Windows)
Quoted 133 lines · ----- "Dor Laor
Quoted 9 lines · Michael Goldish
1) Log into a guest.
2) Take a time reading from the guest and host.
3) Run load on the guest and host.
4) Take a second time reading.
5) Stop the load and rest for a while.
6) Take a third time reading.
7) If the drift immediately after load is higher than a user- specified value (in %), fail.
If the drift after the rest period is higher than a user-specifiedvalue,
Quoted 6 lines
fail.
Signed-off-by Michael Goldish<[email protected]>--- client/tests/kvm/kvm.py | 1 + client/tests/kvm/kvm_tests.py | 161++++++++++++++++++++++++++++++++++++++++-
client/tests/kvm/kvm.py class kvm(test.test): · line 55
Quoted 8 lines
2 files changed, 160 insertions(+), 2 deletions(-)client/tests/kvm/kvm.py +0
-0@@ -55,6 +55,7 @@ class kvm(test.test): "kvm_install": test_routine("kvm_install","
run_kvm_install"),client/tests/kvm/kvm.py class kvm(test.test): · line 56
Quoted 1 line
client/tests/kvm/kvm.py +0
-0"linux_s3": test_routine("kvm_tests","
run_linux_s3"),client/tests/kvm/kvm.py class kvm(test.test): · line 57
Quoted 1 line
client/tests/kvm/kvm.py +0
-0"stress_boot": test_routine("kvm_tests","
run_stress_boot"),client/tests/kvm/kvm.py class kvm(test.test): · line 58
Quoted 1 line
client/tests/kvm/kvm.py +1
-0+ "timedrift": test_routine("kvm_tests","
run_timedrift"),client/tests/kvm/kvm.py class kvm(test.test): · line 61
Quoted 3 lines
client/tests/kvm/kvm.py +0
-0} # Make it possible to import modules from the test'sbindir
Review
Quoted 1 line
(quoted diff) +0
-0
b/client/tests/kvm/kvm_tests.pyQuoted 10 lines
index 5991aed..ca0b8c0 100644
client/tests/kvm/kvm_tests.py +1
-1@@ -1,4 +1,4 @@ -import time, os, logging +import time, os, logging, re, commands from autotest_lib.client.common_lib import utils, error import kvm_utils, kvm_subprocess, ppm_utils, scan_resultsclient/tests/kvm/kvm_tests.py def run_stress_boot(tests, params, env): · line 533
Quoted 6 lines
client/tests/kvm/kvm_tests.py +0
-1@@ -529,7 +529,6 @@ def run_stress_boot(tests, params, env): """ # boot the first vm vm = kvm_utils.env_get_vm(env, params.get("main_vm")) - if not vm: raise error.TestError("VM object not found inenvironment")
Quoted 10 lines
client/tests/kvm/kvm_tests.py +5
-0if not vm.is_alive():@@ -586,3 +585,161 @@ def run_stress_boot(tests, params, env): for se in sessions: se.close() logging.info("Total number booted: %d" % (num -1)) + + +def run_timedrift(test, params, env): + """ + Time drift test (mainly for Windows guests):client/tests/kvm/kvm_tests.py def run_stress_boot(tests, params, env): · line 602
Quoted 10 lines
client/tests/kvm/kvm_tests.py +10
-0+ + 1) Log into a guest. + 2) Take a time reading from the guest and host. + 3) Run load on the guest and host. + 4) Take a second time reading. + 5) Stop the load and rest for a while. + 6) Take a third time reading. + 7) If the drift immediately after load is higher than a user- + specified value (in %), fail. + If the drift after the rest period is higher than auser-specified value,
client/tests/kvm/kvm_tests.py def run_stress_boot(tests, params, env):
Quoted 9 lines
client/tests/kvm/kvm_tests.py +9
-0+ fail. + + @param test: KVM test object. + @param params: Dictionary with test parameters. + @param env: Dictionary with the test environment. + """ + vm = kvm_utils.env_get_vm(env, params.get("main_vm")) + if not vm: + raise error.TestError("VM object not found inenvironment")
client/tests/kvm/kvm_tests.py def run_stress_boot(tests, params, env): · line 613
Quoted 2 lines
client/tests/kvm/kvm_tests.py +2
-0+ if not vm.is_alive(): + raise error.TestError("VM seems to be dead; Test requires aliving VM")
client/tests/kvm/kvm_tests.py def run_stress_boot(tests, params, env):
Quoted 13 lines
client/tests/kvm/kvm_tests.py +13
-0+ + logging.info("Waiting for guest to be up...") + + session = kvm_utils.wait_for(vm.ssh_login, 240, 0, 2) + if not session: + raise error.TestFail("Could not log into guest") + + logging.info("Logged in") + + # Collect test parameters: + # Command to run to get the current time + time_command = params.get("time_command") + # Filter which should match a string to be passed to
time.strptime()client/tests/kvm/kvm_tests.py def run_stress_boot(tests, params, env):
Quoted 5 lines
client/tests/kvm/kvm_tests.py +5
-0+ time_filter_re = params.get("time_filter_re") + # Time format for time.strptime() + time_format = params.get("time_format") + guest_load_command = params.get("guest_load_command") + guest_load_stop_command =
params.get("guest_load_stop_command")client/tests/kvm/kvm_tests.py def run_stress_boot(tests, params, env): · line 633
Quoted 2 lines
client/tests/kvm/kvm_tests.py +2
-0+ host_load_command = params.get("host_load_command") + guest_load_instances = int(params.get("guest_load_instances","1"))
client/tests/kvm/kvm_tests.py def run_stress_boot(tests, params, env): · line 634
Quoted 1 line
client/tests/kvm/kvm_tests.py +1
-0+ host_load_instances = int(params.get("host_load_instances","0"))
client/tests/kvm/kvm_tests.py def run_stress_boot(tests, params, env):
Quoted 6 lines
client/tests/kvm/kvm_tests.py +6
-0+ # CPU affinity mask for taskset + cpu_mask = params.get("cpu_mask", "0xFF") + load_duration = float(params.get("load_duration", "30")) + rest_duration = float(params.get("rest_duration", "10")) + drift_threshold = float(params.get("drift_threshold", "200")) + drift_threshold_after_rest =float(params.get("drift_threshold_after_rest",
client/tests/kvm/kvm_tests.py def run_stress_boot(tests, params, env):
Quoted 7 lines
client/tests/kvm/kvm_tests.py +7
-0+ "200")) + + guest_load_sessions = [] + host_load_sessions = [] + + # Remember the VM's previous CPU affinity + prev_cpu_mask = commands.getoutput("taskset -p %s" %
vm.get_pid())client/tests/kvm/kvm_tests.py def run_stress_boot(tests, params, env):
Quoted 3 lines
client/tests/kvm/kvm_tests.py +3
-0+ prev_cpu_mask = prev_cpu_mask.split()[-1] + # Set the VM's CPU affinity + commands.getoutput("taskset -p %s %s" % (cpu_mask,
Quoted 8 lines
vm.get_pid()))Need to handle guest smp case where we want to pin the guest to several
cpus.Cheers for the test!
cpu_mask is user-specified. If the user specifies 5, the VM will be pinned to CPUs 1 and 3. In smp tests we can set cpu_mask appropriately. Is this OK or were you referring to something else?
Quoted 106 lines
Quoted 17 lines
client/tests/kvm/kvm_tests.py +17
-0+ + try: + # Get time before load + host_time_0 = time.time() + session.sendline(time_command) + (match, s) = session.read_up_to_prompt() + s = re.findall(time_filter_re, s)[0] + guest_time_0 = time.mktime(time.strptime(s, time_format)) + + # Run some load on the guest + logging.info("Starting load on guest...") + for i in range(guest_load_instances): + load_session = vm.ssh_login() + if not load_session: + raise error.TestFail("Could not log into guest") + load_session.set_output_prefix("(guest load %d) " % i) + load_session.set_output_func(logging.debug)client/tests/kvm/kvm_tests.py def run_stress_boot(tests, params, env):
Quoted 10 lines
client/tests/kvm/kvm_tests.py +10
-0+ load_session.sendline(guest_load_command) + guest_load_sessions.append(load_session) + + # Run some load on the host + logging.info("Starting load on host...") + for i in range(host_load_instances): + host_load_sessions.append( + kvm_subprocess.run_bg(host_load_command, + output_func=logging.debug, + output_prefix="(host load %d)" % i,
client/tests/kvm/kvm_tests.py def run_stress_boot(tests, params, env): · line 679
Quoted 2 lines
client/tests/kvm/kvm_tests.py +2
-0+ timeout=0.5)) + # Set the CPU affinity of the shell running the loadprocess
client/tests/kvm/kvm_tests.py def run_stress_boot(tests, params, env): · line 681
Quoted 2 lines
client/tests/kvm/kvm_tests.py +2
-0+ pid = host_load_sessions[-1].get_shell_pid() + commands.getoutput("taskset -p %s %s" % (cpu_mask,pid))
client/tests/kvm/kvm_tests.py def run_stress_boot(tests, params, env): · line 682
Quoted 1 line
client/tests/kvm/kvm_tests.py +1
-0+ # Try setting the CPU affinity of the load processitself
client/tests/kvm/kvm_tests.py def run_stress_boot(tests, params, env):
Quoted 3 lines
client/tests/kvm/kvm_tests.py +3
-0+ pid = host_load_sessions[-1].get_pid() + if pid: + commands.getoutput("taskset -p %s %s" % (cpu_mask,pid))
Quoted 14 lines
client/tests/kvm/kvm_tests.py +14
-0+ + # Sleep for a while (during load) + logging.info("Sleeping for %s seconds..." % load_duration) + time.sleep(load_duration) + + # Get time delta after load + host_time_1 = time.time() + session.sendline(time_command) + (match, s) = session.read_up_to_prompt() + s = re.findall(time_filter_re, s)[0] + guest_time_1 = time.mktime(time.strptime(s, time_format)) + + # Report results + host_delta = host_time_1 - host_time_0client/tests/kvm/kvm_tests.py def run_stress_boot(tests, params, env):
Quoted 10 lines
client/tests/kvm/kvm_tests.py +10
-0+ guest_delta = guest_time_1 - guest_time_0 + drift = 100.0 * (host_delta - guest_delta) / host_delta + logging.info("Host duration: %.2f" % host_delta) + logging.info("Guest duration: %.2f" % guest_delta) + logging.info("Drift: %.2f%%" % drift) + + finally: + logging.info("Cleaning up...") + # Restore the VM's CPU affinity + commands.getoutput("taskset -p %s %s" % (prev_cpu_mask,
vm.get_pid()))Quoted 14 lines
client/tests/kvm/kvm_tests.py +14
-0+ # Stop the guest load + if guest_load_stop_command: + session.get_command_output(guest_load_stop_command) + # Close all load shell sessions + for load_session in guest_load_sessions: + load_session.close() + for load_session in host_load_sessions: + load_session.close() + + # Sleep again (rest) + logging.info("Sleeping for %s seconds..." % rest_duration) + time.sleep(rest_duration) + + # Get time after restclient/tests/kvm/kvm_tests.py def run_stress_boot(tests, params, env):
Quoted 10 lines
client/tests/kvm/kvm_tests.py +10
-0+ host_time_2 = time.time() + session.sendline(time_command) + (match, s) = session.read_up_to_prompt() + s = re.findall(time_filter_re, s)[0] + guest_time_2 = time.mktime(time.strptime(s, time_format)) + + # Report results + host_delta_total = host_time_2 - host_time_0 + guest_delta_total = guest_time_2 - guest_time_0 + drift_total = 100.0 * (host_delta_total - guest_delta_total) /
host_deltaclient/tests/kvm/kvm_tests.py def run_stress_boot(tests, params, env): · line 734
Quoted 1 line
client/tests/kvm/kvm_tests.py +1
-0+ logging.info("Total host duration including rest: %.2f" %
host_delta_total)client/tests/kvm/kvm_tests.py def run_stress_boot(tests, params, env): · line 735
Quoted 1 line
client/tests/kvm/kvm_tests.py +1
-0+ logging.info("Total guest duration including rest: %.2f" %
guest_delta_total)client/tests/kvm/kvm_tests.py def run_stress_boot(tests, params, env):
Quoted 5 lines
client/tests/kvm/kvm_tests.py +5
-0+ logging.info("Total drift after rest: %.2f%%" % drift_total) + + # Fail the test if necessary + if drift> drift_threshold: + raise error.TestFail("Time drift too large: %.2f%%" %drift)
client/tests/kvm/kvm_tests.py def run_stress_boot(tests, params, env): · line 742
Quoted 2 lines
client/tests/kvm/kvm_tests.py +2
-0+ if drift> drift_threshold_after_rest: + raise error.TestFail("Time drift too large after restperiod: %.2f%%"
Quoted 3 lines
client/tests/kvm/kvm_tests.py +3
-0+ % drift_total) + + session.close()
Re: [Autotest] [KVM-AUTOTEST PATCH 12/17] KVM test: add simple timedrift test (mainly for Windows)
Quoted 142 lines · Michael Goldish
Quoted 140 lines · ----- "Dor Laor
Quoted 9 lines · Michael Goldish
1) Log into a guest.
2) Take a time reading from the guest and host.
3) Run load on the guest and host.
4) Take a second time reading.
5) Stop the load and rest for a while.
6) Take a third time reading.
7) If the drift immediately after load is higher than a user- specified value (in %), fail.
If the drift after the rest period is higher than a user-specifiedvalue,
Quoted 6 lines
fail.
Signed-off-by Michael Goldish<[email protected]>--- client/tests/kvm/kvm.py | 1 + client/tests/kvm/kvm_tests.py | 161++++++++++++++++++++++++++++++++++++++++-
client/tests/kvm/kvm.py class kvm(test.test): · line 55
Quoted 8 lines
2 files changed, 160 insertions(+), 2 deletions(-)client/tests/kvm/kvm.py +0
-0@@ -55,6 +55,7 @@ class kvm(test.test): "kvm_install": test_routine("kvm_install","
run_kvm_install"),client/tests/kvm/kvm.py class kvm(test.test): · line 56
Quoted 1 line
client/tests/kvm/kvm.py +0
-0"linux_s3": test_routine("kvm_tests","
run_linux_s3"),client/tests/kvm/kvm.py class kvm(test.test): · line 57
Quoted 1 line
client/tests/kvm/kvm.py +0
-0"stress_boot": test_routine("kvm_tests","
run_stress_boot"),client/tests/kvm/kvm.py class kvm(test.test): · line 58
Quoted 1 line
client/tests/kvm/kvm.py +1
-0+ "timedrift": test_routine("kvm_tests","
run_timedrift"),client/tests/kvm/kvm.py class kvm(test.test): · line 61
Quoted 3 lines
client/tests/kvm/kvm.py +0
-0} # Make it possible to import modules from the test'sbindir
Review
Quoted 1 line
(quoted diff) +0
-0
b/client/tests/kvm/kvm_tests.pyQuoted 10 lines
index 5991aed..ca0b8c0 100644
client/tests/kvm/kvm_tests.py +1
-1@@ -1,4 +1,4 @@ -import time, os, logging +import time, os, logging, re, commands from autotest_lib.client.common_lib import utils, error import kvm_utils, kvm_subprocess, ppm_utils, scan_resultsclient/tests/kvm/kvm_tests.py def run_stress_boot(tests, params, env): · line 533
Quoted 6 lines
client/tests/kvm/kvm_tests.py +0
-1@@ -529,7 +529,6 @@ def run_stress_boot(tests, params, env): """ # boot the first vm vm = kvm_utils.env_get_vm(env, params.get("main_vm")) - if not vm: raise error.TestError("VM object not found inenvironment")
Quoted 10 lines
client/tests/kvm/kvm_tests.py +5
-0if not vm.is_alive():@@ -586,3 +585,161 @@ def run_stress_boot(tests, params, env): for se in sessions: se.close() logging.info("Total number booted: %d" % (num -1)) + + +def run_timedrift(test, params, env): + """ + Time drift test (mainly for Windows guests):client/tests/kvm/kvm_tests.py def run_stress_boot(tests, params, env): · line 602
Quoted 10 lines
client/tests/kvm/kvm_tests.py +10
-0+ + 1) Log into a guest. + 2) Take a time reading from the guest and host. + 3) Run load on the guest and host. + 4) Take a second time reading. + 5) Stop the load and rest for a while. + 6) Take a third time reading. + 7) If the drift immediately after load is higher than a user- + specified value (in %), fail. + If the drift after the rest period is higher than auser-specified value,
client/tests/kvm/kvm_tests.py def run_stress_boot(tests, params, env):
Quoted 9 lines
client/tests/kvm/kvm_tests.py +9
-0+ fail. + + @param test: KVM test object. + @param params: Dictionary with test parameters. + @param env: Dictionary with the test environment. + """ + vm = kvm_utils.env_get_vm(env, params.get("main_vm")) + if not vm: + raise error.TestError("VM object not found inenvironment")
client/tests/kvm/kvm_tests.py def run_stress_boot(tests, params, env): · line 613
Quoted 2 lines
client/tests/kvm/kvm_tests.py +2
-0+ if not vm.is_alive(): + raise error.TestError("VM seems to be dead; Test requires aliving VM")
client/tests/kvm/kvm_tests.py def run_stress_boot(tests, params, env):
Quoted 13 lines
client/tests/kvm/kvm_tests.py +13
-0+ + logging.info("Waiting for guest to be up...") + + session = kvm_utils.wait_for(vm.ssh_login, 240, 0, 2) + if not session: + raise error.TestFail("Could not log into guest") + + logging.info("Logged in") + + # Collect test parameters: + # Command to run to get the current time + time_command = params.get("time_command") + # Filter which should match a string to be passed to
time.strptime()client/tests/kvm/kvm_tests.py def run_stress_boot(tests, params, env):
Quoted 5 lines
client/tests/kvm/kvm_tests.py +5
-0+ time_filter_re = params.get("time_filter_re") + # Time format for time.strptime() + time_format = params.get("time_format") + guest_load_command = params.get("guest_load_command") + guest_load_stop_command =
params.get("guest_load_stop_command")client/tests/kvm/kvm_tests.py def run_stress_boot(tests, params, env): · line 633
Quoted 2 lines
client/tests/kvm/kvm_tests.py +2
-0+ host_load_command = params.get("host_load_command") + guest_load_instances = int(params.get("guest_load_instances","1"))
client/tests/kvm/kvm_tests.py def run_stress_boot(tests, params, env): · line 634
Quoted 1 line
client/tests/kvm/kvm_tests.py +1
-0+ host_load_instances = int(params.get("host_load_instances","0"))
client/tests/kvm/kvm_tests.py def run_stress_boot(tests, params, env):
Quoted 6 lines
client/tests/kvm/kvm_tests.py +6
-0+ # CPU affinity mask for taskset + cpu_mask = params.get("cpu_mask", "0xFF") + load_duration = float(params.get("load_duration", "30")) + rest_duration = float(params.get("rest_duration", "10")) + drift_threshold = float(params.get("drift_threshold", "200")) + drift_threshold_after_rest =float(params.get("drift_threshold_after_rest",
client/tests/kvm/kvm_tests.py def run_stress_boot(tests, params, env):
Quoted 7 lines
client/tests/kvm/kvm_tests.py +7
-0+ "200")) + + guest_load_sessions = [] + host_load_sessions = [] + + # Remember the VM's previous CPU affinity + prev_cpu_mask = commands.getoutput("taskset -p %s" %
vm.get_pid())client/tests/kvm/kvm_tests.py def run_stress_boot(tests, params, env):
Quoted 3 lines
client/tests/kvm/kvm_tests.py +3
-0+ prev_cpu_mask = prev_cpu_mask.split()[-1] + # Set the VM's CPU affinity + commands.getoutput("taskset -p %s %s" % (cpu_mask,
vm.get_pid()))Need to handle guest smp case where we want to pin the guest to several
cpus.
Quoted 5 lines
Quoted 1 line
Cheers for the test!
cpu_maskis user-specified. If the user specifies 5, the VM will be pinned to CPUs 1 and 3. In smp tests we can setcpu_maskappropriately. Is this OK or were you referring to something else?
In that case it is good. Thanks.
Quoted 107 lines
Quoted 107 lines
Quoted 18 lines
client/tests/kvm/kvm_tests.py +17
-0+ + try: + # Get time before load + host_time_0 = time.time() + session.sendline(time_command) + (match, s) = session.read_up_to_prompt() + s = re.findall(time_filter_re, s)[0] + guest_time_0 = time.mktime(time.strptime(s, time_format)) + + # Run some load on the guest + logging.info("Starting load on guest...") + for i in range(guest_load_instances): + load_session = vm.ssh_login() + if not load_session: + raise error.TestFail("Could not log into guest") + load_session.set_output_prefix("(guest load %d) " % i) + load_session.set_output_func(logging.debug)client/tests/kvm/kvm_tests.py def run_stress_boot(tests, params, env):
Quoted 10 lines
client/tests/kvm/kvm_tests.py +10
-0+ load_session.sendline(guest_load_command) + guest_load_sessions.append(load_session) + + # Run some load on the host + logging.info("Starting load on host...") + for i in range(host_load_instances): + host_load_sessions.append( + kvm_subprocess.run_bg(host_load_command, + output_func=logging.debug, + output_prefix="(host load %d)" % i,
client/tests/kvm/kvm_tests.py def run_stress_boot(tests, params, env): · line 679
Quoted 2 lines
client/tests/kvm/kvm_tests.py +2
-0+ timeout=0.5)) + # Set the CPU affinity of the shell running the loadprocess
client/tests/kvm/kvm_tests.py def run_stress_boot(tests, params, env): · line 681
Quoted 2 lines
client/tests/kvm/kvm_tests.py +2
-0+ pid = host_load_sessions[-1].get_shell_pid() + commands.getoutput("taskset -p %s %s" % (cpu_mask,pid))
client/tests/kvm/kvm_tests.py def run_stress_boot(tests, params, env): · line 682
Quoted 1 line
client/tests/kvm/kvm_tests.py +1
-0+ # Try setting the CPU affinity of the load processitself
client/tests/kvm/kvm_tests.py def run_stress_boot(tests, params, env):
Quoted 3 lines
client/tests/kvm/kvm_tests.py +3
-0+ pid = host_load_sessions[-1].get_pid() + if pid: + commands.getoutput("taskset -p %s %s" % (cpu_mask,pid))
Quoted 14 lines
client/tests/kvm/kvm_tests.py +14
-0+ + # Sleep for a while (during load) + logging.info("Sleeping for %s seconds..." % load_duration) + time.sleep(load_duration) + + # Get time delta after load + host_time_1 = time.time() + session.sendline(time_command) + (match, s) = session.read_up_to_prompt() + s = re.findall(time_filter_re, s)[0] + guest_time_1 = time.mktime(time.strptime(s, time_format)) + + # Report results + host_delta = host_time_1 - host_time_0client/tests/kvm/kvm_tests.py def run_stress_boot(tests, params, env):
Quoted 10 lines
client/tests/kvm/kvm_tests.py +10
-0+ guest_delta = guest_time_1 - guest_time_0 + drift = 100.0 * (host_delta - guest_delta) / host_delta + logging.info("Host duration: %.2f" % host_delta) + logging.info("Guest duration: %.2f" % guest_delta) + logging.info("Drift: %.2f%%" % drift) + + finally: + logging.info("Cleaning up...") + # Restore the VM's CPU affinity + commands.getoutput("taskset -p %s %s" % (prev_cpu_mask,
vm.get_pid()))Quoted 14 lines
client/tests/kvm/kvm_tests.py +14
-0+ # Stop the guest load + if guest_load_stop_command: + session.get_command_output(guest_load_stop_command) + # Close all load shell sessions + for load_session in guest_load_sessions: + load_session.close() + for load_session in host_load_sessions: + load_session.close() + + # Sleep again (rest) + logging.info("Sleeping for %s seconds..." % rest_duration) + time.sleep(rest_duration) + + # Get time after restclient/tests/kvm/kvm_tests.py def run_stress_boot(tests, params, env):
Quoted 10 lines
client/tests/kvm/kvm_tests.py +10
-0+ host_time_2 = time.time() + session.sendline(time_command) + (match, s) = session.read_up_to_prompt() + s = re.findall(time_filter_re, s)[0] + guest_time_2 = time.mktime(time.strptime(s, time_format)) + + # Report results + host_delta_total = host_time_2 - host_time_0 + guest_delta_total = guest_time_2 - guest_time_0 + drift_total = 100.0 * (host_delta_total - guest_delta_total) /
host_deltaclient/tests/kvm/kvm_tests.py def run_stress_boot(tests, params, env): · line 734
Quoted 1 line
client/tests/kvm/kvm_tests.py +1
-0+ logging.info("Total host duration including rest: %.2f" %
host_delta_total)client/tests/kvm/kvm_tests.py def run_stress_boot(tests, params, env): · line 735
Quoted 1 line
client/tests/kvm/kvm_tests.py +1
-0+ logging.info("Total guest duration including rest: %.2f" %
guest_delta_total)client/tests/kvm/kvm_tests.py def run_stress_boot(tests, params, env):
Quoted 5 lines
client/tests/kvm/kvm_tests.py +5
-0+ logging.info("Total drift after rest: %.2f%%" % drift_total) + + # Fail the test if necessary + if drift> drift_threshold: + raise error.TestFail("Time drift too large: %.2f%%" %drift)
client/tests/kvm/kvm_tests.py def run_stress_boot(tests, params, env): · line 742
Quoted 2 lines
client/tests/kvm/kvm_tests.py +2
-0+ if drift> drift_threshold_after_rest: + raise error.TestFail("Time drift too large after restperiod: %.2f%%"
Quoted 3 lines
client/tests/kvm/kvm_tests.py +3
-0+ % drift_total) + + session.close()
Re: [Autotest] [KVM-AUTOTEST PATCH 12/17] KVM test: add simple timedrift test (mainly for Windows)
In reply to Dor LaorQuoted 120 lines · Dor Laor
Quoted 104 lines · Michael Goldish
1) Log into a guest.
2) Take a time reading from the guest and host.
3) Run load on the guest and host.
4) Take a second time reading.
5) Stop the load and rest for a while.
6) Take a third time reading.
7) If the drift immediately after load is higher than a user- specified value (in %), fail.
If the drift after the rest period is higher than a user-specified value, fail.Signed-off-by Michael Goldish<[email protected]>--- client/tests/kvm/kvm.py | 1 + client/tests/kvm/kvm_tests.py | 161 ++++++++++++++++++++++++++++++++++++++++- 2 files changed, 160 insertions(+), 2 deletions(-)client/tests/kvm/kvm.py +1
-0@@ -55,6 +55,7 @@ class kvm(test.test): "kvm_install": test_routine("kvm_install", "run_kvm_install"), "linux_s3": test_routine("kvm_tests", "run_linux_s3"), "stress_boot": test_routine("kvm_tests", "run_stress_boot"), + "timedrift": test_routine("kvm_tests", "run_timedrift"), } # Make it possible to import modules from the test's bindirclient/tests/kvm/kvm_tests.py +54
-2@@ -1,4 +1,4 @@ -import time, os, logging +import time, os, logging, re, commands from autotest_lib.client.common_lib import utils, error import kvm_utils, kvm_subprocess, ppm_utils, scan_results@@ -529,7 +529,6 @@ def run_stress_boot(tests, params, env): """ # boot the first vm vm = kvm_utils.env_get_vm(env, params.get("main_vm")) - if not vm: raise error.TestError("VM object not found in environment") if not vm.is_alive():@@ -586,3 +585,161 @@ def run_stress_boot(tests, params, env): for se in sessions: se.close() logging.info("Total number booted: %d" % (num -1)) + + +def run_timedrift(test, params, env): + """ + Time drift test (mainly for Windows guests): + + 1) Log into a guest. + 2) Take a time reading from the guest and host. + 3) Run load on the guest and host. + 4) Take a second time reading. + 5) Stop the load and rest for a while. + 6) Take a third time reading. + 7) If the drift immediately after load is higher than a user- + specified value (in %), fail. + If the drift after the rest period is higher than a user-specified value, + fail. + + @param test: KVM test object. + @param params: Dictionary with test parameters. + @param env: Dictionary with the test environment. + """ + vm = kvm_utils.env_get_vm(env, params.get("main_vm")) + if not vm: + raise error.TestError("VM object not found in environment") + if not vm.is_alive(): + raise error.TestError("VM seems to be dead; Test requires a living VM") + + logging.info("Waiting for guest to be up...") + + session = kvm_utils.wait_for(vm.ssh_login, 240, 0, 2) + if not session: + raise error.TestFail("Could not log into guest") + + logging.info("Logged in") + + # Collect test parameters: + # Command to run to get the current time + time_command = params.get("time_command") + # Filter which should match a string to be passed to time.strptime() + time_filter_re = params.get("time_filter_re") + # Time format for time.strptime() + time_format = params.get("time_format") + guest_load_command = params.get("guest_load_command") + guest_load_stop_command = params.get("guest_load_stop_command") + host_load_command = params.get("host_load_command") + guest_load_instances = int(params.get("guest_load_instances", "1")) + host_load_instances = int(params.get("host_load_instances", "0")) + # CPU affinity mask for taskset + cpu_mask = params.get("cpu_mask", "0xFF") + load_duration = float(params.get("load_duration", "30")) + rest_duration = float(params.get("rest_duration", "10")) + drift_threshold = float(params.get("drift_threshold", "200")) + drift_threshold_after_rest = float(params.get("drift_threshold_after_rest",client/tests/kvm/kvm_tests.py def run_stress_boot(tests, params, env): · line 650
Quoted 10 lines
client/tests/kvm/kvm_tests.py +10
-0+ "200")) + + guest_load_sessions = [] + host_load_sessions = [] + + # Remember the VM's previous CPU affinity + prev_cpu_mask = commands.getoutput("taskset -p %s" % vm.get_pid()) + prev_cpu_mask = prev_cpu_mask.split()[-1] + # Set the VM's CPU affinity + commands.getoutput("taskset -p %s %s" % (cpu_mask, vm.get_pid()))Need to handle guest smp case where we want to pin the guest to several cpus.
Quoted 1 line
Cheers for the test!
Yes, very nice.
A improvement suggestion: the system load / cpu pin characteristics should be generic, and not specific to this particular test. Because other tests also benefit from it (one obvious candidate is migration) ?
Regarding timedrift, it should know if there is an ntp client running on the guest (now whether ntp configuration should be part of the testcase, or should belong outside of it, independently somehow, i'm unsure).
Its necessary to measure with/without ntp client.
Re: [Autotest] [KVM-AUTOTEST PATCH 12/17] KVM test: add simple timedrift test (mainly for Windows)
In reply to Michael GoldishQuoted 112 lines · Michael Goldish
1) Log into a guest.
2) Take a time reading from the guest and host.
3) Run load on the guest and host.
4) Take a second time reading.
5) Stop the load and rest for a while.
6) Take a third time reading.
7) If the drift immediately after load is higher than a user- specified value (in %), fail.
If the drift after the rest period is higher than a user-specified value, fail.Signed-off-by Michael Goldish <[email protected]>--- client/tests/kvm/kvm.py | 1 + client/tests/kvm/kvm_tests.py | 161 ++++++++++++++++++++++++++++++++++++++++- 2 files changed, 160 insertions(+), 2 deletions(-)client/tests/kvm/kvm.py +1
-0@@ -55,6 +55,7 @@ class kvm(test.test): "kvm_install": test_routine("kvm_install", "run_kvm_install"), "linux_s3": test_routine("kvm_tests", "run_linux_s3"), "stress_boot": test_routine("kvm_tests", "run_stress_boot"), + "timedrift": test_routine("kvm_tests", "run_timedrift"), } # Make it possible to import modules from the test's bindirclient/tests/kvm/kvm_tests.py +62
-2@@ -1,4 +1,4 @@ -import time, os, logging +import time, os, logging, re, commands from autotest_lib.client.common_lib import utils, error import kvm_utils, kvm_subprocess, ppm_utils, scan_results@@ -529,7 +529,6 @@ def run_stress_boot(tests, params, env): """ # boot the first vm vm = kvm_utils.env_get_vm(env, params.get("main_vm")) - if not vm: raise error.TestError("VM object not found in environment") if not vm.is_alive():@@ -586,3 +585,161 @@ def run_stress_boot(tests, params, env): for se in sessions: se.close() logging.info("Total number booted: %d" % (num -1)) + + +def run_timedrift(test, params, env): + """ + Time drift test (mainly for Windows guests): + + 1) Log into a guest. + 2) Take a time reading from the guest and host. + 3) Run load on the guest and host. + 4) Take a second time reading. + 5) Stop the load and rest for a while. + 6) Take a third time reading. + 7) If the drift immediately after load is higher than a user- + specified value (in %), fail. + If the drift after the rest period is higher than a user-specified value, + fail. + + @param test: KVM test object. + @param params: Dictionary with test parameters. + @param env: Dictionary with the test environment. + """ + vm = kvm_utils.env_get_vm(env, params.get("main_vm")) + if not vm: + raise error.TestError("VM object not found in environment") + if not vm.is_alive(): + raise error.TestError("VM seems to be dead; Test requires a living VM") + + logging.info("Waiting for guest to be up...") + + session = kvm_utils.wait_for(vm.ssh_login, 240, 0, 2) + if not session: + raise error.TestFail("Could not log into guest") + + logging.info("Logged in") + + # Collect test parameters: + # Command to run to get the current time + time_command = params.get("time_command") + # Filter which should match a string to be passed to time.strptime() + time_filter_re = params.get("time_filter_re") + # Time format for time.strptime() + time_format = params.get("time_format") + guest_load_command = params.get("guest_load_command") + guest_load_stop_command = params.get("guest_load_stop_command") + host_load_command = params.get("host_load_command") + guest_load_instances = int(params.get("guest_load_instances", "1")) + host_load_instances = int(params.get("host_load_instances", "0")) + # CPU affinity mask for taskset + cpu_mask = params.get("cpu_mask", "0xFF") + load_duration = float(params.get("load_duration", "30")) + rest_duration = float(params.get("rest_duration", "10")) + drift_threshold = float(params.get("drift_threshold", "200")) + drift_threshold_after_rest = float(params.get("drift_threshold_after_rest", + "200")) + + guest_load_sessions = [] + host_load_sessions = [] + + # Remember the VM's previous CPU affinity + prev_cpu_mask = commands.getoutput("taskset -p %s" % vm.get_pid()) + prev_cpu_mask = prev_cpu_mask.split()[-1]
client/tests/kvm/kvm_tests.py def run_stress_boot(tests, params, env): · line 658
Quoted 10 lines
client/tests/kvm/kvm_tests.py +10
-0+ # Set the VM's CPU affinity + commands.getoutput("taskset -p %s %s" % (cpu_mask, vm.get_pid())) + + try: + # Get time before load + host_time_0 = time.time() + session.sendline(time_command) + (match, s) = session.read_up_to_prompt() + s = re.findall(time_filter_re, s)[0] + guest_time_0 = time.mktime(time.strptime(s, time_format))
Hi Machael, this test looks good for me, but I have a little suggestion:
Why not write a common function to get guest time which really save many duplicate codes here? It could has four parameters and I would also help write it for you :-)
def get_guest_time(session, time_command, filter_re, format):
session.sendline(time_command)
(match, s) = session.read_up_to_prompt()
s = re.findall(filter_re, s)[0]
curr_time = time.mktime(time.strptime(s, format)) return curr_time
Quoted 94 lines
client/tests/kvm/kvm_tests.py +87
-0+ + # Run some load on the guest + logging.info("Starting load on guest...") + for i in range(guest_load_instances): + load_session = vm.ssh_login() + if not load_session: + raise error.TestFail("Could not log into guest") + load_session.set_output_prefix("(guest load %d) " % i) + load_session.set_output_func(logging.debug) + load_session.sendline(guest_load_command) + guest_load_sessions.append(load_session) + + # Run some load on the host + logging.info("Starting load on host...") + for i in range(host_load_instances): + host_load_sessions.append( + kvm_subprocess.run_bg(host_load_command, + output_func=logging.debug, + output_prefix="(host load %d) " % i, + timeout=0.5)) + # Set the CPU affinity of the shell running the load process + pid = host_load_sessions[-1].get_shell_pid() + commands.getoutput("taskset -p %s %s" % (cpu_mask, pid)) + # Try setting the CPU affinity of the load process itself + pid = host_load_sessions[-1].get_pid() + if pid: + commands.getoutput("taskset -p %s %s" % (cpu_mask, pid)) + + # Sleep for a while (during load) + logging.info("Sleeping for %s seconds..." % load_duration) + time.sleep(load_duration) + + # Get time delta after load + host_time_1 = time.time() + session.sendline(time_command) + (match, s) = session.read_up_to_prompt() + s = re.findall(time_filter_re, s)[0] + guest_time_1 = time.mktime(time.strptime(s, time_format)) + + # Report results + host_delta = host_time_1 - host_time_0 + guest_delta = guest_time_1 - guest_time_0 + drift = 100.0 * (host_delta - guest_delta) / host_delta + logging.info("Host duration: %.2f" % host_delta) + logging.info("Guest duration: %.2f" % guest_delta) + logging.info("Drift: %.2f%%" % drift) + + finally: + logging.info("Cleaning up...") + # Restore the VM's CPU affinity + commands.getoutput("taskset -p %s %s" % (prev_cpu_mask, vm.get_pid())) + # Stop the guest load + if guest_load_stop_command: + session.get_command_output(guest_load_stop_command) + # Close all load shell sessions + for load_session in guest_load_sessions: + load_session.close() + for load_session in host_load_sessions: + load_session.close() + + # Sleep again (rest) + logging.info("Sleeping for %s seconds..." % rest_duration) + time.sleep(rest_duration) + + # Get time after rest + host_time_2 = time.time() + session.sendline(time_command) + (match, s) = session.read_up_to_prompt() + s = re.findall(time_filter_re, s)[0] + guest_time_2 = time.mktime(time.strptime(s, time_format)) + + # Report results + host_delta_total = host_time_2 - host_time_0 + guest_delta_total = guest_time_2 - guest_time_0 + drift_total = 100.0 * (host_delta_total - guest_delta_total) / host_delta + logging.info("Total host duration including rest: %.2f" % host_delta_total) + logging.info("Total guest duration including rest: %.2f" % guest_delta_total) + logging.info("Total drift after rest: %.2f%%" % drift_total) + + # Fail the test if necessary + if drift > drift_threshold: + raise error.TestFail("Time drift too large: %.2f%%" % drift) + if drift > drift_threshold_after_rest: + raise error.TestFail("Time drift too large after rest period: %.2f%%" + % drift_total) + + session.close()Signature · 6 lines
1.5.4.1 _______________________________________________ Autotest mailing list [email protected] http://test.kernel.org/cgi-bin/mailman/listinfo/autotest
Lucas Meneghel Rodrigues
AppliedRe: [Autotest] [KVM-AUTOTEST PATCH 11/17] KVM test: kvm_tests.cfg.sample: convert PPM files to PNG by default
In reply to Michael GoldishQuoted 8 lines · Michael Goldish
By default, always remove PPM files, and keep PNG files only for failed tests. This shouldn't do much harm, because while PPMs can be incorporated directly into step files, PNGs can be converted back to PPMs easily, and take less disk space.
(PNG is a lossless compression format.)The '
keep_ppm_files' and 'keep_ppm_files_on_error' settings are commented out so the user can easily enable them if desired.
Applied.
Quoted 29 lines
Signed-off-by Michael Goldish <[email protected]>---
client/tests/kvm/kvm_tests.cfg.sample | 5 +++--1 files changed, 3 insertions(+), 2 deletions(-)client/tests/kvm/kvm_tests.cfg.sample +0
-0@@ -8,8 +8,9 @@ main_vm = vm1 # Some preprocessor/postprocessor paramsstart_vm = yes
client/tests/kvm/kvm_tests.cfg.sample +3
-2-keep_ppm_files = no -keep_ppm_files_on_error = yes +convert_ppm_files_to_png_on_error = yes +#keep_ppm_files = yes +#keep_ppm_files_on_error = yeskill_vm = no
kill_vm_gracefully = yes--
1.5.4.1_______________________________________________
Autotest mailing list
[email protected]
http://test.kernel.org/cgi-bin/mailman/listinfo/autotest
Signature · 1 line
Lucas Meneghel
Lucas Meneghel Rodrigues
AppliedRe: [Autotest] [KVM-AUTOTEST PATCH 10/17] KVM test: optionally convert PPM files to PNG format after test
In reply to Michael GoldishQuoted 15 lines · Michael Goldish
This is intended to save disk space. Requires ImageMagick (uses mogrify).
To enable:
convert_ppm_files_to_png= yesTo enable only for failed tests:
convert_ppm_files_to_png_on_error= yesReminder: by default PPM files are removed after the test (and after the conversion, if requested). To keep them specify:
keep_ppm_files= yesTo keep them only for failed tests:
keep_ppm_files_on_error= yes
Quoted 8 lines
A reasonable choice would be to keep PNG files for failed tests, and never keep PPM files. To do this specify
convert_ppm_files_to_png_on_error= yes
without specifying 'keep_ppm_files= yes' or 'keep_ppm_files_on_error= yes' (or explicitly set to 'no', if it was set to 'yes' on a higher level in the config hierarchy).This patch also makes small cosmetic changes to the '
keep_ppm_files' feature.
Applied.
Quoted 41 lines
Signed-off-by Michael Goldish <[email protected]>---
client/tests/kvm/kvm_preprocessing.py | 16 ++++++++++++----1 files changed, 12 insertions(+), 4 deletions(-)client/tests/kvm/kvm_preprocessing.py +0
-0@@ -264,11 +264,19 @@ def postprocess(test, params, env):"""
process(test, params, env, postprocess_image, postprocess_vm)- # See if we should get rid of all PPM files
- if notparams.get("keep_ppm_files")== "yes":
- # Remove them all
+ # Should we convert PPM files to PNG format?
+ ifparams.get("convert_ppm_files_to_png")== "yes":
+ logging.debug("'convert_ppm_files_to_png' specified; converting PPM"
+ " files to PNG format...")
+ mogrify_cmd = ("mogrify -format png %s" %
+os.path.join(test.debugdir, "*.ppm"))
+ kvm_subprocess.run_fg(mogrify_cmd, logging.debug, "(mogrify) ",
+ timeout=30.0)
+
+ # Should we keep the PPM files?
+ ifparams.get("keep_ppm_files")!= "yes":
logging.debug("'keep_ppm_files' not specified; removing all PPM files"
- " from results dir...")
+ " from debug dir...")
rm_cmd= "rm -vf %s" %os.path.join(test.debugdir, "*.ppm")kvm_subprocess.run_fg(rm_cmd, logging.debug, "(rm) ", timeout=5.0)--
1.5.4.1_______________________________________________
Autotest mailing list
[email protected]
http://test.kernel.org/cgi-bin/mailman/listinfo/autotest
Signature · 1 line
Lucas Meneghel
Lucas Meneghel Rodrigues
AppliedRe: [Autotest] [KVM-AUTOTEST PATCH 08/17] kvm_guest_wizard: allow keeping screendump history for debugging purposes
In reply to Michael GoldishQuoted 8 lines · Michael Goldish
Add two new step file test parameters:
-keep_screendump_history: if equals 'yes', screendump history is saved in test.debugdir/barrier_history in JPG format. Each screendump taken by the test is saved if it differs from the previous screendump. By default, when a barrier succeeds all history is removed, so eventually the remaining files are only those of the (last) failed barrier, if any.
-keep_all_history: if equals 'yes', screendump history is not removed upon barrier success. The test leaves behind all the collected screendump history.
Applied.
Quoted 93 lines
Signed-off-by Michael Goldish <[email protected]>---
client/tests/kvm/kvm_guest_wizard.py | 38 ++++++++++++++++++++++++++++-----1 files changed, 32 insertions(+), 6 deletions(-)client/tests/kvm/kvm_guest_wizard.py +0
-0@@ -1,6 +1,6 @@import os, time, md5, re, shutil, logging
from autotest_lib.client.common_lib import utils, errorclient/tests/kvm/kvm_guest_wizard.py +1
-1-import kvm_utils, ppm_utils +import kvm_utils, ppm_utils, kvm_subprocess"""
Utilities to perform automatic guest installation using step files.client/tests/kvm/kvm_guest_wizard.py +0
-0@@ -53,6 +53,11 @@ def barrier_2(vm, words, params, debug_dir, data_scrdump_filename,else:
stuck_detection_history= 2+ keep_screendump_history =
params.get("keep_screendump_history")== "yes"
+ ifkeep_screendump_history:
+ keep_all_history =params.get("keep_all_history")== "yes"
+ history_dir =os.path.join(debug_dir, "barrier_history")+
end_time=time.time()+ timeout
end_time_stuck=time.time()+fail_if_stuck_for
start_time=time.time()client/tests/kvm/kvm_guest_wizard.py +18
-0@@ -91,21 +96,42 @@ def barrier_2(vm, words, params, debug_dir, data_scrdump_filename, # Read image file (w, h, data) = ppm_utils.image_read_from_ppm_file(scrdump_filename) + # Compute md5sum of whole image + whole_image_md5sum = ppm_utils.image_md5sum(w, h, data) + + # Write screendump to history_dir (as JPG) if requested + # and if the screendump differs from the previous one + if (keep_screendump_history and + whole_image_md5sum not in prev_whole_image_md5sums[:1]): + try: + os.makedirs(history_dir) + except: + pass + history_scrdump_filename = os.path.join(history_dir, + "scrdump-step_%s-%s.jpg" % (current_step_num, + time.strftime("%Y%m%d-%H%M%S"))) + kvm_subprocess.run_fg("convert -quality 30 %s %s" % + (scrdump_filename, history_scrdump_filename), + logging.debug, "(convert) ", timeout=30) + # Compare md5sum of barrier region with the expected md5sum calced_md5sum = ppm_utils.get_region_md5sum(w, h, data, x1, y1, dx, dy, cropped_scrdump_filename)if
calced_md5sum== md5sum:
+ # Success -- remove screendump history unless requested not to
+ ifkeep_screendump_historyand notkeep_all_history:
+ kvm_subprocess.run_fg("rm -rvf %s" %history_dir,
+ logging.debug, "(rm) ", timeout=30)
+ # Report success
return True- # Compute md5sum of whole image in order to compare it with
- # previous ones
- whole_image_md5sum =ppm_utils.image_md5sum(w, h, data)
+ # Insert image md5sum into queue of last seen images:
# If md5sum is already in queue...
ifwhole_image_md5suminprev_whole_image_md5sums:
# Remove md5sum from queueprev_whole_image_md5sums.remove(whole_image_md5sum)else:
- # Extend 'stuck' timeout
+ # Otherwise extend 'stuck' timeoutend_time_stuck=time.time()+fail_if_stuck_for# Insert md5sum at beginning of queueprev_whole_image_md5sums.insert(0, whole_image_md5sum)client/tests/kvm/kvm_guest_wizard.py +0
-0@@ -129,7 +155,7 @@ def barrier_2(vm, words, params, debug_dir, data_scrdump_filename,if
data_scrdump_filenameandos.path.exists(data_scrdump_filename):
# Read expected screendump image
(ew, eh, edata) = \
- ppm_utils.image_read_from_ppm_file(data_scrdump_filename)
+ ppm_utils.image_read_from_ppm_file(data_scrdump_filename) # Write it indebug_dirppm_utils.image_write_to_ppm_file(expected_scrdump_filename, ew, eh, edata)
--
1.5.4.1_______________________________________________
Autotest mailing list
[email protected]
http://test.kernel.org/cgi-bin/mailman/listinfo/autotest
Signature · 1 line
Lucas Meneghel
Lucas Meneghel Rodrigues
AppliedRe: [Autotest] [KVM-AUTOTEST PATCH 07/17] kvm_guest_wizard: pass 'params' directly to barrier_2()
In reply to Michael GoldishQuoted 4 lines · Michael Goldish
Currently parameters for
barrier_2()are extracted from 'params' in the mainrun_steps()test routine, and then passed tobarrier_2(). Instead, letbarrier_2()extract parameters from 'params' as it sees fit. This will make adding new parameters slightly easier and cleaner.
Applied.
Quoted 84 lines
Signed-off-by Michael Goldish <[email protected]>---
client/tests/kvm/kvm_guest_wizard.py | 37 ++++++++++++++++-----------------1 files changed, 18 insertions(+), 19 deletions(-)client/tests/kvm/kvm_guest_wizard.py +0
-0@@ -17,8 +17,8 @@ def handle_var(vm, params, varname):return True
client/tests/kvm/kvm_guest_wizard.py +2
-2-def barrier_2(vm, words, fail_if_stuck_for, stuck_detection_history, - debug_dir, data_scrdump_filename, current_step_num): +def barrier_2(vm, words, params, debug_dir, data_scrdump_filename, + current_step_num):if len(words) < 7:
logging.error("Bad barrier_2 command line")return Falseclient/tests/kvm/kvm_guest_wizard.py +12
-0@@ -41,6 +41,18 @@ def barrier_2(vm, words, fail_if_stuck_for, stuck_detection_history, "cropped_scrdump_expected.ppm") comparison_filename = os.path.join(debug_dir, "comparison.ppm") + fail_if_stuck_for = params.get("fail_if_stuck_for") + if fail_if_stuck_for: + fail_if_stuck_for = float(fail_if_stuck_for) + else: + fail_if_stuck_for = 1e308 + + stuck_detection_history = params.get("stuck_detection_history") + if stuck_detection_history: + stuck_detection_history = int(stuck_detection_history) + else: + stuck_detection_history = 2 +
end_time=time.time()+ timeoutend_time_stuck=time.time()+fail_if_stuck_forstart_time=time.time()client/tests/kvm/kvm_guest_wizard.py +0
-0@@ -151,18 +163,6 @@ def run_steps(test, params, env):if not
os.path.exists(steps_filename):
raiseerror.TestError("Steps file not found: %s" % steps_filename)- fail_if_stuck_for =
params.get("fail_if_stuck_for")
- iffail_if_stuck_for:
- fail_if_stuck_for =float(fail_if_stuck_for)
- else:
- fail_if_stuck_for = 1e308client/tests/kvm/kvm_guest_wizard.py +0
-7- - stuck_detection_history = params.get("stuck_detection_history") - if stuck_detection_history: - stuck_detection_history = int(stuck_detection_history) - else: - stuck_detection_history = 2 -sf =
open(steps_filename, "r")
lines =sf.readlines()
sf.close()client/tests/kvm/kvm_guest_wizard.py +0
-0@@ -201,13 +201,12 @@ def run_steps(test, params, env): logging.error("Variable not defined: %s" % words[1])elif words[0] == "
barrier_2":
ifcurrent_screendump:
- scrdump_filename = (
- os.path.join(ppm_utils.get_data_dir(steps_filename),
-current_screendump))
+ scrdump_filename = os.path.join(
+ ppm_utils.get_data_dir(steps_filename),
+ current_screendump)
else:
scrdump_filename= None
- if not barrier_2(vm, words,fail_if_stuck_for,
-stuck_detection_history, test.debugdir,
+ if notbarrier_2(vm, words, params, test.debugdir, scrdump_filename, current_step_num):
skip_current_step= True
else:
--
1.5.4.1_______________________________________________
Autotest mailing list
[email protected]
http://test.kernel.org/cgi-bin/mailman/listinfo/autotest
Signature · 1 line
Lucas Meneghel
Lucas Meneghel Rodrigues
AppliedRe: [Autotest] [KVM-AUTOTEST PATCH 06/17] kvm_guest_wizard: rename output_dir to debug_dir in barrier_2()
In reply to Michael GoldishQuoted 1 line · Michael Goldish
The name '
debug_dir' makes it clearer that it corresponds to test.debugdir.
Applied.
Quoted 77 lines
Signed-off-by Michael Goldish <[email protected]>---
client/tests/kvm/kvm_guest_wizard.py | 20 ++++++++++----------1 files changed, 10 insertions(+), 10 deletions(-)client/tests/kvm/kvm_guest_wizard.py +1
-1@@ -18,7 +18,7 @@ def handle_var(vm, params, varname): def barrier_2(vm, words, fail_if_stuck_for, stuck_detection_history, - output_dir, data_scrdump_filename, current_step_num): + debug_dir, data_scrdump_filename, current_step_num):if len(words) < 7:
logging.error("Bad barrier_2 command line")return Falseclient/tests/kvm/kvm_guest_wizard.py +0
-0@@ -34,12 +34,12 @@ def barrier_2(vm, words, fail_if_stuck_for, stuck_detection_history,if
sleep_duration< 1.0:sleep_duration= 1.0 ifsleep_duration> 10.0:sleep_duration= 10.0- scrdump_filename =
os.path.join(output_dir, "scrdump.ppm")
- cropped_scrdump_filename =os.path.join(output_dir, "cropped_scrdump.ppm")
- expected_scrdump_filename =os.path.join(output_dir, "scrdump_expected.ppm")
- expected_cropped_scrdump_filename = os.path.join(output_dir,
+ scrdump_filename =os.path.join(debug_dir, "scrdump.ppm")
+ cropped_scrdump_filename =os.path.join(debug_dir, "cropped_scrdump.ppm")
+ expected_scrdump_filename =os.path.join(debug_dir, "scrdump_expected.ppm")
+ expected_cropped_scrdump_filename =os.path.join(debug_dir, "cropped_scrdump_expected.ppm")
- comparison_filename =os.path.join(output_dir, "comparison.ppm")
+ comparison_filename =os.path.join(debug_dir, "comparison.ppm")
end_time=time.time()+ timeoutend_time_stuck=time.time()+fail_if_stuck_forclient/tests/kvm/kvm_guest_wizard.py +1
-1@@ -99,7 +99,7 @@ def barrier_2(vm, words, fail_if_stuck_for, stuck_detection_history, prev_whole_image_md5sums.insert(0, whole_image_md5sum) # Limit queue length to stuck_detection_history prev_whole_image_md5sums = \ - prev_whole_image_md5sums[:stuck_detection_history] + prev_whole_image_md5sums[:stuck_detection_history] # Sleep for a while time.sleep(sleep_duration)@@ -113,12 +113,12 @@ def barrier_2(vm, words, fail_if_stuck_for, stuck_detection_history, logging.info(message)return False
else:
- # Collect information and put it inoutput_dir
+ # Collect information and put it indebug_dir
ifdata_scrdump_filenameandos.path.exists(data_scrdump_filename):
# Read expected screendump image
(ew, eh, edata) = \
ppm_utils.image_read_from_ppm_file(data_scrdump_filename)
- # Write it inoutput_dir
+ # Write it indebug_dirppm_utils.image_write_to_ppm_file(expected_scrdump_filename, ew, eh, edata)
# Write the cropped version as wellclient/tests/kvm/kvm_guest_wizard.py +1
-1@@ -131,7 +131,7 @@ def barrier_2(vm, words, fail_if_stuck_for, stuck_detection_history, ppm_utils.image_write_to_ppm_file(comparison_filename, w, h, data) # Print error messages and fail the test - long_message = message + "\n(see analysis at %s)" % output_dir + long_message = message + "\n(see analysis at %s)" % debug_dir logging.error(long_message)raise error.TestFail, message
--
1.5.4.1_______________________________________________
Autotest mailing list
[email protected]
http://test.kernel.org/cgi-bin/mailman/listinfo/autotest
Signature · 1 line
Lucas Meneghel
Lucas Meneghel Rodrigues
AppliedRe: [Autotest] [KVM-AUTOTEST PATCH 05/17] Remove kvm_spawn and run_bg() from kvm_utils.py.
In reply to Michael GoldishApplied.
Quoted 510 lines · Michael Goldish
These are now provided by
kvm_subprocess.py.Signed-off-by Michael Goldish <[email protected]>---
client/tests/kvm/kvm_utils.py | 477 +----------------------------------------1 files changed, 2 insertions(+), 475 deletions(-)client/tests/kvm/kvm_utils.py +2
-384@@ -227,390 +227,8 @@ def check_kvm_source_dir(source_dir): raise error.TestError("Unknown source dir layout, cannot proceed.") -# The following are a class and functions used for SSH, SCP and Telnet -# communication with guests. - -class kvm_spawn: - """ - This class is used for spawning and controlling a child process. - """ - - def __init__(self, command, linesep="\n"): - """ - Initialize the class and run command as a child process. - - @param command: Command that will be run. - @param linesep: Line separator for the given platform. - """ - self.exitstatus = None - self.linesep = linesep - (pid, fd) = pty.fork() - if pid == 0: - os.execv("/bin/sh", ["/bin/sh", "-c", command]) - else: - self.pid = pid - self.fd = fd - - - def set_linesep(self, linesep): - """ - Sets the line separator string (usually "\\n"). - - @param linesep: Line separator character. - """ - self.linesep = linesep - - - def is_responsive(self, timeout=5.0): - """ - Return True if the session is responsive. - - Send a newline to the child process (e.g. SSH or Telnet) and read some - output using read_nonblocking. - If all is OK, some output should be available (e.g. the shell prompt). - In that case return True. Otherwise return False. - - @param timeout: Timeout that will happen before we consider the - process unresponsive - """ - self.read_nonblocking(timeout=0.1) - self.sendline() - output = self.read_nonblocking(timeout=timeout) - if output.strip(): - return True - return False - - - def poll(self): - """ - If the process exited, return its exit status. Otherwise return None. - The exit status is stored for use in subsequent calls. - """ - if self.exitstatus != None: - return self.exitstatus - pid, status = os.waitpid(self.pid, os.WNOHANG) - if pid: - self.exitstatus = os.WEXITSTATUS(status) - return self.exitstatus - else: - return None - - - def close(self): - """ - Close the session (close the process filedescriptors and kills the - process ID), and return the exit status. - """ - try: - os.close(self.fd) - os.kill(self.pid, signal.SIGTERM) - except OSError: - pass - return self.poll() - - - def sendline(self, str=""): - """ - Sends a string followed by a line separator to the child process. - - @param str: String that will be sent to the child process. - """ - try: - os.write(self.fd, str + self.linesep) - except OSError: - pass - - - def read_nonblocking(self, timeout=1.0): - """ - Read from child until there is nothing to read for timeout seconds. - - @param timeout: Time (seconds) of wait before we give up reading from - the child process. - """ - data = "" - while True: - r, w, x = select.select([self.fd], [], [], timeout) - if self.fd in r: - try: - data += os.read(self.fd, 1024) - except OSError: - return data - else: - return data - - - def match_patterns(self, str, patterns): - """ - Match str against a list of patterns. - - Return the index of the first pattern that matches a substring of str. - None and empty strings in patterns are ignored. - If no match is found, return None. - - @param patterns: List of strings (regular expression patterns). - """ - for i in range(len(patterns)): - if not patterns[i]: - continue - exp = re.compile(patterns[i]) - if exp.search(str): - return i - - - def read_until_output_matches(self, patterns, filter=lambda(x):x, - timeout=30.0, internal_timeout=1.0, - print_func=None): - """ - Read using read_nonblocking until a match is found using match_patterns, - or until timeout expires. Before attempting to search for a match, the - data is filtered using the filter function provided. - - @brief: Read from child using read_nonblocking until a pattern - matches. - @param patterns: List of strings (regular expression patterns) - @param filter: Function to apply to the data read from the child before - attempting to match it against the patterns (should take and - return a string) - @param timeout: The duration (in seconds) to wait until a match is - found - @param internal_timeout: The timeout to pass to read_nonblocking - @param print_func: A function to be used to print the data being read - (should take a string parameter) - @return: Tuple containing the match index (or None if no match was - found) and the data read so far. - """ - match = None - data = "" - - end_time = time.time() + timeout - while time.time() < end_time: - # Read data from child - newdata = self.read_nonblocking(internal_timeout) - # Print it if necessary - if print_func and newdata: - map(print_func, newdata.splitlines()) - data += newdata - - done = False - # Look for patterns - match = self.match_patterns(filter(data), patterns) - if match != None: - done = True - # Check if child has died - if self.poll() != None: - logging.debug("Process terminated with status %d", self.poll()) - done = True - # Are we done? - if done: break - - # Print some debugging info - if match == None and self.poll() != 0: - logging.debug("Timeout elapsed or process terminated. Output: %s", - format_str_for_message(data.strip())) - - return (match, data) - - - def get_last_word(self, str): - """ - Return the last word in str. - - @param str: String that will be analyzed. - """ - if str: - return str.split()[-1] - else: - return "" - - - def get_last_line(self, str): - """ - Return the last non-empty line in str. - - @param str: String that will be analyzed. - """ - last_line = "" - for line in str.splitlines(): - if line != "": - last_line = line - return last_line - - - def read_until_last_word_matches(self, patterns, timeout=30.0, - internal_timeout=1.0, print_func=None): - """ - Read using read_nonblocking until the last word of the output matches - one of the patterns (using match_patterns), or until timeout expires. - - @param patterns: A list of strings (regular expression patterns) - @param timeout: The duration (in seconds) to wait until a match is - found - @param internal_timeout: The timeout to pass to read_nonblocking - @param print_func: A function to be used to print the data being read - (should take a string parameter) - @return: A tuple containing the match index (or None if no match was - found) and the data read so far. - """ - return self.read_until_output_matches(patterns, self.get_last_word, - timeout, internal_timeout, - print_func) - - - def read_until_last_line_matches(self, patterns, timeout=30.0, - internal_timeout=1.0, print_func=None): - """ - Read using read_nonblocking until the last non-empty line of the output - matches one of the patterns (using match_patterns), or until timeout - expires. Return a tuple containing the match index (or None if no match - was found) and the data read so far. - - @brief: Read using read_nonblocking until the last non-empty line - matches a pattern. - - @param patterns: A list of strings (regular expression patterns) - @param timeout: The duration (in seconds) to wait until a match is - found - @param internal_timeout: The timeout to pass to read_nonblocking - @param print_func: A function to be used to print the data being read - (should take a string parameter) - """ - return self.read_until_output_matches(patterns, self.get_last_line, - timeout, internal_timeout, - print_func) - - - def set_prompt(self, prompt): - """ - Set the prompt attribute for later use by read_up_to_prompt. - - @param: String that describes the prompt contents. - """ - self.prompt = prompt - - - def read_up_to_prompt(self, timeout=30.0, internal_timeout=1.0, - print_func=None): - """ - Read using read_nonblocking until the last non-empty line of the output - matches the prompt regular expression set by set_prompt, or until - timeout expires. - - @brief: Read using read_nonblocking until the last non-empty line - matches the prompt. - - @param timeout: The duration (in seconds) to wait until a match is - found - @param internal_timeout: The timeout to pass to read_nonblocking - @param print_func: A function to be used to print the data being - read (should take a string parameter) - - @return: A tuple containing True/False indicating whether the prompt - was found, and the data read so far. - """ - (match, output) = self.read_until_last_line_matches([self.prompt], - timeout, - internal_timeout, - print_func) - if match == None: - return (False, output) - else: - return (True, output) - - - def set_status_test_command(self, status_test_command): - """ - Set the command to be sent in order to get the last exit status. - - @param status_test_command: Command that will be sent to get the last - exit status. - """ - self.status_test_command = status_test_command - - - def get_command_status_output(self, command, timeout=30.0, - internal_timeout=1.0, print_func=None): - """ - Send a command and return its exit status and output. - - @param command: Command to send - @param timeout: The duration (in seconds) to wait until a match is - found - @param internal_timeout: The timeout to pass to read_nonblocking - @param print_func: A function to be used to print the data being read - (should take a string parameter) - - @return: A tuple (status, output) where status is the exit status or - None if no exit status is available (e.g. timeout elapsed), and - output is the output of command. - """ - # Print some debugging info - logging.debug("Sending command: %s" % command) - - # Read everything that's waiting to be read - self.read_nonblocking(0.1) - - # Send the command and get its output - self.sendline(command) - (match, output) = self.read_up_to_prompt(timeout, internal_timeout, - print_func) - if not match: - return (None, "\n".join(output.splitlines()[1:])) - output = "\n".join(output.splitlines()[1:-1]) - - # Send the 'echo ...' command to get the last exit status - self.sendline(self.status_test_command) - (match, status) = self.read_up_to_prompt(10.0, internal_timeout) - if not match: - return (None, output) - status = int("\n".join(status.splitlines()[1:-1]).strip()) - - # Print some debugging info - if status != 0: - logging.debug("Command failed; status: %d, output:%s", status, - format_str_for_message(output.strip())) - - return (status, output) - - - def get_command_status(self, command, timeout=30.0, internal_timeout=1.0, - print_func=None): - """ - Send a command and return its exit status. - - @param command: Command to send - @param timeout: The duration (in seconds) to wait until a match is - found - @param internal_timeout: The timeout to pass to read_nonblocking - @param print_func: A function to be used to print the data being read - (should take a string parameter) - - @return: Exit status or None if no exit status is available (e.g. - timeout elapsed). - """ - (status, output) = self.get_command_status_output(command, timeout, - internal_timeout, - print_func) - return status - - - def get_command_output(self, command, timeout=30.0, internal_timeout=1.0, - print_func=None): - """ - Send a command and return its output. - - @param command: Command to send - @param timeout: The duration (in seconds) to wait until a match is - found - @param internal_timeout: The timeout to pass to read_nonblocking - @param print_func: A function to be used to print the data being read - (should take a string parameter) - """ - (status, output) = self.get_command_status_output(command, timeout, - internal_timeout, - print_func) - return output - +# The following are functions used for SSH, SCP and Telnet communication with +# guests.def
remote_login(command, password, prompt, linesep="\n", timeout=10):
"""client/tests/kvm/kvm_utils.py +0
-91@@ -810,97 +428,6 @@ def telnet(host, port, username, password, prompt, timeout=10): return remote_login(command, password, prompt, "\r\n", timeout) -# The following are functions used for running commands in the background. - -def track_process(sub, status_output=None, term_func=None, stdout_func=None, - prefix=""): - """ - Read lines from the stdout pipe of the subprocess. Pass each line to - stdout_func prefixed by prefix. Place the lines in status_output[1]. - When the subprocess exits, call term_func. Place the exit status in - status_output[0]. - - @brief Track a subprocess and report its output and termination. - - @param sub: An object returned by subprocess.Popen - @param status_output: A list in which the exit status and output are to be - stored. - @param term_func: A function to call when the process terminates - (should take no parameters) - @param stdout_func: A function to call with each line of output from the - subprocess (should take a string parameter) - - @param prefix -- a string to pre-pend to each line of the output, before - passing it to stdout_func - """ - while True: - # Read a single line from stdout - text = sub.stdout.readline() - # If the subprocess exited... - if text == "": - # Get exit code - status = sub.wait() - # Report it - if status_output: - status_output[0] = status - # Call term_func - if term_func: - term_func() - return - # Report the text - if status_output: - status_output[1] += text - # Call stdout_func with the returned text - if stdout_func: - text = prefix + text.strip() - # We need to sanitize the text before passing it to the logging - # system - text = text.decode('utf-8', 'replace') - stdout_func(text) - - -def run_bg(command, term_func=None, stdout_func=None, prefix="", timeout=1.0): - """ - Run command as a subprocess. Call stdout_func with each line of output from - the subprocess (prefixed by prefix). Call term_func when the subprocess - terminates. If timeout expires and the subprocess is still running, return. - - @brief: Run a subprocess in the background and collect its output and - exit status. - - @param command: The shell command to execute - @param term_func: A function to call when the process terminates - (should take no parameters) - @param stdout_func: A function to call with each line of output from - the subprocess (should take a string parameter) - @param prefix: A string to pre-pend to each line of the output, before - passing it to stdout_func - @param timeout: Time duration (in seconds) to wait for the subprocess to - terminate before returning - - @return: A 3-tuple containing the exit status (None if the subprocess is - still running), the PID of the subprocess (None if the subprocess - terminated), and the output collected so far. - """ - # Start the process - sub = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, - stderr=subprocess.STDOUT) - # Start the tracking thread - status_output = [None, ""] - thread.start_new_thread(track_process, (sub, status_output, term_func, - stdout_func, prefix)) - # Wait up to timeout secs for the process to exit - end_time = time.time() + timeout - while time.time() < end_time: - # If the process exited, return - if status_output[0] != None: - return (status_output[0], None, status_output[1]) - # Otherwise, sleep for a while - time.sleep(0.1) - # Report the PID and the output collected so far - return (None, sub.pid, status_output[1]) - - # The following are utility functions related to ports.def
is_sshd_running(host, port, timeout=10.0):
--
1.5.4.1_______________________________________________
Autotest mailing list
[email protected]
http://test.kernel.org/cgi-bin/mailman/listinfo/autotest
Signature · 1 line
Lucas Meneghel
Lucas Meneghel Rodrigues
AppliedRe: [Autotest] [KVM-AUTOTEST PATCH 04/17] Modify run_autotest() in kvm_tests.py to use the new kvm_subprocess module.
In reply to Michael GoldishApplied.
Quoted 45 lines · Michael Goldish
Signed-off-by Michael Goldish <[email protected]>---
client/tests/kvm/kvm_tests.py | 9 +++++----1 files changed, 5 insertions(+), 4 deletions(-)client/tests/kvm/kvm_tests.py +0
-0@@ -1,6 +1,6 @@import time, os, logging
from autotest_lib.client.common_lib import utils, errorclient/tests/kvm/kvm_tests.py +1
-1-import kvm_utils, ppm_utils, scan_results +import kvm_utils, kvm_subprocess, ppm_utils, scan_results"""
KVM test definitions.client/tests/kvm/kvm_tests.py +0
-0@@ -274,15 +274,16 @@ def run_autotest(test, params, env):cmd += " --exclude=*.pyc"
cmd += " --exclude=*.svn"
cmd += " --exclude=*.git"
- kvm_utils.run_bg(cmd % (test.bindir,tarred_autotest_path), timeout=30)
+ kvm_subprocess.run_fg(cmd % (test.bindir,tarred_autotest_path), timeout=30)# tar the contents of bindir/autotest/tests/<test_name> cmd = "cd %s; tar cvjf %s %s/*" cmd += " --exclude=*.pyc" cmd += " --exclude=*.svn" cmd += " --exclude=*.git" - kvm_utils.run_bg(cmd % (os.path.join(test.bindir, "autotest", "tests"), - tarred_test_path, test_name), timeout=30) + kvm_subprocess.run_fg(cmd % + (os.path.join(test.bindir, "autotest", "tests"), + tarred_test_path, test_name), timeout=30)# Check if we need to copy autotest.tar.bz2
copy = False
--
1.5.4.1_______________________________________________
Autotest mailing list
[email protected]
http://test.kernel.org/cgi-bin/mailman/listinfo/autotest
Signature · 1 line
Lucas Meneghel
Re: [Autotest] [KVM-AUTOTEST PATCH 03/17] Modify remote_login and remote_scp in kvm_utils to use kvm_subprocess
In reply to Michael GoldishQuoted 61 lines · Michael Goldish
Also remove a reference to
kvm_logthat was left behind.Signed-off-by Michael Goldish <[email protected]>---
client/tests/kvm/kvm_utils.py | 13 ++++++++-----1 files changed, 8 insertions(+), 5 deletions(-)client/tests/kvm/kvm_utils.py +0
-0@@ -2,6 +2,7 @@ import md5, thread, subprocess, time, string, random, socket, os, signal, ptyimport select, re, logging
from autotest_lib.client.bin import utils
from autotest_lib.client.common_lib import error
+importkvm_subprocess"""
KVM test utility functions.client/tests/kvm/kvm_utils.py +0
-0@@ -631,8 +632,9 @@ def remote_login(command, password, prompt, linesep="\n", timeout=10):@return Return the
kvm_spawnobject on success and None on failure. """
- sub =kvm_spawn(command, linesep)
- sub.set_prompt(prompt)
+ sub = kvm_subprocess.kvm_shell_session(command,
+ linesep=linesep,
+ prompt=prompt)
password_prompt_count= 0client/tests/kvm/kvm_utils.py +0
-0@@ -698,7 +700,7 @@ def remote_scp(command, password, timeout=300, login_timeout=10):@return: True if the transfer succeeds and False on failure. """
- sub =kvm_spawn(command)
+ sub =kvm_subprocess.kvm_expect(command)
password_prompt_count= 0
_timeout=login_timeoutclient/tests/kvm/kvm_utils.py +0
-0@@ -729,9 +731,10 @@ def remote_scp(command, password, timeout=300, login_timeout=10): sub.close()return False
else: # match == None
- logging.debug("Timeout or process terminated")
+ logging.debug("Timeout elapsed or process terminated")
+ status =sub.get_status()
sub.close()
- returnsub.poll()== 0
+ return status == 0def scp_to_remote(host, port, username, password,
local_path,remote_path, --
1.5.4.1_______________________________________________
Autotest mailing list
[email protected]
http://test.kernel.org/cgi-bin/mailman/listinfo/autotest
Signature · 1 line
Lucas Meneghel
Lucas Meneghel Rodrigues
AppliedRe: [Autotest] [KVM-AUTOTEST PATCH 03/17] Modify remote_login and remote_scp in kvm_utils to use kvm_subprocess
Applied.
Quoted 68 lines · Lucas Meneghel Rodrigues
Quoted 52 lines · Michael Goldish
Also remove a reference to
kvm_logthat was left behind.Signed-off-by Michael Goldish <[email protected]>---
client/tests/kvm/kvm_utils.py | 13 ++++++++-----1 files changed, 8 insertions(+), 5 deletions(-)client/tests/kvm/kvm_utils.py +0
-0@@ -2,6 +2,7 @@ import md5, thread, subprocess, time, string, random, socket, os, signal, ptyimport select, re, logging
from autotest_lib.client.bin import utils
from autotest_lib.client.common_lib import error
+importkvm_subprocess"""
KVM test utility functions.client/tests/kvm/kvm_utils.py +0
-0@@ -631,8 +632,9 @@ def remote_login(command, password, prompt, linesep="\n", timeout=10):@return Return the
kvm_spawnobject on success and None on failure. """
- sub =kvm_spawn(command, linesep)
- sub.set_prompt(prompt)
+ sub = kvm_subprocess.kvm_shell_session(command,
+ linesep=linesep,
+ prompt=prompt)
password_prompt_count= 0client/tests/kvm/kvm_utils.py +0
-0@@ -698,7 +700,7 @@ def remote_scp(command, password, timeout=300, login_timeout=10):@return: True if the transfer succeeds and False on failure. """
- sub =kvm_spawn(command)
+ sub =kvm_subprocess.kvm_expect(command)
password_prompt_count= 0
_timeout=login_timeoutclient/tests/kvm/kvm_utils.py +0
-0@@ -729,9 +731,10 @@ def remote_scp(command, password, timeout=300, login_timeout=10): sub.close()return False
else: # match == None
- logging.debug("Timeout or process terminated")
+ logging.debug("Timeout elapsed or process terminated")
+ status =sub.get_status()
sub.close()
- returnsub.poll()== 0
+ return status == 0Quoted 9 lines
def scp_to_remote(host, port, username, password,
local_path,remote_path, --
1.5.4.1_______________________________________________
Autotest mailing list
[email protected]
http://test.kernel.org/cgi-bin/mailman/listinfo/autotest--
Lucas Meneghel
Signature · 1 line
Lucas Meneghel
Re: [Autotest] [KVM-AUTOTEST PATCH 02/17] Modify kvm_vm and kvm_preprocessing to use the new kvm_subprocess module
In reply to Michael GoldishLooks good to me. Applied.
Quoted 295 lines · Michael Goldish
Signed-off-by Michael Goldish <[email protected]>---
client/tests/kvm/kvm_preprocessing.py | 27 ++------client/tests/kvm/kvm_vm.py| 111 +++++++++++++++-----------------2 files changed, 59 insertions(+), 79 deletions(-)client/tests/kvm/kvm_preprocessing.py +0
-0@@ -1,7 +1,7 @@import sys, os, time, commands, re, logging, signal from autotest_lib.client.bin import test from autotest_lib.client.common_lib import error
client/tests/kvm/kvm_preprocessing.py +1
-1-import kvm_vm, kvm_utils +import kvm_vm, kvm_utils, kvm_subprocessdef
preprocess_image(test, params):client/tests/kvm/kvm_preprocessing.py +0
-0@@ -75,7 +75,7 @@ def preprocess_vm(test, params, env, name): qemu_path, image_dir,
iso_dir):
- logging.debug("VM's qemu command differs from requested one;"
+ logging.debug("VM's qemu command differs from requested one; " "restarting it...")
start_vm= Trueclient/tests/kvm/kvm_preprocessing.py +3
-4@@ -158,13 +158,11 @@ def process_command(test, params, env, command, command_timeout, # execute command logging.info("Executing command '%s'..." % command) timeout = int(command_timeout) - (status, pid, output) = kvm_utils.run_bg("cd %s; %s" % (test.bindir, + (status, output) = kvm_subprocess.run_fg("cd %s; %s" % (test.bindir, command), - None, logging.debug, - "(command) ", - timeout = timeout) + logging.debug, "(command) ", + timeout=timeout)if status != 0:
- kvm_utils.safe_kill(pid, signal.SIGTERM)logging.warn("Custom processing command failed: '%s'..." % command)ifcommand_noncritical!= "yes":
raiseerror.TestError("Custom processing command failed")client/tests/kvm/kvm_preprocessing.py +0
-0@@ -204,17 +202,6 @@ def preprocess(test, params, env):@param params: A dict containing all VM and image parameters. @param env: The environment (a dict-like object).
"""
- # Verify the identities of all living VMs
- for vm inenv.values():
- if notkvm_utils.is_vm(vm):
- continue
- ifvm.is_dead():
- continue
- if notvm.verify_process_identity():
- logging.debug("VM '%s' seems to have been replaced by another"
- " process" % vm.name)
- vm.pid = None
-
# Destroy and remove VMs that are no longer needed in the environmentrequested_vms=kvm_utils.get_sub_dict_names(params, "vms")for key inenv.keys():client/tests/kvm/kvm_preprocessing.py +0
-0@@ -282,8 +269,8 @@ def postprocess(test, params, env): # Remove them all
logging.debug("'keep_ppm_files' not specified; removing all PPM files" " from results dir...")
- kvm_utils.run_bg("rm -vf %s" %os.path.join(test.debugdir, "*.ppm"),
- None, logging.debug, "(rm) ", timeout=5.0)
+ rm_cmd = "rm -vf %s" %os.path.join(test.debugdir, "*.ppm")
+ kvm_subprocess.run_fg(rm_cmd, logging.debug, "(rm) ", timeout=5.0)#execute any
post_commands
ifparams.get("post_command"):client/tests/kvm/kvm_vm.py +0
-0@@ -1,6 +1,6 @@ #!/usr/bin/pythonimport time, socket, os, logging, fcntl
client/tests/kvm/kvm_vm.py +1
-1-import kvm_utils +import kvm_utils, kvm_subprocess"""
Utility classes and functions to handle Virtual Machine creation using qemu.client/tests/kvm/kvm_vm.py +0
-0@@ -54,17 +54,22 @@ def create_image(params, qemu_img_path, image_dir):
qemu_img_cmd+= " %s" % size
logging.debug("Running qemu-img command:\n%s" % qemu_img_cmd)
- (status, pid, output) = kvm_utils.run_bg(qemu_img_cmd, None,
- logging.debug, "(qemu-img) ",
- timeout=30)
+ (status, output) = kvm_subprocess.run_fg(qemu_img_cmd, logging.debug,
+ "(qemu-img) ", timeout=30)- if status:
- logging.debug("qemu-img exited with status %d" % status)
- logging.error("Could not create image %s" %image_filename)
+ if status is None:
+ logging.error("Timeout elapsed while waiting for qemu-img command "
+ "to complete:\n%s" %qemu_img_cmd)
+ return None
+ elif status != 0:
+ logging.error("Could not create image; "
+ "qemu-img command failed:\n%s" %qemu_img_cmd)
+ logging.error("Status: %s" % status)
+ logging.error("Output:" +kvm_utils.format_str_for_message(output)) return None
if notos.path.exists(image_filename):
- logging.debug("Image file does not exist for some reason")
- logging.error("Could not create image %s" %image_filename)
+ logging.error("Image could not be created for some reason; "
+ "qemu-img command:\n%s" %qemu_img_cmd) return None
logging.info("Image created in %s" % image_filename)client/tests/kvm/kvm_vm.py +0
-0@@ -106,7 +111,7 @@ class VM:@param
image_dir: The directory where images reside @paramiso_dir: The directory where ISOs reside """
- self.pid = None
+ self.process = None
self.uuid = Noneself.name = name
client/tests/kvm/kvm_vm.py +0
-22@@ -154,28 +159,6 @@ class VM: return VM(name, params, qemu_path, image_dir, iso_dir) - def verify_process_identity(self): - """ - Make sure .pid really points to the original qemu process. If .pid - points to the same process that was created with the create method, - or to a dead process, return True. Otherwise return False. - """ - if self.is_dead(): - return True - filename = "/proc/%d/cmdline" % self.pid - if not os.path.exists(filename): - logging.debug("Filename %s does not exist" % filename) - return False - file = open(filename) - cmdline = file.read() - file.close() - if not self.qemu_path in cmdline: - return False - if not self.monitor_file_name in cmdline: - return False - return True - - def make_qemu_command(self, name=None, params=None, qemu_path=None,image_dir=None, iso_dir=None):
"""client/tests/kvm/kvm_vm.py +0
-0@@ -394,25 +377,26 @@ class VM:
qemu_command+= " -incoming tcp:0:%d" % self.migration_port
logging.debug("Running qemu command:\n%s", qemu_command)
- (status, pid, output) = kvm_utils.run_bg(qemu_command, None,
- logging.debug, "(qemu) ")client/tests/kvm/kvm_vm.py +10
-5- - if status: - logging.debug("qemu exited with status %d", status) - logging.error("VM could not be created -- qemu command" - " failed:\n%s", qemu_command) + self.process = kvm_subprocess.run_bg(qemu_command, None, + logging.debug, "(qemu) ") + + if not self.process.is_alive(): + logging.error("VM could not be created; " + "qemu command failed:\n%s" % qemu_command) + logging.error("Status: %s" % self.process.get_status()) + logging.error("Output:" + kvm_utils.format_str_for_message( + self.process.get_output())) + self.destroy()return False
- self.pid = pid
-
if notkvm_utils.wait_for(self.is_alive, timeout, 0, 1):
- logging.debug("VM is not alive for some reason")
- logging.error("VM could not be created with"
- " command:\n%s",qemu_command)
+ logging.error("VM is not alive for some reason; "
+ "qemu command:\n%s" %qemu_command)self.destroy()
return False- logging.debug("VM appears to be alive with PID %d", self.pid)
+ logging.debug("VM appears to be alive with PID %d",
+ self.process.get_pid())
return Truefinally:
client/tests/kvm/kvm_vm.py +0
-0@@ -511,9 +495,11 @@ class VM: # Is it already dead?if
self.is_dead():
logging.debug("VM is already down")
+ if self.process:
+ self.process.close()
return- logging.debug("Destroying VM with PID %d..." % self.pid)
+ logging.debug("Destroying VM with PID %d..." %self.process.get_pid())if gracefully and
self.params.get("cmd_shutdown"):
# Try to destroy with SSH commandclient/tests/kvm/kvm_vm.py +0
-0@@ -521,12 +507,11 @@ class VM: (status, output) = self.ssh(self.params.get("cmd_shutdown")) # Was the command sent successfully?if status == 0:
- #if self.ssh(self.params.get("cmd_shutdown")):
- logging.debug("Shutdown command sent; Waiting for VM to go"
+ logging.debug("Shutdown command sent; waiting for VM to go " "down...")
ifkvm_utils.wait_for(self.is_dead, 60, 1, 1):
logging.debug("VM is down")
- self.pid = None
+ self.process.close()
return# Try to destroy with a monitor command
client/tests/kvm/kvm_vm.py +0
-0@@ -537,28 +522,29 @@ class VM: # Wait for the VM to be really deadif
kvm_utils.wait_for(self.is_dead, 5, 0.5, 0.5):
logging.debug("VM is down")
- self.pid = None
+ self.process.close()
return# If the VM isn't dead yet...
- logging.debug("Cannot quit normally; Sending a kill to close the"
- " deal...")
- kvm_utils.safe_kill(self.pid, 9)
+ logging.debug("Cannot quit normally; sending a kill to close the "
+ "deal...")
+ kvm_utils.safe_kill(self.process.get_pid(), 9)
# Wait for the VM to be really dead
ifkvm_utils.wait_for(self.is_dead, 5, 0.5, 0.5):
logging.debug("VM is down")
- self.pid = None
+ self.process.close()
return- logging.error("We have a zombie! PID %d is a zombie!" % self.pid)
+ logging.error("Process %s is a zombie!" %self.process.get_pid())
+ self.process.close()def
is_alive(self):
"""
Return True if the VM's monitor is responsive. """
- # Check if the process exists
- if notkvm_utils.pid_exists(self.pid):
+ # Check if the process is running
+ ifself.is_dead():
return False
# Try sending a monitor command (status, output) =self.send_monitor_cmd("help")client/tests/kvm/kvm_vm.py +0
-0@@ -569,9 +555,9 @@ class VM:def
is_dead(self):
"""
- Return True iff the VM's PID does not exist.
+ Return True if the qemu process is dead.
"""
- return notkvm_utils.pid_exists(self.pid)
+ return not self.process or notself.process.is_alive()def
get_params(self):client/tests/kvm/kvm_vm.py +0
-0@@ -610,6 +596,13 @@ class VM:return None
+ def
get_pid(self):
+ """
+ Return the VM's PID.
+ """
+ returnself.process.get_pid()client/tests/kvm/kvm_vm.py +2
-0+ +def
is_sshd_running(self, timeout=10):
"""
Return True iff the guest's SSH port is responsive. --
1.5.4.1_______________________________________________
Autotest mailing list
[email protected]
http://test.kernel.org/cgi-bin/mailman/listinfo/autotest
Signature · 1 line
Lucas Meneghel
Re: [Autotest] [KVM-AUTOTEST PATCH 01/17] Add new module kvm_subprocess
In reply to Michael GoldishOk, I have made minor remarks for the first version of this module
http://codereview.appspot.com/79042/diff/1/4
and Michael either commented or addressed the questions. I am going to commit this new module.
Thanks for your work, Michael!
Quoted 1,184 lines · Michael Goldish
This module is intended to be used for controlling all child processes in KVM
tests: both QEMU processes and SSH/SCP/Telnet processes. Processes started with this module keep running and can be interacted with even after the parent process exits.The current
run_bg()utility tracks a child process as long as the parent process is running. When the parent process exits, the tracking thread terminates and cannot resume when needed.Currently SSH/SCP/Telnet communication is handled by kvm_utils.kvm_spawn, which does not allow the child process to run after the parent process exits. Thus, open SSH/SCP/Telnet sessions cannot be reused by tests following the one in which they are opened.
The new module provides a solution to these two problems, and also saves some code by reusing common code required both for QEMU processes and SSH/SCP/Telnet processes.
Signed-off-by Michael Goldish <[email protected]>---
client/tests/kvm/kvm_subprocess.py | 1146 ++++++++++++++++++++++++++++++++++++1 files changed, 1146 insertions(+), 0 deletions(-)create mode 100644
client/tests/kvm/kvm_subprocess.pyclient/tests/kvm/kvm_subprocess.py new file +1146
-1@@ -0,0 +1,1146 @@ +#!/usr/bin/python +import sys, subprocess, pty, select, os, time, signal, re, termios, fcntl +import threading, logging, commands +import common, kvm_utils + +""" +A class and functions used for running and controlling child processes. + +@copyright: 2008-2009 Red Hat Inc. +""" + + +def run_bg(command, termination_func=None, output_func=None, output_prefix="", + timeout=1.0): + """ + Run command as a subprocess. Call output_func with each line of output + from the subprocess (prefixed by output_prefix). Call termination_func + when the subprocess terminates. Return when timeout expires or when the + subprocess exits -- whichever occurs first. + + @brief: Run a subprocess in the background and collect its output and + exit status. + + @param command: The shell command to execute + @param termination_func: A function to call when the process terminates + (should take an integer exit status parameter) + @param output_func: A function to call with each line of output from + the subprocess (should take a string parameter) + @param output_prefix: A string to pre-pend to each line of the output, + before passing it to stdout_func + @param timeout: Time duration (in seconds) to wait for the subprocess to + terminate before returning + + @return: A kvm_tail object. + """ + process = kvm_tail(command=command, + termination_func=termination_func, + output_func=output_func, + output_prefix=output_prefix) + + end_time = time.time() + timeout + while time.time() < end_time and process.is_alive(): + time.sleep(0.1) + + return process + + +def run_fg(command, output_func=None, output_prefix="", timeout=1.0): + """ + Run command as a subprocess. Call output_func with each line of output + from the subprocess (prefixed by prefix). Return when timeout expires or + when the subprocess exits -- whichever occurs first. If timeout expires + and the subprocess is still running, kill it before returning. + + @brief: Run a subprocess in the foreground and collect its output and + exit status. + + @param command: The shell command to execute + @param output_func: A function to call with each line of output from + the subprocess (should take a string parameter) + @param output_prefix: A string to pre-pend to each line of the output, + before passing it to stdout_func + @param timeout: Time duration (in seconds) to wait for the subprocess to + terminate before killing it and returning + + @return: A 2-tuple containing the exit status of the process and its + STDOUT/STDERR output. If timeout expires before the process + terminates, the returned status is None. + """ + process = run_bg(command, None, output_func, output_prefix, timeout) + output = process.get_output() + if process.is_alive(): + status = None + else: + status = process.get_status() + process.close() + return (status, output) + + +def _lock(filename): + if not os.path.exists(filename): + open(filename, "w").close() + fd = os.open(filename, os.O_RDWR) + fcntl.lockf(fd, fcntl.LOCK_EX) + return fd + + +def _unlock(fd): + fcntl.lockf(fd, fcntl.LOCK_UN) + os.close(fd) + + +def _locked(filename): + try: + fd = os.open(filename, os.O_RDWR) + except: + return False + try: + fcntl.lockf(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + except: + os.close(fd) + return True + fcntl.lockf(fd, fcntl.LOCK_UN) + os.close(fd) + return False + + +def _wait(filename): + fd = _lock(filename) + _unlock(fd) + + +def _get_filenames(base_dir, id): + return [os.path.join(base_dir, s + id) for s in + "shell-pid-", "status-", "output-", "inpipe-", + "lock-server-running-", "lock-client-starting-"] + + +def _get_reader_filename(base_dir, id, reader): + return os.path.join(base_dir, "outpipe-%s-%s" % (reader, id)) + + +class kvm_spawn: + """ + This class is used for spawning and controlling a child process. + + A new instance of this class can either run a new server (a small Python + program that reads output from the child process and reports it to the + client and to a text file) or attach to an already running server. + When a server is started it runs the child process. + The server writes output from the child's STDOUT and STDERR to a text file. + The text file can be accessed at any time using get_output(). + In addition, the server opens as many pipes as requested by the client and + writes the output to them. + The pipes are requested and accessed by classes derived from kvm_spawn. + These pipes are referred to as "readers". + The server also receives input from the client and sends it to the child + process. + An instance of this class can be pickled. Every derived class is + responsible for restoring its own state by properly defining + __getinitargs__(). + + The first named pipe is used by _tail(), a function that runs in the + background and reports new output from the child as it is produced. + The second named pipe is used by a set of functions that read and parse + output as requested by the user in an interactive manner, similar to + pexpect. + When unpickled it automatically + resumes _tail() if needed. + """ + + def __init__(self, command=None, id=None, echo=False, linesep="\n"): + """ + Initialize the class and run command as a child process. + + @param command: Command to run, or None if accessing an already running + server. + @param id: ID of an already running server, if accessing a running + server, or None if starting a new one. + @param echo: Boolean indicating whether echo should be initially + enabled for the pseudo terminal running the subprocess. This + parameter has an effect only when starting a new server. + @param linesep: Line separator to be appended to strings sent to the + child process by sendline(). + """ + self.id = id or kvm_utils.generate_random_string(8) + + # Define filenames for communication with server + base_dir = "/tmp/kvm_spawn" + try: + os.makedirs(base_dir) + except: + pass + (self.shell_pid_filename, + self.status_filename, + self.output_filename, + self.inpipe_filename, + self.lock_server_running_filename, + self.lock_client_starting_filename) = _get_filenames(base_dir, + self.id) + + # Remember some attributes + self.echo = echo + self.linesep = linesep + + # Make sure the 'readers' and 'close_hooks' attributes exist + if not hasattr(self, "readers"): + self.readers = [] + if not hasattr(self, "close_hooks"): + self.close_hooks = [] + + # Define the reader filenames + self.reader_filenames = dict( + (reader, _get_reader_filename(base_dir, self.id, reader)) + for reader in self.readers) + + # Let the server know a client intends to open some pipes; + # if the executed command terminates quickly, the server will wait for + # the client to release the lock before exiting + lock_client_starting = _lock(self.lock_client_starting_filename) + + # Start the server (which runs the command) + if command: + sub = subprocess.Popen("python %s" % __file__, + shell=True, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + # Send parameters to the server + sub.stdin.write("%s\n" % self.id) + sub.stdin.write("%s\n" % echo) + sub.stdin.write("%s\n" % ",".join(self.readers)) + sub.stdin.write("%s\n" % command) + # Wait for the server to complete its initialization + sub.stdout.readline() + + # Open the reading pipes + self.reader_fds = {} + try: + assert(_locked(self.lock_server_running_filename)) + for reader, filename in self.reader_filenames.items(): + self.reader_fds[reader] = os.open(filename, os.O_RDONLY) + except: + pass + + # Allow the server to continue + _unlock(lock_client_starting) + + + # The following two functions are defined to make sure the state is set + # exclusively by the constructor call as specified in __getinitargs__(). + + def __getstate__(self): + pass + + + def __setstate__(self, state): + pass + + + def __getinitargs__(self): + # Save some information when pickling -- will be passed to the + # constructor upon unpickling + return (None, self.id, self.echo, self.linesep) + + + def _add_reader(self, reader): + """ + Add a reader whose file descriptor can be obtained with _get_fd(). + Should be called before __init__(). Intended for use by derived + classes. + + @param reader: The name of the reader. + """ + if not hasattr(self, "readers"): + self.readers = [] + self.readers.append(reader) + + + def _add_close_hook(self, hook): + """ + Add a close hook function to be called when close() is called. + The function will be called after the process terminates but before + final cleanup. Intended for use by derived classes. + + @param hook: The hook function. + """ + if not hasattr(self, "close_hooks"): + self.close_hooks = [] + self.close_hooks.append(hook) + + + def _get_fd(self, reader): + """ + Return an open file descriptor corresponding to the specified reader + pipe. If no such reader exists, or the pipe could not be opened, + return None. Intended for use by derived classes. + + @param reader: The name of the reader. + """ + return self.reader_fds.get(reader) + + + def get_id(self): + """ + Return the instance's id attribute, which may be used to access the + process in the future. + """ + return self.id + + + def get_shell_pid(self): + """ + Return the PID of the subshell process, or None if not available. + The subshell is the shell that runs the command. + """ + try: + file = open(self.shell_pid_filename, "r") + pid = int(file.read()) + file.close() + return pid + except: + return None + + + def get_pid(self, index=0): + """ + Try to get and return the PID of a child process of the subshell. + This is usually the PID of the process executed in the subshell. + There are 3 exceptions: + - If the subshell couldn't start the process for some reason, no + PID can be returned. + - If the subshell is running several processes in parallel, + multiple PIDs can be returned. Use the index parameter in this + case. + - Before starting the process, after the process has terminated, + or while running shell code that doesn't start any processes -- + no PID can be returned. + + @param index: The index of the child process whose PID is requested. + Normally this should remain 0. + @return: The PID of the child process, or None if none could be found. + """ + parent_pid = self.get_shell_pid() + if not parent_pid: + return None + pids = commands.getoutput("ps --ppid %d -o pid=" % parent_pid).split() + try: + return int(pids[index]) + except: + return None + + + def get_status(self): + """ + Wait for the process to exit and return its exit status, or None + if the exit status is not available. + """ + _wait(self.lock_server_running_filename) + try: + file = open(self.status_filename, "r") + status = int(file.read()) + file.close() + return status + except: + return None + + + def get_output(self): + """ + Return the STDOUT and STDERR output of the process so far. + """ + try: + file = open(self.output_filename, "r") + output = file.read() + file.close() + return output + except: + return "" + + + def is_alive(self): + """ + Return True if the process is running. + """ + pid = self.get_shell_pid() + # See if the PID exists + try: + os.kill(pid, 0) + except: + return False + # Make sure the PID belongs to the original process + filename = "/proc/%d/cmdline" % pid + try: + file = open(filename, "r") + cmdline = file.read() + file.close() + except: + # If we couldn't find the file for some reason, skip the check + return True + if self.id in cmdline: + return True + return False + + + def close(self, sig=signal.SIGTERM): + """ + Kill the child process if it's alive and remove temporary files. + + @param sig: The signal to send the process when attempting to kill it. + """ + # Kill it if it's alive + if self.is_alive(): + try: + os.kill(self.get_shell_pid(), sig) + except: + pass + # Wait for the server to exit + _wait(self.lock_server_running_filename) + # Call all cleanup routines + for hook in self.close_hooks: + hook() + # Close reader file descriptors + for fd in self.reader_fds.values(): + try: + os.close(fd) + except: + pass + # Remove all used files + for filename in (_get_filenames("/tmp/kvm_spawn", self.id) + + self.reader_filenames.values()): + try: + os.unlink(filename) + except OSError: + pass + + + def set_linesep(self, linesep): + """ + Sets the line separator string (usually "\\n"). + + @param linesep: Line separator string. + """ + self.linesep = linesep + + + def send(self, str=""): + """ + Send a string to the child process. + + @param str: String to send to the child process. + """ + try: + fd = os.open(self.inpipe_filename, os.O_RDWR) + os.write(fd, str) + os.close(fd) + except: + pass + + + def sendline(self, str=""): + """ + Send a string followed by a line separator to the child process. + + @param str: String to send to the child process. + """ + self.send(str + self.linesep) + + +class kvm_tail(kvm_spawn): + """ + This class runs a child process in the background and sends its output in + real time, line-by-line, to a callback function. + + See kvm_spawn's docstring. + + This class uses a single pipe reader to read data in real time from the + child process and report it to a given callback function. + When the child process exits, its exit status is reported to an additional + callback function. + + When this class is unpickled, it automatically resumes reporting output. + """ + + def __init__(self, command=None, id=None, echo=False, linesep="\n", + termination_func=None, output_func=None, output_prefix=""): + """ + Initialize the class and run command as a child process. + + @param command: Command to run, or None if accessing an already running + server. + @param id: ID of an already running server, if accessing a running + server, or None if starting a new one. + @param echo: Boolean indicating whether echo should be initially + enabled for the pseudo terminal running the subprocess. This + parameter has an effect only when starting a new server. + @param linesep: Line separator to be appended to strings sent to the + child process by sendline(). + @param termination_func: Function to call when the process exits. The + function must accept a single exit status parameter. + @param output_func: Function to call whenever a line of output is + available from the STDOUT or STDERR streams of the process. + The function must accept a single string parameter. The string + does not include the final newline. + @param output_prefix: String to prepend to lines sent to output_func. + """ + # Add a reader and a close hook + self._add_reader("tail") + self._add_close_hook(self._join_thread) + + # Init the superclass + kvm_spawn.__init__(self, command, id, echo, linesep) + + # Remember some attributes + self.termination_func = termination_func + self.output_func = output_func + self.output_prefix = output_prefix + + # Start the thread in the background + self.tail_thread = threading.Thread(None, self._tail) + self.tail_thread.start() + + + def __getinitargs__(self): + return kvm_spawn.__getinitargs__(self) + (self.termination_func, + self.output_func, + self.output_prefix) + + + def set_termination_func(self, termination_func): + """ + Set the termination_func attribute. See __init__() for details. + + @param termination_func: Function to call when the process terminates. + Must take a single parameter -- the exit status. + """ + self.termination_func = termination_func + + + def set_output_func(self, output_func): + """ + Set the output_func attribute. See __init__() for details. + + @param output_func: Function to call for each line of STDOUT/STDERR + output from the process. Must take a single string parameter. + """ + self.output_func = output_func + + + def set_output_prefix(self, output_prefix): + """ + Set the output_prefix attribute. See __init__() for details. + + @param output_prefix: String to pre-pend to each line sent to + output_func (see set_output_callback()). + """ + self.output_prefix = output_prefix + + + def _tail(self): + def print_line(text): + # Pre-pend prefix and remove trailing whitespace + text = self.output_prefix + text.rstrip() + # Sanitize text + text = text.decode("utf-8", "replace") + # Pass it to output_func + try: + self.output_func(text) + except TypeError: + pass + + fd = self._get_fd("tail") + buffer = "" + while True: + try: + # See if there's any data to read from the pipe + r, w, x = select.select([fd], [], [], 0.05) + except: + break + if fd in r: + # Some data is available; read it + new_data = os.read(fd, 1024) + if not new_data: + break + buffer += new_data + # Send the output to output_func line by line + # (except for the last line) + if self.output_func: + lines = buffer.split("\n") + for line in lines[:-1]: + print_line(line) + # Leave only the last line + last_newline_index = buffer.rfind("\n") + buffer = buffer[last_newline_index+1:] + else: + # No output is available right now; flush the buffer + if buffer: + print_line(buffer) + buffer = "" + # The process terminated; print any remaining output + if buffer: + print_line(buffer) + # Get the exit status, print it and send it to termination_func + status = self.get_status() + if status is None: + return + print_line("(Process terminated with status %s)" % status) + try: + self.termination_func(status) + except TypeError: + pass + + + def _join_thread(self): + # Wait for the tail thread to exit + if self.tail_thread: + self.tail_thread.join() + + +class kvm_expect(kvm_tail): + """ + This class runs a child process in the background and provides expect-like + services. + + It also provides all of kvm_tail's functionality. + """ + + def __init__(self, command=None, id=None, echo=False, linesep="\n", + termination_func=None, output_func=None, output_prefix=""): + """ + Initialize the class and run command as a child process. + + @param command: Command to run, or None if accessing an already running + server. + @param id: ID of an already running server, if accessing a running + server, or None if starting a new one. + @param echo: Boolean indicating whether echo should be initially + enabled for the pseudo terminal running the subprocess. This + parameter has an effect only when starting a new server. + @param linesep: Line separator to be appended to strings sent to the + child process by sendline(). + @param termination_func: Function to call when the process exits. The + function must accept a single exit status parameter. + @param output_func: Function to call whenever a line of output is + available from the STDOUT or STDERR streams of the process. + The function must accept a single string parameter. The string + does not include the final newline. + @param output_prefix: String to prepend to lines sent to output_func. + """ + # Add a reader + self._add_reader("expect") + + # Init the superclass + kvm_tail.__init__(self, command, id, echo, linesep, + termination_func, output_func, output_prefix) + + + def __getinitargs__(self): + return kvm_tail.__getinitargs__(self) + + + def read_nonblocking(self, timeout=None): + """ + Read from child until there is nothing to read for timeout seconds. + + @param timeout: Time (seconds) to wait before we give up reading from + the child process, or None to use the default value. + """ + if timeout is None: + timeout = 0.1 + fd = self._get_fd("expect") + data = "" + while True: + try: + r, w, x = select.select([fd], [], [], timeout) + except: + return data + if fd in r: + new_data = os.read(fd, 1024) + if not new_data: + return data + data += new_data + else: + return data + + + def match_patterns(self, str, patterns): + """ + Match str against a list of patterns. + + Return the index of the first pattern that matches a substring of str. + None and empty strings in patterns are ignored. + If no match is found, return None. + + @param patterns: List of strings (regular expression patterns). + """ + for i in range(len(patterns)): + if not patterns[i]: + continue + if re.search(patterns[i], str): + return i + + + def read_until_output_matches(self, patterns, filter=lambda x: x, + timeout=30.0, internal_timeout=None, + print_func=None): + """ + Read using read_nonblocking until a match is found using match_patterns, + or until timeout expires. Before attempting to search for a match, the + data is filtered using the filter function provided. + + @brief: Read from child using read_nonblocking until a pattern + matches. + @param patterns: List of strings (regular expression patterns) + @param filter: Function to apply to the data read from the child before + attempting to match it against the patterns (should take and + return a string) + @param timeout: The duration (in seconds) to wait until a match is + found + @param internal_timeout: The timeout to pass to read_nonblocking + @param print_func: A function to be used to print the data being read + (should take a string parameter) + @return: Tuple containing the match index (or None if no match was + found) and the data read so far. + """ + match = None + data = "" + + end_time = time.time() + timeout + while time.time() < end_time: + # Read data from child + newdata = self.read_nonblocking(internal_timeout) + # Print it if necessary + if print_func and newdata: + str = newdata + if str.endswith("\n"): + str = str[:-1] + for line in str.split("\n"): + print_func(line.decode("utf-8", "replace")) + data += newdata + + done = False + # Look for patterns + match = self.match_patterns(filter(data), patterns) + if match is not None: + done = True + # Check if child has died + if not self.is_alive(): + logging.debug("Process terminated with status %s" % self.get_status()) + done = True + # Are we done? + if done: break + + # Print some debugging info + if match is None and (self.is_alive() or self.get_status() != 0): + logging.debug("Timeout elapsed or process terminated. Output:" + + kvm_utils.format_str_for_message(data.strip())) + + return (match, data) + + + def read_until_last_word_matches(self, patterns, timeout=30.0, + internal_timeout=None, print_func=None): + """ + Read using read_nonblocking until the last word of the output matches + one of the patterns (using match_patterns), or until timeout expires. + + @param patterns: A list of strings (regular expression patterns) + @param timeout: The duration (in seconds) to wait until a match is + found + @param internal_timeout: The timeout to pass to read_nonblocking + @param print_func: A function to be used to print the data being read + (should take a string parameter) + @return: A tuple containing the match index (or None if no match was + found) and the data read so far. + """ + def get_last_word(str): + if str: + return str.split()[-1] + else: + return "" + + return self.read_until_output_matches(patterns, get_last_word, + timeout, internal_timeout, + print_func) + + + def read_until_last_line_matches(self, patterns, timeout=30.0, + internal_timeout=None, print_func=None): + """ + Read using read_nonblocking until the last non-empty line of the output + matches one of the patterns (using match_patterns), or until timeout + expires. Return a tuple containing the match index (or None if no match + was found) and the data read so far. + + @brief: Read using read_nonblocking until the last non-empty line + matches a pattern. + + @param patterns: A list of strings (regular expression patterns) + @param timeout: The duration (in seconds) to wait until a match is + found + @param internal_timeout: The timeout to pass to read_nonblocking + @param print_func: A function to be used to print the data being read + (should take a string parameter) + """ + def get_last_nonempty_line(str): + nonempty_lines = [l for l in str.splitlines() if l.strip()] + if nonempty_lines: + return nonempty_lines[-1] + else: + return "" + + return self.read_until_output_matches(patterns, get_last_nonempty_line, + timeout, internal_timeout, + print_func) + + +class kvm_shell_session(kvm_expect): + """ + This class runs a child process in the background. It it suited for + processes that provide an interactive shell, such as SSH and Telnet. + + It provides all services of kvm_expect and kvm_tail. In addition, it + provides command running services, and a utility function to test the + process for responsiveness. + """ + + def __init__(self, command=None, id=None, echo=False, linesep="\n", + termination_func=None, output_func=None, output_prefix="", + prompt=r"[\#\$]\s*$", status_test_command="echo $?"): + """ + Initialize the class and run command as a child process. + + @param command: Command to run, or None if accessing an already running + server. + @param id: ID of an already running server, if accessing a running + server, or None if starting a new one. + @param echo: Boolean indicating whether echo should be initially + enabled for the pseudo terminal running the subprocess. This + parameter has an effect only when starting a new server. + @param linesep: Line separator to be appended to strings sent to the + child process by sendline(). + @param termination_func: Function to call when the process exits. The + function must accept a single exit status parameter. + @param output_func: Function to call whenever a line of output is + available from the STDOUT or STDERR streams of the process. + The function must accept a single string parameter. The string + does not include the final newline. + @param output_prefix: String to prepend to lines sent to output_func. + @param prompt: Regular expression describing the shell's prompt line. + @param status_test_command: Command to be used for getting the last + exit status of commands run inside the shell (used by + get_command_status_output() and friends). + """ + # Init the superclass + kvm_expect.__init__(self, command, id, echo, linesep, + termination_func, output_func, output_prefix) + + # Remember some attributes + self.prompt = prompt + self.status_test_command = status_test_command + + + def __getinitargs__(self): + return kvm_expect.__getinitargs__(self) + (self.prompt, + self.status_test_command) + + + def set_prompt(self, prompt): + """ + Set the prompt attribute for later use by read_up_to_prompt. + + @param: String that describes the prompt contents. + """ + self.prompt = prompt + + + def set_status_test_command(self, status_test_command): + """ + Set the command to be sent in order to get the last exit status. + + @param status_test_command: Command that will be sent to get the last + exit status. + """ + self.status_test_command = status_test_command + + + def is_responsive(self, timeout=5.0): + """ + Return True if the process responds to STDIN/terminal input. + + Send a newline to the child process (e.g. SSH or Telnet) and read some + output using read_nonblocking(). + If all is OK, some output should be available (e.g. the shell prompt). + In that case return True. Otherwise return False. + + @param timeout: Time duration to wait before the process is considered + unresponsive. + """ + # Read all output that's waiting to be read, to make sure the output + # we read next is in response to the newline sent + self.read_nonblocking(timeout=0.1) + # Send a newline + self.sendline() + # Wait up to timeout seconds for some output from the child + end_time = time.time() + timeout + while time.time() < end_time: + time.sleep(0.5) + if self.read_nonblocking(timeout=0).strip(): + return True + # No output -- report unresponsive + return False + + + def read_up_to_prompt(self, timeout=30.0, internal_timeout=None, + print_func=None): + """ + Read using read_nonblocking until the last non-empty line of the output + matches the prompt regular expression set by set_prompt, or until + timeout expires. + + @brief: Read using read_nonblocking until the last non-empty line + matches the prompt. + + @param timeout: The duration (in seconds) to wait until a match is + found + @param internal_timeout: The timeout to pass to read_nonblocking + @param print_func: A function to be used to print the data being + read (should take a string parameter) + + @return: A tuple containing True/False indicating whether the prompt + was found, and the data read so far. + """ + (match, output) = self.read_until_last_line_matches([self.prompt], + timeout, + internal_timeout, + print_func) + return (match is not None, output) + + + def get_command_status_output(self, command, timeout=30.0, + internal_timeout=None, print_func=None): + """ + Send a command and return its exit status and output. + + @param command: Command to send (must not contain newline characters) + @param timeout: The duration (in seconds) to wait until a match is + found + @param internal_timeout: The timeout to pass to read_nonblocking + @param print_func: A function to be used to print the data being read + (should take a string parameter) + + @return: A tuple (status, output) where status is the exit status or + None if no exit status is available (e.g. timeout elapsed), and + output is the output of command. + """ + def remove_command_echo(str, cmd): + if str and str.splitlines()[0] == cmd: + str = "".join(str.splitlines(True)[1:]) + return str + + def remove_last_nonempty_line(str): + return "".join(str.rstrip().splitlines(True)[:-1]) + + # Print some debugging info + logging.debug("Sending command: %s" % command) + + # Read everything that's waiting to be read + self.read_nonblocking(0.1) + + # Send the command and get its output + self.sendline(command) + (match, output) = self.read_up_to_prompt(timeout, internal_timeout, + print_func) + # Remove the echoed command from the output + output = remove_command_echo(output, command) + # If the prompt was not found, return the output so far + if not match: + return (None, output) + # Remove the final shell prompt from the output + output = remove_last_nonempty_line(output) + + # Send the 'echo ...' command to get the last exit status + self.sendline(self.status_test_command) + (match, status) = self.read_up_to_prompt(10.0, internal_timeout) + if not match: + return (None, output) + status = remove_command_echo(status, self.status_test_command) + status = remove_last_nonempty_line(status) + # Get the first line consisting of digits only + digit_lines = [l for l in status.splitlines() if l.strip().isdigit()] + if not digit_lines: + return (None, output) + status = int(digit_lines[0].strip()) + + # Print some debugging info + if status != 0: + logging.debug("Command failed; status: %d, output:%s", status, + kvm_utils.format_str_for_message(output.strip())) + + return (status, output) + + + def get_command_status(self, command, timeout=30.0, internal_timeout=None, + print_func=None): + """ + Send a command and return its exit status. + + @param command: Command to send + @param timeout: The duration (in seconds) to wait until a match is + found + @param internal_timeout: The timeout to pass to read_nonblocking + @param print_func: A function to be used to print the data being read + (should take a string parameter) + + @return: Exit status or None if no exit status is available (e.g. + timeout elapsed). + """ + (status, output) = self.get_command_status_output(command, timeout, + internal_timeout, + print_func) + return status + + + def get_command_output(self, command, timeout=30.0, internal_timeout=None, + print_func=None): + """ + Send a command and return its output. + + @param command: Command to send + @param timeout: The duration (in seconds) to wait until a match is + found + @param internal_timeout: The timeout to pass to read_nonblocking + @param print_func: A function to be used to print the data being read + (should take a string parameter) + """ + (status, output) = self.get_command_status_output(command, timeout, + internal_timeout, + print_func) + return output + + +# The following is the server part of the module. + +def _server_main(): + id = sys.stdin.readline().strip() + echo = sys.stdin.readline().strip() == "True" + readers = sys.stdin.readline().strip().split(",") + command = sys.stdin.readline().strip() + " && echo %s > /dev/null" % id + + # Define filenames to be used for communication + base_dir = "/tmp/kvm_spawn" + (shell_pid_filename, + status_filename, + output_filename, + inpipe_filename, + lock_server_running_filename, + lock_client_starting_filename) = _get_filenames(base_dir, id) + + # Populate the reader filenames list + reader_filenames = [_get_reader_filename(base_dir, id, reader) + for reader in readers] + + # Set $TERM = dumb + os.putenv("TERM", "dumb") + + (shell_pid, shell_fd) = pty.fork() + if shell_pid == 0: + # Child process: run the command in a subshell + os.execv("/bin/sh", ["/bin/sh", "-c", command]) + else: + # Parent process + lock_server_running = _lock(lock_server_running_filename) + + # Set terminal echo on/off and disable pre- and post-processing + attr = termios.tcgetattr(shell_fd) + attr[0] &= ~termios.INLCR + attr[0] &= ~termios.ICRNL + attr[0] &= ~termios.IGNCR + attr[1] &= ~termios.OPOST + if echo: + attr[3] |= termios.ECHO + else: + attr[3] &= ~termios.ECHO + termios.tcsetattr(shell_fd, termios.TCSANOW, attr) + + # Open output file + output_file = open(output_filename, "w") + # Open input pipe + os.mkfifo(inpipe_filename) + inpipe_fd = os.open(inpipe_filename, os.O_RDWR) + # Open output pipes (readers) + reader_fds = [] + for filename in reader_filenames: + os.mkfifo(filename) + reader_fds.append(os.open(filename, os.O_RDWR)) + + # Write shell PID to file + file = open(shell_pid_filename, "w") + file.write(str(shell_pid)) + file.close() + + # Print something to stdout so the client can start working + print "hello" + sys.stdout.flush() + + # Initialize buffers + buffers = ["" for reader in readers] + + # Read from child and write to files/pipes + while True: + # Make a list of reader pipes whose buffers are not empty + fds = [fd for (i, fd) in enumerate(reader_fds) if buffers[i]] + # Wait until there's something to do + r, w, x = select.select([shell_fd, inpipe_fd], fds, []) + # If a reader pipe is ready for writing -- + for (i, fd) in enumerate(reader_fds): + if fd in w: + bytes_written = os.write(fd, buffers[i]) + buffers[i] = buffers[i][bytes_written:] + # If there's data to read from the child process -- + if shell_fd in r: + try: + data = os.read(shell_fd, 16384) + except OSError: + break + # Remove carriage returns from the data -- they often cause + # trouble and are normally not needed + data = data.replace("\r", "") + output_file.write(data) + output_file.flush() + for i in range(len(readers)): + buffers[i] += data + # If there's data to read from the client -- + if inpipe_fd in r: + data = os.read(inpipe_fd, 1024) + os.write(shell_fd, data) + + # Wait for the shell process to exit and get its exit status + status = os.waitpid(shell_pid, 0)[1] + status = os.WEXITSTATUS(status) + file = open(status_filename, "w") + file.write(str(status)) + file.close() + + # Wait for the client to finish initializing + _wait(lock_client_starting_filename) + + # Delete FIFOs + for filename in reader_filenames + [inpipe_filename]: + try: + os.unlink(filename) + except OSError: + pass + + # Close all files and pipes + output_file.close() + os.close(inpipe_fd) + for fd in reader_fds: + os.close(fd) + + _unlock(lock_server_running) + + +if __name__ == "__main__": + _server_main() --1.5.4.1
_______________________________________________
Autotest mailing list
[email protected]
http://test.kernel.org/cgi-bin/mailman/listinfo/autotest
Signature · 1 line
Lucas Meneghel
Re: [KVM-AUTOTEST,01/17] Add new module kvm_subprocess
In reply to Michael GoldishHi, Michael,
I found that if the sessions initialized using kvm_subprcoess are not closed, the processes will never exit, and /tmp/kvm_spawn will be filled with the temporary files.
And we can find in the code,
# kvm_subprocess.py
...
# Read from child and write to files/pipes
while True:
check_termination = False
# Make a list of reader pipes whose buffers are not empty
fds = [fd for (i, fd) in enumerate(reader_fds) if buffers[i]]
# Wait until there's something to do
r, w, x = select.select([shell_fd, inpipe_fd], fds, [], 0.5)
# If a reader pipe is ready for writing --
for (i, fd) in enumerate(reader_fds):
if fd in w:
bytes_written = os.write(fd, buffers[i])
buffers[i] = buffers[i][bytes_written:]
# If there's data to read from the child process --
if shell_fd in r:
try:
data = os.read(shell_fd, 16384)
except OSError:
data = ""
if not data:
check_termination = True
# Remove carriage returns from the data -- they often cause
# trouble and are normally not needed
data = data.replace("\r", "")
output_file.write(data)
output_file.flush()
for i in range(len(readers)):
buffers[i] += data
# If os.read() raised an exception or there was nothing to read --
if check_termination or shell_fd not in r:
pid, status = os.waitpid(shell_pid, os.WNOHANG)
if pid:
status = os.WEXITSTATUS(status)
break
# If there's data to read from the client --
if inpipe_fd in r:
data = os.read(inpipe_fd, 1024)
os.write(shell_fd, data)
...that if session.close() is not called, we will loop in the 'while' forever.
So, user have to make sure that unnecessary sessions are all killed, otherwise, running some testcase(s) for huge number of times will suck out all the system resource, which I think is very inconvenient. Especially when we have to take care of many exceptions that may be raised by our program.
e.g.
...
session = kvm_test_utils.wait_for_login(vm)
...
session2 = kvm_test_utils.wait_for_login(vm_x)
...
try:
...
except ...:
......
(other code may raise exceptions)
...
try:
...
except ...:
...
...
try:
...
except ...:
......
cleaning up the sessions will be exhausting here.
Do we have a good (or better) way to handle this?
Thanks.
Regards,
Cao, Chen
2009-10-12
Quoted 35 lines · Michael Goldish
This module is intended to be used for controlling all child processes in KVM
tests: both QEMU processes and SSH/SCP/Telnet processes. Processes started with
this module keep running and can be interacted with even after the parent
process exits.The current
run_bg()utility tracks a child process as long as the parent
process is running. When the parent process exits, the tracking thread
terminates and cannot resume when needed.Currently SSH/SCP/Telnet communication is handled by kvm_utils.kvm_spawn, which
does not allow the child process to run after the parent process exits. Thus,
open SSH/SCP/Telnet sessions cannot be reused by tests following the one in
which they are opened.The new module provides a solution to these two problems, and also saves some
code by reusing common code required both for QEMU processes and SSH/SCP/Telnet
processes.Signed-off-by Michael Goldish <[email protected]>--- client/tests/kvm/kvm_subprocess.py | 1146 ++++++++++++++++++++++++++++++++++++ 1 files changed, 1146 insertions(+), 0 deletions(-) create mode 100644 client/tests/kvm/kvm_subprocess.py
PATCH 1/1 KVM test: kvm_subprocess: add function kill_tail_thread()
Normally all threads are killed when the test process exits. However, when running multiple iterations of a test (with iterations=n) the test process remains alive between iterations, and new threads are started but old threads are not killed. This function allow to stop thread execution explicitly.
This patch also makes the postprocessor call the function for all VMs.
--- client/tests/kvm/kvm_preprocessing.py | 4 ++++ client/tests/kvm/kvm_subprocess.py | 12 ++++++++++++ client/tests/kvm/kvm_vm.py | 8 ++++++++ 3 files changed, 24 insertions(+), 0 deletions(-)
client/tests/kvm/kvm_preprocessing.py +4 -0
@@ -288,6 +288,10 @@ def postprocess(test, params, env): int(params.get("post_command_timeout", "600")), params.get("post_command_noncritical") == "yes") + # Kill the tailing threads of all VMs + for vm in kvm_utils.env_get_all_vms(env): + vm.kill_tail_thread() + # Terminate tcpdump if no VMs are alive living_vms = [vm for vm in kvm_utils.env_get_all_vms(env) if vm.is_alive()] if not living_vms and env.has_key("tcpdump"):
client/tests/kvm/kvm_subprocess.py +12 -0
@@ -489,6 +489,7 @@ class kvm_tail(kvm_spawn): self.output_prefix = output_prefix # Start the thread in the background + self.__thread_kill_requested = False self.tail_thread = threading.Thread(None, self._tail) self.tail_thread.start()
@@ -551,6 +552,15 @@ class kvm_tail(kvm_spawn): self.output_prefix = output_prefix + def kill_tail_thread(self): + """ + Stop the tailing thread which calls output_func() and + termination_func(). + """ + self.__thread_kill_requested = True + self._join_thread() + + def _tail(self): def print_line(text): # Pre-pend prefix and remove trailing whitespace
@@ -567,6 +577,8 @@ class kvm_tail(kvm_spawn): fd = self._get_fd("tail") buffer = "" while True: + if self.__thread_kill_requested: + return try: # See if there's any data to read from the pipe r, w, x = select.select([fd], [], [], 0.05)
client/tests/kvm/kvm_vm.py +8 -0
@@ -573,6 +573,14 @@ class VM: return not self.process or not self.process.is_alive() + def kill_tail_thread(self): + """ + Stop the tailing thread which reports the output of qemu. + """ + if self.process: + self.process.kill_tail_thread() + + def get_params(self): """ Return the VM's params dict. Most modified params take effect only
Signature · 1 line
1.5.4.1
PATCH 1/1 KVM test: kvm_tests.cfg.sample: improve shell_prompt regular expressions
Require that shell prompts start at the beginning of the line (^) and be followed by nothing more than whitespace (\s*$).
Also remove the length limitation from Windows prompts, and do not restrict them to C:\.
Also do not require RHEL and Fedora prompts to contain 'root@'.
--- client/tests/kvm/kvm_tests.cfg.sample | 8 ++++---- 1 files changed, 4 insertions(+), 4 deletions(-)
client/tests/kvm/kvm_tests.cfg.sample +4 -4
@@ -171,7 +171,7 @@ variants: variants: - Fedora: no setup - shell_prompt = "\[root@.{0,50}][\#\$] " + shell_prompt = "^\[.*\][\#\$]\s*$" variants: - 8.32:
@@ -253,7 +253,7 @@ variants: md5sum_1m = 768ca32503ef92c28f2d144f2a87e4d0 - @Ubuntu: - shell_prompt = "root@.{0,50}[\#\$] " + shell_prompt = "^root@.*[\#\$]\s*$" variants: - Ubuntu-6.10-32:
@@ -285,7 +285,7 @@ variants: - RHEL: no setup - shell_prompt = "\[root@.{0,50}][\#\$] " + shell_prompt = "^\[.*\][\#\$]\s*$" variants: - 5.3.i386:
@@ -348,7 +348,7 @@ variants: shutdown_command = shutdown /s /t 0 reboot_command = shutdown /r /t 0 status_test_command = echo %errorlevel% - shell_prompt = "C:\\.{0,50}>" + shell_prompt = "^\w:\\.*>\s*$" shell_client = nc shell_port = 22 # File transfers are currently unsupported
Signature · 1 line
1.5.4.1