Skip to contentSkip to conversation

kvm_vm.py: create a lock file to avoid VM creation in parallel

PATCH Applied Lucas Meneghel Rodrigues 1 participant 12 messages Jun 9, 2009 – Jun 10, 2009 KVM

lore
Share & tools
lore
#0 lore raw

PATCH 1/6 kvm_vm.py: create a lock file to avoid VM creation in parallel

VM.create() does a few things (such as finding free ports) which are not safe to execute in parallel. Use a lock file to make sure this doesn't happen. The lock is released only after the VM is started or fails to start.

---
 client/tests/kvm/kvm_vm.py |   91 ++++++++++++++++++++++++--------------------
 1 files changed, 50 insertions(+), 41 deletions(-)
client/tests/kvm/kvm_vm.py +49 -40
index eb9717b..e2b684c 100644
@@ -335,51 +335,60 @@ class VM:
                     logging.error("Actual MD5 sum differs from expected one")
                     return False
 
-        # Handle port redirections
-        redir_names = kvm_utils.get_sub_dict_names(params, "redirs")
-        host_ports = kvm_utils.find_free_ports(5000, 6000, len(redir_names))
-        self.redirs = {}
-        for i in range(len(redir_names)):
-            redir_params = kvm_utils.get_sub_dict(params, redir_names[i])
-            guest_port = int(redir_params.get("guest_port"))
-            self.redirs[guest_port] = host_ports[i]
-
-        # Find available VNC port, if needed
-        if params.get("display") == "vnc":
-            self.vnc_port = kvm_utils.find_free_port(5900, 6000)
-
-        # Make qemu command
-        qemu_command = self.make_qemu_command()
+        # Make sure the following code is not executed by more than one thread
+        # at the same time
+        lockfile = open("/tmp/kvm-autotest-vm-create.lock", "w+")
+        fcntl.lockf(lockfile, fcntl.LOCK_EX)
 
-        # Is this VM supposed to accept incoming migrations?
-        if for_migration:
-            # Find available migration port
-            self.migration_port = kvm_utils.find_free_port(5200, 6000)
-            # Add -incoming option to the qemu command
-            qemu_command += " -incoming tcp:0:%d" % self.migration_port
-
-        logging.debug("Running qemu command:\n%s" % qemu_command)
-        (status, pid, output) = kvm_utils.run_bg(qemu_command, None,
-                                                 logging.debug, "(qemu) ")
-
-        if status:
-            logging.debug("qemu exited with status %d" % status)
-            logging.error("VM could not be created -- qemu command"
-                          " failed:\n%s" % qemu_command)
-            return False
+        try:
+            # Handle port redirections
+            redir_names = kvm_utils.get_sub_dict_names(params, "redirs")
+            host_ports = kvm_utils.find_free_ports(5000, 6000, len(redir_names))
+            self.redirs = {}
+            for i in range(len(redir_names)):
+                redir_params = kvm_utils.get_sub_dict(params, redir_names[i])
+                guest_port = int(redir_params.get("guest_port"))
+                self.redirs[guest_port] = host_ports[i]
+
+            # Find available VNC port, if needed
+            if params.get("display") == "vnc":
+                self.vnc_port = kvm_utils.find_free_port(5900, 6000)
+
+            # Make qemu command
+            qemu_command = self.make_qemu_command()
+
+            # Is this VM supposed to accept incoming migrations?
+            if for_migration:
+                # Find available migration port
+                self.migration_port = kvm_utils.find_free_port(5200, 6000)
+                # Add -incoming option to the qemu command
+                qemu_command += " -incoming tcp:0:%d" % self.migration_port
+
+            logging.debug("Running qemu command:\n%s", qemu_command)
+            (status, pid, output) = kvm_utils.run_bg(qemu_command, None,
+                                                     logging.debug, "(qemu) ")
+
+            if status:
+                logging.debug("qemu exited with status %d", status)
+                logging.error("VM could not be created -- qemu command"
+                              " failed:\n%s", qemu_command)
+                return False
 
-        self.pid = pid
+            self.pid = pid
 
-        if not kvm_utils.wait_for(self.is_alive, timeout, 0, 1):
-            logging.debug("VM is not alive for some reason")
-            logging.error("VM could not be created with"
-                          " command:\n%s" % qemu_command)
-            self.destroy()
-            return False
+            if not kvm_utils.wait_for(self.is_alive, timeout, 0, 1):
+                logging.debug("VM is not alive for some reason")
+                logging.error("VM could not be created with"
+                              " command:\n%s", qemu_command)
+                self.destroy()
+                return False
 
-        logging.debug("VM appears to be alive with PID %d" % self.pid)
+            logging.debug("VM appears to be alive with PID %d", self.pid)
+            return True
 
-        return True
+        finally:
+            fcntl.lockf(lockfile, fcntl.LOCK_UN)
+            lockfile.close()
 
 
     def send_monitor_cmd(self, command, block=True, timeout=20.0):
Signature · 1 line
1.6.2.2
#1 lore raw

PATCH 2/6 kvm_runtest_2.py: Use env filename specified by the 'env' parameter

Do not use hardcoded environment filename 'env'. Instead use the value specified by the 'env' parameter. If unspecified, use 'env' as the filename.

This is important for parallel execution; it may be necessary to use a separate
environment file for each process.

---
 client/tests/kvm/kvm.py |    2 +-
 1 files changed, 1 insertions(+), 1 deletions(-)
client/tests/kvm/kvm.py +1 -1
index a658425..9428162 100644
@@ -82,7 +82,7 @@ class kvm(test.test):
             self.write_test_keyval({key: params[key]})
 
         # Open the environment file
-        env_filename = os.path.join(self.bindir, "env")
+        env_filename = os.path.join(self.bindir, params.get("env", "env"))
         env = load_env(env_filename, {})
         logging.debug("Contents of environment: %s" % str(env))
 
Signature · 1 line
1.6.2.2
#2 lore raw

PATCH 3/6 kvm_vm.py: add new VM parameter 'x11_display' that controls $DISPLAY

If x11_display is specified, the DISPLAY environment variable is set to this value for the QEMU process. This may be useful for SDL rendering. Also add some comments.

---
 client/tests/kvm/kvm_vm.py |   13 ++++++++++++-
 1 files changed, 12 insertions(+), 1 deletions(-)
client/tests/kvm/kvm_vm.py +12 -1
index e2b684c..6de3989 100644
@@ -196,6 +196,9 @@ class VM:
                (iso_dir is pre-pended to the ISO filename)
                extra_params -- a string to append to the qemu command
                ssh_port -- should be 22 for SSH, 23 for Telnet
+               x11_display -- if specified, the DISPLAY environment variable
+               will be be set to this value for the qemu process (useful for
+               SDL rendering)
                images -- a list of image object names, separated by spaces
                nics -- a list of NIC object names, separated by spaces
 
@@ -222,8 +225,16 @@ class VM:
         if iso_dir == None:
             iso_dir = self.iso_dir
 
-        qemu_cmd = qemu_path
+        # Start constructing the qemu command
+        qemu_cmd = ""
+        # Set the X11 display parameter if requested
+        if params.get("x11_display"):
+            qemu_cmd += "DISPLAY=%s " % params.get("x11_display")
+        # Add the qemu binary
+        qemu_cmd += qemu_path
+        # Add the VM's name
         qemu_cmd += " -name '%s'" % name
+        # Add the monitor socket parameter
         qemu_cmd += " -monitor unix:%s,server,nowait" % self.monitor_file_name
 
         for image_name in kvm_utils.get_sub_dict_names(params, "images"):
Signature · 1 line
1.6.2.2
#3 lore raw

PATCH 4/6 VM.create(): always destroy() the VM before attempting to start it

Also, don't do it in kvm_preprocessing.py since it's now done in kvm_vm.py.

---
 client/tests/kvm/kvm_preprocessing.py |    1 -
 client/tests/kvm/kvm_vm.py            |    2 ++
 2 files changed, 2 insertions(+), 1 deletions(-)
client/tests/kvm/kvm_preprocessing.py +0 -1
index e395d98..f60cfe8 100644
@@ -80,7 +80,6 @@ def preprocess_vm(test, params, env, name):
             start_vm = True
 
     if start_vm:
-        vm.destroy()
         if not vm.create(name, params, qemu_path, image_dir, iso_dir,
                          for_migration):
             message = "Could not start VM"
client/tests/kvm/kvm_vm.py +2 -0
index 6de3989..d7b0735 100644
@@ -303,6 +303,8 @@ class VM:
         @param for_migration: If True, start the VM with the -incoming
         option
         """
+        self.destroy()
+
         if name != None:
             self.name = name
         if params != None:
Signature · 1 line
1.6.2.2
#4 lore raw

PATCH 5/6 kvm_utils.py: remote_login(): improve regular expression matching

1. Make the 'login:' regular expression stricter so it doesn't match 'Last login: ...' messages.
2. Make the 'password:' regular expression stricter.
3. Handle 'Connection refused' messages.

---
 client/tests/kvm/kvm_utils.py |   14 +++++++++-----
 1 files changed, 9 insertions(+), 5 deletions(-)
client/tests/kvm/kvm_utils.py +9 -5
index 434190d..e9d90ac 100644
@@ -539,8 +539,8 @@ def remote_login(command, password, prompt, linesep="\n", timeout=10):
 
     while True:
         (match, text) = sub.read_until_last_line_matches(
-                ["[Aa]re you sure", "[Pp]assword:", "[Ll]ogin:",
-                 "[Cc]onnection.*closed", prompt],
+                [r"[Aa]re you sure", r"[Pp]assword:\s*$", r"^\s*[Ll]ogin:\s*$",
+                 r"[Cc]onnection.*closed", r"[Cc]onnection.*refused", prompt],
                  timeout=timeout, internal_timeout=0.5)
         if match == 0:  # "Are you sure you want to continue connecting"
             logging.debug("Got 'Are you sure...'; sending 'yes'")
@@ -564,11 +564,15 @@ def remote_login(command, password, prompt, linesep="\n", timeout=10):
             logging.debug("Got 'Connection closed'")
             sub.close()
             return None
-        elif match == 4:  # prompt
+        elif match == 4:  # "Connection refused"
+            kvm_log.debug("Got 'Connection refused'")
+            sub.close()
+            return None
+        elif match == 5:  # prompt
             logging.debug("Got shell prompt -- logged in")
             return sub
         else:  # match == None
-            logging.debug("Timeout or process terminated")
+            logging.debug("Timeout elapsed or process terminated")
             sub.close()
             return None
 
@@ -602,7 +606,7 @@ def remote_scp(command, password, timeout=300, login_timeout=10):
 
     while True:
         (match, text) = sub.read_until_last_line_matches(
-                ["[Aa]re you sure", "[Pp]assword:", "lost connection"],
+                [r"[Aa]re you sure", r"[Pp]assword:\s*$", r"lost connection"],
                 timeout=_timeout, internal_timeout=0.5)
         if match == 0:  # "Are you sure you want to continue connecting"
             logging.debug("Got 'Are you sure...'; sending 'yes'")
Signature · 1 line
1.6.2.2
#5 lore raw

PATCH 6/6 kvm_config: Make split_and_strip function to strip the right quotes

In some occasions, the current logic to strip quotest from config parameters might remove valid quotes from the params. This patch fixes this problem.

---
 client/tests/kvm/kvm_config.py |    5 ++++-
 1 files changed, 4 insertions(+), 1 deletions(-)
client/tests/kvm/kvm_config.py +4 -1
index 40f16f1..5d9b69e 100755
@@ -139,7 +139,10 @@ class config:
         temp = str.split(sep, 1)
         for i in range(len(temp)):
             temp[i] = temp[i].strip()
-            temp[i] = temp[i].strip("\"\'")
+            if re.findall("^\".*\"$", temp[i]):
+                temp[i] = temp[i].strip("\"")
+            elif re.findall("^\'.*\'$", temp[i]):
+                temp[i] = temp[i].strip("\'")
         return temp
 
 
Signature · 1 line
1.6.2.2
#6 lore raw

Re: [KVM-AUTOTEST PATCH 06/06] kvm_config: Make split_and_strip function to strip the right quotes

Quoted 3 lines · Lucas Meneghel Rodrigues

In some occasions, the current logic to strip quotest from config parameters might remove valid quotes from the params. This patch fixes this problem.

Applied, thanks!

Quoted 21 lines
---
 client/tests/kvm/kvm_config.py |    5 ++++-
 1 files changed, 4 insertions(+), 1 deletions(-)
client/tests/kvm/kvm_config.py +4 -1
index 40f16f1..5d9b69e 100755
@@ -139,7 +139,10 @@ class config:
         temp = str.split(sep, 1)
         for i in range(len(temp)):
             temp[i] = temp[i].strip()
-            temp[i] = temp[i].strip("\"\'")
+            if re.findall("^\".*\"$", temp[i]):
+                temp[i] = temp[i].strip("\"")
+            elif re.findall("^\'.*\'$", temp[i]):
+                temp[i] = temp[i].strip("\'")
         return temp
 
 
Signature · 3 lines
Lucas Meneghel Rodrigues
Software Engineer (QE)
Red Hat - Emerging Technologies
#7 lore raw

Re: [KVM-AUTOTEST PATCH 05/06] kvm_utils.py: remote_login(): improve regular expression matching

In reply to Lucas Meneghel Rodrigues
Quoted 4 lines · Lucas Meneghel Rodrigues

1. Make the 'login:' regular expression stricter so it doesn't match 'Last login: ...' messages.
2. Make the 'password:' regular expression stricter.
3. Handle 'Connection refused' messages.

Applied, thanks!

Quoted 47 lines
---
 client/tests/kvm/kvm_utils.py |   14 +++++++++-----
 1 files changed, 9 insertions(+), 5 deletions(-)
client/tests/kvm/kvm_utils.py +9 -5
index 434190d..e9d90ac 100644
@@ -539,8 +539,8 @@ def remote_login(command, password, prompt, linesep="\n", timeout=10):
 
     while True:
         (match, text) = sub.read_until_last_line_matches(
-                ["[Aa]re you sure", "[Pp]assword:", "[Ll]ogin:",
-                 "[Cc]onnection.*closed", prompt],
+                [r"[Aa]re you sure", r"[Pp]assword:\s*$", r"^\s*[Ll]ogin:\s*$",
+                 r"[Cc]onnection.*closed", r"[Cc]onnection.*refused", prompt],
                  timeout=timeout, internal_timeout=0.5)
         if match == 0:  # "Are you sure you want to continue connecting"
             logging.debug("Got 'Are you sure...'; sending 'yes'")
@@ -564,11 +564,15 @@ def remote_login(command, password, prompt, linesep="\n", timeout=10):
             logging.debug("Got 'Connection closed'")
             sub.close()
             return None
-        elif match == 4:  # prompt
+        elif match == 4:  # "Connection refused"
+            kvm_log.debug("Got 'Connection refused'")
+            sub.close()
+            return None
+        elif match == 5:  # prompt
             logging.debug("Got shell prompt -- logged in")
             return sub
         else:  # match == None
-            logging.debug("Timeout or process terminated")
+            logging.debug("Timeout elapsed or process terminated")
             sub.close()
             return None
 
@@ -602,7 +606,7 @@ def remote_scp(command, password, timeout=300, login_timeout=10):
 
     while True:
         (match, text) = sub.read_until_last_line_matches(
-                ["[Aa]re you sure", "[Pp]assword:", "lost connection"],
+                [r"[Aa]re you sure", r"[Pp]assword:\s*$", r"lost connection"],
                 timeout=_timeout, internal_timeout=0.5)
         if match == 0:  # "Are you sure you want to continue connecting"
             logging.debug("Got 'Are you sure...'; sending 'yes'")
Signature · 3 lines
Lucas Meneghel Rodrigues
Software Engineer (QE)
Red Hat - Emerging Technologies
#8 lore raw

Re: [KVM-AUTOTEST PATCH 04/06] VM.create(): always destroy() the VM before attempting to start it

In reply to Lucas Meneghel Rodrigues
Quoted 2 lines · Lucas Meneghel Rodrigues

Also, don't do it in kvm_preprocessing.py since it's now done in kvm_vm.py.

Applied, thanks!

Quoted 31 lines
---
 client/tests/kvm/kvm_preprocessing.py |    1 -
 client/tests/kvm/kvm_vm.py            |    2 ++
 2 files changed, 2 insertions(+), 1 deletions(-)
client/tests/kvm/kvm_preprocessing.py +0 -1
index e395d98..f60cfe8 100644
@@ -80,7 +80,6 @@ def preprocess_vm(test, params, env, name):
             start_vm = True
 
     if start_vm:
-        vm.destroy()
         if not vm.create(name, params, qemu_path, image_dir, iso_dir,
                          for_migration):
             message = "Could not start VM"
client/tests/kvm/kvm_vm.py +2 -0
index 6de3989..d7b0735 100644
@@ -303,6 +303,8 @@ class VM:
         @param for_migration: If True, start the VM with the -incoming
         option
         """
+        self.destroy()
+
         if name != None:
             self.name = name
         if params != None:
Signature · 3 lines
Lucas Meneghel Rodrigues
Software Engineer (QE)
Red Hat - Emerging Technologies
#9 lore raw

Re: [KVM-AUTOTEST PATCH 03/06] kvm_vm.py: add new VM parameter 'x11_display' that controls $DISPLAY

In reply to Lucas Meneghel Rodrigues
Quoted 3 lines · Lucas Meneghel Rodrigues

If x11_display is specified, the DISPLAY environment variable is set to this value for the QEMU process. This may be useful for SDL rendering. Also add some comments.

Applied, thanks!

Quoted 37 lines
---
 client/tests/kvm/kvm_vm.py |   13 ++++++++++++-
 1 files changed, 12 insertions(+), 1 deletions(-)
client/tests/kvm/kvm_vm.py +12 -1
index e2b684c..6de3989 100644
@@ -196,6 +196,9 @@ class VM:
                (iso_dir is pre-pended to the ISO filename)
                extra_params -- a string to append to the qemu command
                ssh_port -- should be 22 for SSH, 23 for Telnet
+               x11_display -- if specified, the DISPLAY environment variable
+               will be be set to this value for the qemu process (useful for
+               SDL rendering)
                images -- a list of image object names, separated by spaces
                nics -- a list of NIC object names, separated by spaces
 
@@ -222,8 +225,16 @@ class VM:
         if iso_dir == None:
             iso_dir = self.iso_dir
 
-        qemu_cmd = qemu_path
+        # Start constructing the qemu command
+        qemu_cmd = ""
+        # Set the X11 display parameter if requested
+        if params.get("x11_display"):
+            qemu_cmd += "DISPLAY=%s " % params.get("x11_display")
+        # Add the qemu binary
+        qemu_cmd += qemu_path
+        # Add the VM's name
         qemu_cmd += " -name '%s'" % name
+        # Add the monitor socket parameter
         qemu_cmd += " -monitor unix:%s,server,nowait" % self.monitor_file_name
 
         for image_name in kvm_utils.get_sub_dict_names(params, "images"):
Signature · 3 lines
Lucas Meneghel Rodrigues
Software Engineer (QE)
Red Hat - Emerging Technologies
#10 lore raw

Re: [KVM-AUTOTEST PATCH 02/06] kvm_runtest_2.py: Use env filename specified by the 'env' parameter

In reply to Lucas Meneghel Rodrigues
Quoted 7 lines · Lucas Meneghel Rodrigues

Do not use hardcoded environment filename 'env'. Instead use the value specified by the 'env' parameter. If unspecified, use 'env' as the filename.

This is important for parallel execution; it may be necessary to use a separate
environment file for each process.

Applied, thanks!

Quoted 18 lines
---
 client/tests/kvm/kvm.py |    2 +-
 1 files changed, 1 insertions(+), 1 deletions(-)
client/tests/kvm/kvm.py +1 -1
index a658425..9428162 100644
@@ -82,7 +82,7 @@ class kvm(test.test):
             self.write_test_keyval({key: params[key]})
 
         # Open the environment file
-        env_filename = os.path.join(self.bindir, "env")
+        env_filename = os.path.join(self.bindir, params.get("env", "env"))
         env = load_env(env_filename, {})
         logging.debug("Contents of environment: %s" % str(env))
 
Signature · 3 lines
Lucas Meneghel Rodrigues
Software Engineer (QE)
Red Hat - Emerging Technologies
#11 lore raw
In reply to Lucas Meneghel Rodrigues
Quoted 4 lines · Lucas Meneghel Rodrigues

VM.create() does a few things (such as finding free ports) which are not safe to execute in parallel. Use a lock file to make sure this doesn't happen. The lock is released only after the VM is started or fails to start.

Applied, thanks!

Quoted 110 lines
---
 client/tests/kvm/kvm_vm.py |   91 ++++++++++++++++++++++++--------------------
 1 files changed, 50 insertions(+), 41 deletions(-)
client/tests/kvm/kvm_vm.py +49 -40
index eb9717b..e2b684c 100644
@@ -335,51 +335,60 @@ class VM:
                     logging.error("Actual MD5 sum differs from expected one")
                     return False
 
-        # Handle port redirections
-        redir_names = kvm_utils.get_sub_dict_names(params, "redirs")
-        host_ports = kvm_utils.find_free_ports(5000, 6000, len(redir_names))
-        self.redirs = {}
-        for i in range(len(redir_names)):
-            redir_params = kvm_utils.get_sub_dict(params, redir_names[i])
-            guest_port = int(redir_params.get("guest_port"))
-            self.redirs[guest_port] = host_ports[i]
-
-        # Find available VNC port, if needed
-        if params.get("display") == "vnc":
-            self.vnc_port = kvm_utils.find_free_port(5900, 6000)
-
-        # Make qemu command
-        qemu_command = self.make_qemu_command()
+        # Make sure the following code is not executed by more than one thread
+        # at the same time
+        lockfile = open("/tmp/kvm-autotest-vm-create.lock", "w+")
+        fcntl.lockf(lockfile, fcntl.LOCK_EX)
 
-        # Is this VM supposed to accept incoming migrations?
-        if for_migration:
-            # Find available migration port
-            self.migration_port = kvm_utils.find_free_port(5200, 6000)
-            # Add -incoming option to the qemu command
-            qemu_command += " -incoming tcp:0:%d" % self.migration_port
-
-        logging.debug("Running qemu command:\n%s" % qemu_command)
-        (status, pid, output) = kvm_utils.run_bg(qemu_command, None,
-                                                 logging.debug, "(qemu) ")
-
-        if status:
-            logging.debug("qemu exited with status %d" % status)
-            logging.error("VM could not be created -- qemu command"
-                          " failed:\n%s" % qemu_command)
-            return False
+        try:
+            # Handle port redirections
+            redir_names = kvm_utils.get_sub_dict_names(params, "redirs")
+            host_ports = kvm_utils.find_free_ports(5000, 6000, len(redir_names))
+            self.redirs = {}
+            for i in range(len(redir_names)):
+                redir_params = kvm_utils.get_sub_dict(params, redir_names[i])
+                guest_port = int(redir_params.get("guest_port"))
+                self.redirs[guest_port] = host_ports[i]
+
+            # Find available VNC port, if needed
+            if params.get("display") == "vnc":
+                self.vnc_port = kvm_utils.find_free_port(5900, 6000)
+
+            # Make qemu command
+            qemu_command = self.make_qemu_command()
+
+            # Is this VM supposed to accept incoming migrations?
+            if for_migration:
+                # Find available migration port
+                self.migration_port = kvm_utils.find_free_port(5200, 6000)
+                # Add -incoming option to the qemu command
+                qemu_command += " -incoming tcp:0:%d" % self.migration_port
+
+            logging.debug("Running qemu command:\n%s", qemu_command)
+            (status, pid, output) = kvm_utils.run_bg(qemu_command, None,
+                                                     logging.debug, "(qemu) ")
+
+            if status:
+                logging.debug("qemu exited with status %d", status)
+                logging.error("VM could not be created -- qemu command"
+                              " failed:\n%s", qemu_command)
+                return False
 
-        self.pid = pid
+            self.pid = pid
 
-        if not kvm_utils.wait_for(self.is_alive, timeout, 0, 1):
-            logging.debug("VM is not alive for some reason")
-            logging.error("VM could not be created with"
-                          " command:\n%s" % qemu_command)
-            self.destroy()
-            return False
+            if not kvm_utils.wait_for(self.is_alive, timeout, 0, 1):
+                logging.debug("VM is not alive for some reason")
+                logging.error("VM could not be created with"
+                              " command:\n%s", qemu_command)
+                self.destroy()
+                return False
 
-        logging.debug("VM appears to be alive with PID %d" % self.pid)
+            logging.debug("VM appears to be alive with PID %d", self.pid)
+            return True
 
-        return True
+        finally:
+            fcntl.lockf(lockfile, fcntl.LOCK_UN)
+            lockfile.close()
 
 
     def send_monitor_cmd(self, command, block=True, timeout=20.0):
Signature · 3 lines
Lucas Meneghel Rodrigues
Software Engineer (QE)
Red Hat - Emerging Technologies