Automated hardware testing improvements

new file mode 100755
index 0000000..7550ea5
@@ -0,0 +1,53 @@
+#!/bin/sh
+#
+# Simple script to show a manual power prompt for when you want to use
+# automated hardware testing with testimage.bbclass but you don't have a
+# web-enabled power strip or similar to do the power on/off/cycle.
+#
+# You can enable it by enabling testimage (see the Yocto Project
+# Development manual "Performing Automated Runtime Testing" section)
+# and setting the following in your local.conf:
+#
+# TEST_POWERCONTROL_CMD = "${COREBASE}/scripts/contrib/dialog-power-control"
+#
+
+PROMPT=""
+while true; do
+    case $1 in
+        on)
+            PROMPT="Please turn device power on";;
+        off)
+            PROMPT="Please turn device power off";;
+        cycle)
+            PROMPT="Please click Done, then turn the device power off then on";;
+        "")
+            break;;
+    esac
+    shift
+done
+
+if [ "$PROMPT" = "" ] ; then
+    echo "ERROR: no power action specified on command line"
+    exit 2
+fi
+
+if [ "`which kdialog 2>/dev/null`" != "" ] ; then
+    DIALOGUTIL="kdialog"
+elif [ "`which zenity 2>/dev/null`" != "" ] ; then
+    DIALOGUTIL="zenity"
+else
+    echo "ERROR: couldn't find program to display a message, install kdialog or zenity"
+    exit 3
+fi
+
+if [ "$DIALOGUTIL" = "kdialog" ] ; then
+    kdialog --yesno "$PROMPT" --title "TestImage Power Control" --yes-label "Done" --no-label "Cancel test"
+elif [ "$DIALOGUTIL" = "zenity" ] ; then
+    zenity --question --text="$PROMPT" --title="TestImage Power Control" --ok-label="Done" --cancel-label="Cancel test"
+fi
+
+if [ "$?" != "0" ] ; then
+    echo "User cancelled test at power prompt"
+    exit 1
+fi
+

index 691c7f6..285c6a9 100644
@@ -209,8 +209,8 @@ def testimage_main(d):
 
     target.deploy()
 
+    target.start()
     try:
-        target.start()
         if export:
             exportTests(d,tc)
         else:

index 02cb370..ff1bb89 100644
@@ -124,6 +124,7 @@ class QemuTarget(BaseTarget):
             self.server_ip = self.runner.server_ip
             self.connection = SSHControl(ip=self.ip, logfile=self.sshlog)
         else:
+            self.stop()
             raise bb.build.FuncFailed("%s - FAILED to start qemu - check the task log and the boot log" % self.pn)
 
     def stop(self):

new file mode 100755
index 0000000..74bd7b7
@@ -0,0 +1,60 @@
+#!/bin/sh
+
+# Copyright (C) 2014 Intel Corporation
+#
+# Released under the MIT license (see COPYING.MIT)
+
+if [ "$1" = "" -o "$1" = "--help" ] ; then
+    echo "Usage: $0 <serial terminal command>"
+    echo
+    echo "Simple script to handle maintaining a terminal for serial devices that"
+    echo "disappear when a device is powered down or reset, such as the USB"
+    echo "serial console on the original BeagleBone (white version)."
+    echo
+    echo "e.g. $0 picocom -b 115200 /dev/ttyUSB0"
+    echo
+    exit
+fi
+
+args="$@"
+DEVICE=""
+while [ "$1" != "" ]; do
+    case "$1" in
+        /dev/*)
+            DEVICE=$1
+            break;;
+    esac
+    shift
+done
+
+if [ "$DEVICE" != "" ] ; then
+    while true; do
+        if [ ! -e $DEVICE ] ; then
+            echo "serdevtry: waiting for $DEVICE to exist..."
+            while [ ! -e $DEVICE ]; do
+                sleep 0.1
+            done
+        fi
+        if [ ! -w $DEVICE ] ; then
+            # Sometimes (presumably because of a race with udev) we get to
+            # the device before its permissions have been set up
+            RETRYNUM=0
+            while [ ! -w $DEVICE ]; do
+                if [ "$RETRYNUM" = "2" ] ; then
+                    echo "Device $DEVICE exists but is not writable!"
+                    exit 1
+                fi
+                RETRYNUM=$((RETRYNUM+1))
+                sleep 0.1
+            done
+        fi
+        $args
+        if [ -e $DEVICE ] ; then
+            break
+        fi
+    done
+else
+    echo "Unable to determine device node from command: $args"
+    exit 1
+fi
+

index d151e24..f2585d4 100644
@@ -20,6 +20,7 @@ import subprocess
 import oeqa.targetcontrol
 import oeqa.utils.sshcontrol as sshcontrol
 import oeqa.utils.commands as commands
+from oeqa.utils import CommandError
 
 from abc import ABCMeta, abstractmethod
 

@@ -94,7 +95,10 @@ class MasterImageHardwareTarget(oeqa.targetcontrol.BaseTarget):
     def power_ctl(self, msg):
         if self.powercontrol_cmd:
             cmd = "%s %s" % (self.powercontrol_cmd, msg)
-            commands.runCmd(cmd, preexec_fn=os.setsid, env=self.origenv)
+            try:
+                commands.runCmd(cmd, assert_error=False, preexec_fn=os.setsid, env=self.origenv)
+            except CommandError as e:
+                bb.fatal(str(e))
 
     def power_cycle(self, conn):
         if self.powercontrol_cmd:

index 8eda927..2260046 100644
@@ -1,3 +1,15 @@
 # Enable other layers to have modules in the same named directory
 from pkgutil import extend_path
 __path__ = extend_path(__path__, __name__)
+
+
+# Borrowed from CalledProcessError
+
+class CommandError(Exception):
+    def __init__(self, retcode, cmd, output = None):
+        self.retcode = retcode
+        self.cmd = cmd
+        self.output = output
+    def __str__(self):
+        return "Command '%s' returned non-zero exit status %d with output: %s" % (self.cmd, self.retcode, self.output)
+

index 9b42620..7637b9d 100644
@@ -1,4 +1,4 @@
-# Copyright (c) 2013 Intel Corporation
+# Copyright (c) 2013-2014 Intel Corporation
 #
 # Released under the MIT license (see COPYING.MIT)
 

@@ -14,6 +14,7 @@ import signal
 import subprocess
 import threading
 import logging
+from oeqa.utils import CommandError
 
 class Command(object):
     def __init__(self, command, bg=False, timeout=None, data=None, **options):

@@ -84,8 +85,8 @@ class Command(object):
 class Result(object):
     pass
 
-def runCmd(command, ignore_status=False, timeout=None, **options):
 
+def runCmd(command, ignore_status=False, timeout=None, assert_error=True, **options):
     result = Result()
 
     cmd = Command(command, timeout=timeout, **options)

@@ -97,7 +98,10 @@ def runCmd(command, ignore_status=False, timeout=None, **options):
     result.pid = cmd.process.pid
 
     if result.status and not ignore_status:
-        raise AssertionError("Command '%s' returned non-zero exit status %d:\n%s" % (command, result.status, result.output))
+        if assert_error:
+            raise AssertionError("Command '%s' returned non-zero exit status %d:\n%s" % (command, result.status, result.output))
+        else:
+            raise CommandError(result.status, command, result.output)
 
     return result
 

index c6fc7d6..d151e24 100644
@@ -87,7 +87,6 @@ class MasterImageHardwareTarget(oeqa.targetcontrol.BaseTarget):
         if self.powercontrol_cmd:
             if self.powercontrol_args:
                 self.powercontrol_cmd = "%s %s" % (self.powercontrol_cmd, self.powercontrol_args)
-            self.power_ctl("on")
         if self.serialcontrol_cmd:
             if self.serialcontrol_args:
                 self.serialcontrol_cmd = "%s %s" % (self.serialcontrol_cmd, self.serialcontrol_args)

@@ -108,13 +107,25 @@ class MasterImageHardwareTarget(oeqa.targetcontrol.BaseTarget):
             if status != 0:
                 bb.error("Failed rebooting target and no power control command defined. You need to manually reset the device.\n%s" % output)
 
+    def _wait_until_booted(self):
+        ''' Waits until the target device has booted (if we have just power cycled it) '''
+        # Subclasses with better methods of determining boot can override this
+        time.sleep(120)
+
     def deploy(self):
-        bb.plain("%s - deploying image on target" % self.pn)
         # base class just sets the ssh log file for us
         super(MasterImageHardwareTarget, self).deploy()
         self.master = sshcontrol.SSHControl(ip=self.ip, logfile=self.sshlog, timeout=600, port=self.port)
         status, output = self.master.run("cat /etc/masterimage")
         if status != 0:
+            # We're not booted into the master image, so try rebooting
+            bb.plain("%s - booting into the master image" % self.pn)
+            self.power_ctl("cycle")
+            self._wait_until_booted()
+
+        bb.plain("%s - deploying image on target" % self.pn)
+        status, output = self.master.run("cat /etc/masterimage")
+        if status != 0:
             bb.fatal("No ssh connectivity or target isn't running a master image.\n%s" % output)
         if self.user_cmds:
             self.deploy_cmds = self.user_cmds.split("\n")

index e8d321f..c6fc7d6 100644
@@ -70,10 +70,12 @@ class MasterImageHardwareTarget(oeqa.targetcontrol.BaseTarget):
         # the command should take as the last argument "off" and "on" and "cycle" (off, on)
         self.powercontrol_cmd = d.getVar("TEST_POWERCONTROL_CMD", True) or None
         self.powercontrol_args = d.getVar("TEST_POWERCONTROL_EXTRA_ARGS") or ""
+
+        self.serialcontrol_cmd = d.getVar("TEST_SERIALCONTROL_CMD", True) or None
+        self.serialcontrol_args = d.getVar("TEST_SERIALCONTROL_EXTRA_ARGS") or ""
+
         self.origenv = os.environ
-        if self.powercontrol_cmd:
-            if self.powercontrol_args:
-                self.powercontrol_cmd = "%s %s" % (self.powercontrol_cmd, self.powercontrol_args)
+        if self.powercontrol_cmd or self.serialcontrol_cmd:
             # the external script for controlling power might use ssh
             # ssh + keys means we need the original user env
             bborigenv = d.getVar("BB_ORIGENV", False) or {}

@@ -81,7 +83,14 @@ class MasterImageHardwareTarget(oeqa.targetcontrol.BaseTarget):
                 val = bborigenv.getVar(key, True)
                 if val is not None:
                     self.origenv[key] = str(val)
+
+        if self.powercontrol_cmd:
+            if self.powercontrol_args:
+                self.powercontrol_cmd = "%s %s" % (self.powercontrol_cmd, self.powercontrol_args)
             self.power_ctl("on")
+        if self.serialcontrol_cmd:
+            if self.serialcontrol_args:
+                self.serialcontrol_cmd = "%s %s" % (self.serialcontrol_cmd, self.serialcontrol_args)
 
     def power_ctl(self, msg):
         if self.powercontrol_cmd:

@@ -172,4 +181,3 @@ class GummibootTarget(MasterImageHardwareTarget):
         self.power_cycle(self.master)
         # there are better ways than a timeout but this should work for now
         time.sleep(120)
-

index 1bd0ab4..e8d321f 100644
@@ -133,9 +133,6 @@ class MasterImageHardwareTarget(oeqa.targetcontrol.BaseTarget):
         bb.plain("%s - reboot/powercycle target" % self.pn)
         self.power_cycle(self.connection)
 
-    def restart(self):
-        pass
-
 
 class GummibootTarget(MasterImageHardwareTarget):
 

index 873a664..02cb370 100644
@@ -70,9 +70,9 @@ class BaseTarget(object):
     def stop(self):
         pass
 
-    @abstractmethod
     def restart(self, params=None):
-        pass
+        self.stop()
+        self.start(params)
 
     def run(self, cmd, timeout=None):
         return self.connection.run(cmd, timeout)

@@ -170,6 +170,3 @@ class SimpleRemoteTarget(BaseTarget):
         self.connection = None
         self.ip = None
         self.server_ip = None
-
-    def restart(self, params=None):
-        pass

index 188c630..1bd0ab4 100644
@@ -1,17 +1,50 @@
+# Copyright (C) 2014 Intel Corporation
+#
+# Released under the MIT license (see COPYING.MIT)
+
+# This module adds support to testimage.bbclass to deploy images and run
+# tests using a "master image" - this is a "known good" image that is
+# installed onto the device as part of initial setup and will be booted into
+# with no interaction; we can then use it to deploy the image to be tested
+# to a second partition before running the tests.
+#
+# For an example master image, see core-image-testmaster
+# (meta/recipes-extended/images/core-image-testmaster.bb)
+
 import os
 import bb
 import traceback
 import time
+import subprocess
 
 import oeqa.targetcontrol
 import oeqa.utils.sshcontrol as sshcontrol
 import oeqa.utils.commands as commands
 
-class GummibootTarget(oeqa.targetcontrol.SimpleRemoteTarget):
+from abc import ABCMeta, abstractmethod
+
+class MasterImageHardwareTarget(oeqa.targetcontrol.BaseTarget):
+
+    __metaclass__ = ABCMeta
 
     def __init__(self, d):
-        # let our base class do the ip thing
-        super(GummibootTarget, self).__init__(d)
+        super(MasterImageHardwareTarget, self).__init__(d)
+
+        # target ip
+        addr = d.getVar("TEST_TARGET_IP", True) or bb.fatal('Please set TEST_TARGET_IP with the IP address of the machine you want to run the tests on.')
+        self.ip = addr.split(":")[0]
+        try:
+            self.port = addr.split(":")[1]
+        except IndexError:
+            self.port = None
+        bb.note("Target IP: %s" % self.ip)
+        self.server_ip = d.getVar("TEST_SERVER_IP", True)
+        if not self.server_ip:
+            try:
+                self.server_ip = subprocess.check_output(['ip', 'route', 'get', self.ip ]).split("\n")[0].split()[-1]
+            except Exception as e:
+                bb.fatal("Failed to determine the host IP address (alternatively you can set TEST_SERVER_IP with the IP address of this machine): %s" % e)
+        bb.note("Server IP: %s" % self.server_ip)
 
         # test rootfs + kernel
         self.rootfs = os.path.join(d.getVar("DEPLOY_DIR_IMAGE", True), d.getVar("IMAGE_LINK_NAME", True) + '.tar.gz')

@@ -26,36 +59,11 @@ class GummibootTarget(oeqa.targetcontrol.SimpleRemoteTarget):
         if not os.path.isfile(self.kernel):
             bb.fatal("No kernel found. Expected path: %s" % self.kernel)
 
-        # if the user knows what he's doing, then by all means...
-        # test-rootfs.tar.gz and test-kernel are hardcoded names in other places
-        # they really have to be used like that in commands though
-        cmds = d.getVar("TEST_DEPLOY_CMDS", True)
-
-        # this the value we need to set in the LoaderEntryOneShot EFI variable
-        # so the system boots the 'test' bootloader label and not the default
-        # The first four bytes are EFI bits, and the rest is an utf-16le string
-        # (EFI vars values need to be utf-16)
-        # $ echo -en "test\0" | iconv -f ascii -t utf-16le | hexdump -C
-        # 00000000  74 00 65 00 73 00 74 00  00 00                    |t.e.s.t...|
-        self.efivarvalue = r'\x07\x00\x00\x00\x74\x00\x65\x00\x73\x00\x74\x00\x00\x00'
-
-        if cmds:
-            self.deploy_cmds = cmds.split("\n")
-        else:
-            self.deploy_cmds = [
-                'mount -L boot /boot',
-                'mkdir -p /mnt/testrootfs',
-                'mount -L testrootfs /mnt/testrootfs',
-                'modprobe efivarfs',
-                'mount -t efivarfs efivarfs /sys/firmware/efi/efivars',
-                'cp ~/test-kernel /boot',
-                'rm -rf /mnt/testrootfs/*',
-                'tar xzvf ~/test-rootfs.tar.gz -C /mnt/testrootfs',
-                'printf "%s" > /sys/firmware/efi/efivars/LoaderEntryOneShot-4a67b082-0a4c-41cf-b6c7-440b29bb8c4f' % self.efivarvalue
-                ]
-
         # master ssh connection
         self.master = None
+        # if the user knows what they are doing, then by all means...
+        self.user_cmds = d.getVar("TEST_DEPLOY_CMDS", True)
+        self.deploy_cmds = None
 
         # this is the name of the command that controls the power for a board
         # e.g: TEST_POWERCONTROL_CMD = "/home/user/myscripts/powercontrol.py ${MACHINE} what-ever-other-args-the-script-wants"

@@ -94,22 +102,67 @@ class GummibootTarget(oeqa.targetcontrol.SimpleRemoteTarget):
     def deploy(self):
         bb.plain("%s - deploying image on target" % self.pn)
         # base class just sets the ssh log file for us
-        super(GummibootTarget, self).deploy()
+        super(MasterImageHardwareTarget, self).deploy()
         self.master = sshcontrol.SSHControl(ip=self.ip, logfile=self.sshlog, timeout=600, port=self.port)
+        status, output = self.master.run("cat /etc/masterimage")
+        if status != 0:
+            bb.fatal("No ssh connectivity or target isn't running a master image.\n%s" % output)
+        if self.user_cmds:
+            self.deploy_cmds = self.user_cmds.split("\n")
         try:
             self._deploy()
         except Exception as e:
             bb.fatal("Failed deploying test image: %s" % e)
 
+    @abstractmethod
     def _deploy(self):
-        # make sure we are in the right image
-        status, output = self.master.run("cat /etc/masterimage")
-        if status != 0:
-            raise Exception("No ssh connectivity or target isn't running a master image.\n%s" % output)
+        pass
+
+    def start(self, params=None):
+        bb.plain("%s - boot test image on target" % self.pn)
+        self._start()
+        # set the ssh object for the target/test image
+        self.connection = sshcontrol.SSHControl(self.ip, logfile=self.sshlog, port=self.port)
+        bb.plain("%s - start running tests" % self.pn)
+
+    @abstractmethod
+    def _start(self):
+        pass
 
+    def stop(self):
+        bb.plain("%s - reboot/powercycle target" % self.pn)
+        self.power_cycle(self.connection)
+
+    def restart(self):
+        pass
+
+
+class GummibootTarget(MasterImageHardwareTarget):
+
+    def __init__(self, d):
+        super(GummibootTarget, self).__init__(d)
+        # this the value we need to set in the LoaderEntryOneShot EFI variable
+        # so the system boots the 'test' bootloader label and not the default
+        # The first four bytes are EFI bits, and the rest is an utf-16le string
+        # (EFI vars values need to be utf-16)
+        # $ echo -en "test\0" | iconv -f ascii -t utf-16le | hexdump -C
+        # 00000000  74 00 65 00 73 00 74 00  00 00                    |t.e.s.t...|
+        self.efivarvalue = r'\x07\x00\x00\x00\x74\x00\x65\x00\x73\x00\x74\x00\x00\x00'
+        self.deploy_cmds = [
+                'mount -L boot /boot',
+                'mkdir -p /mnt/testrootfs',
+                'mount -L testrootfs /mnt/testrootfs',
+                'modprobe efivarfs',
+                'mount -t efivarfs efivarfs /sys/firmware/efi/efivars',
+                'cp ~/test-kernel /boot',
+                'rm -rf /mnt/testrootfs/*',
+                'tar xzvf ~/test-rootfs.tar.gz -C /mnt/testrootfs',
+                'printf "%s" > /sys/firmware/efi/efivars/LoaderEntryOneShot-4a67b082-0a4c-41cf-b6c7-440b29bb8c4f' % self.efivarvalue
+                ]
+
+    def _deploy(self):
         # make sure these aren't mounted
         self.master.run("umount /boot; umount /mnt/testrootfs; umount /sys/firmware/efi/efivars;")
-
         # from now on, every deploy cmd should return 0
         # else an exception will be thrown by sshcontrol
         self.master.ignore_status = False

@@ -118,16 +171,8 @@ class GummibootTarget(oeqa.targetcontrol.SimpleRemoteTarget):
         for cmd in self.deploy_cmds:
             self.master.run(cmd)
 
-
-    def start(self, params=None):
-        bb.plain("%s - boot test image on target" % self.pn)
+    def _start(self, params=None):
         self.power_cycle(self.master)
         # there are better ways than a timeout but this should work for now
         time.sleep(120)
-        # set the ssh object for the target/test image
-        self.connection = sshcontrol.SSHControl(self.ip, logfile=self.sshlog, port=self.port)
-        bb.plain("%s - start running tests" % self.pn)
 
-    def stop(self):
-        bb.plain("%s - reboot/powercycle target" % self.pn)
-        self.power_cycle(self.connection)