autobuild-run improvements

new file mode 100644
index 0000000..58ff1bc
@@ -0,0 +1,23 @@
+Copyright (c) 2012 Vladimir Keleshev, <[email protected]>
+
+Permission is hereby granted, free of charge, to any person
+obtaining a copy of this software and associated
+documentation files (the "Software"), to deal in the Software
+without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense,
+and/or sell copies of the Software, and to permit persons to
+whom the Software is furnished to do so, subject to the
+following conditions:
+
+The above copyright notice and this permission notice shall
+be included in all copies or substantial portions of the
+Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
+KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
+WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

581 lines not on this page · raw message

index 4d9ef90..b686a78 100755
@@ -57,6 +57,37 @@
 #   BR2_PACKAGE_CLASSPATH=y, improve the script to detect whether the
 #   necessary host machine requirements are there to build classpath.
 
+"""autobuild-run - run Buildroot autobuilder
+
+Usage: autobuild-run [options]
+
+Options:
+  -h, --help                     show this help message and exit
+  -V, --version                  show version
+  -n, --ninstances NINSTANCES    number of parallel instances [default: 1]
+  -j, --njobs NJOBS              number of parallel jobs [default: 1]
+  -s, --submitter SUBMITTER      name/machine of submitter [default: N/A]
+  --http-login LOGIN             username to send results with
+  --http-password PASSWORD       password to send results with (for security
+                                 reasons it is recommended to define this in the
+                                 config file instead, with user-read permissions
+                                 only)
+  -c, --config CONFIG            path to configuration file
+
+Format of the configuration file:
+
+  All arguments can also be specified in the configuration file specified with
+  --config, using 'key = value' format (not including the leading --
+  characters). For example:
+
+   [main]
+   ninstances = <value>
+   njobs = <value>
+   http-login = <value>
+   http-password = <value>
+   submitter = <value>
+"""
+
 import urllib2
 import csv
 from random import randint

@@ -70,6 +101,7 @@ import sys
 import hashlib
 import argparse
 import ConfigParser
+import docopt
 
 MAX_DURATION = 60 * 60 * 4
 VERSION = 1

@@ -522,81 +554,59 @@ def run_instance(instance, njobs, http_login, http_password, submitter, sysinfo)
         ret = do_build(instance, njobs, instance_log)
         send_results(instance, http_login, http_password, submitter, instance_log, ret)
 
-def config_get():
-    """Get configuration parameters, either from the command line or the config file"""
+# args / config file merging inspired by:
+# https://github.com/docopt/docopt/blob/master/examples/config_file_example.py
 
-    epilog_text = """
-Format of the configuration file:
+def load_ini_config(configfile):
+    """Load configuration from file, returning a docopt-like dictionary"""
 
-   [main]
-   ninstances = <value>
-   njobs = <value>
-   http-login = <value>
-   http-password = <value>
-   submitter = <value>
-"""
+    if not os.path.exists(configfile):
+        print "ERROR: configuration file %s does not exist" % configfile
+        sys.exit(1)
+
+    config = ConfigParser.RawConfigParser()
+    if not config.read(configfile):
+        print "ERROR: cannot parse configuration file %s" % configfile
+        sys.exit(1)
+
+    # Prepend '--' to options specified in the config file, so they can be
+    # merged with those given on the command-line
+    return dict(('--%s' % key, value) for key, value in config.items('main'))
 
-    parser = argparse.ArgumentParser(description='Run Buildroot autobuilder',
-                                     formatter_class=argparse.RawDescriptionHelpFormatter,
-                                     epilog=epilog_text)
-    parser.add_argument("--ninstances", '-n', metavar="NINSTANCES",
-                        help="Number of parallel instances", default=None)
-    parser.add_argument("--njobs", '-j', metavar="NJOBS",
-                        help="Number of parallel jobs", default=None)
-    parser.add_argument("--submitter", '-s', metavar="SUBMITTER",
-                        help="Name/machine of submitter")
-    parser.add_argument("--config", '-c', metavar="CONFIG",
-                        help="Path to configuration file")
-    args = parser.parse_args()
-
-    ninstances = 1
-    njobs = 1
-    http_login = None
-    http_password = None
-    submitter = "N/A"
-
-    if args.config:
-        if not os.path.exists(args.config):
-            print "ERROR: configuration file %s does not exist" % args.config
-            sys.exit(1)
-        parser = ConfigParser.RawConfigParser()
-        if not parser.read([args.config]):
-            print "ERROR: cannot parse configuration file %s" % args.config
-            sys.exit(1)
-        if parser.has_option('main', 'ninstances'):
-            ninstances = parser.getint('main', 'ninstances')
-        if parser.has_option('main', 'njobs'):
-            njobs = parser.getint('main', 'njobs')
-        if parser.has_option('main', 'http-login'):
-            http_login = parser.get('main', 'http-login')
-        if parser.has_option('main', 'http-password'):
-            http_password = parser.get('main', 'http-password')
-        if parser.has_option('main', 'submitter'):
-            submitter = parser.get('main', 'submitter')
-
-    if args.njobs:
-        njobs = int(args.njobs)
-    if args.ninstances:
-        ninstances = int(args.ninstances)
-    if args.submitter:
-        submitter = args.submitter
-
-    return (ninstances, njobs, http_login, http_password, submitter)
+
+def merge(dict_1, dict_2):
+    """Merge two dictionaries.
+
+    Values that evaluate to true take priority over falsy values.
+    `dict_1` takes priority over `dict_2`.
+
+    """
+    return dict((str(key), dict_1.get(key) or dict_2.get(key))
+                for key in set(dict_2) | set(dict_1))
 
 def main():
     check_version()
     sysinfo = SystemInfo()
-    (ninstances, njobs, http_login, http_password, submitter) = config_get()
-    check_requirements(http_login, http_password)
-    if http_login is None or http_password is None:
+
+    args = docopt.docopt(__doc__, version=VERSION)
+
+    if args['--config']:
+        ini_config = load_ini_config(args['--config'])
+        # merge config/args, priority given to config
+        args = merge(ini_config, args)
+
+    check_requirements(args['--http-login'], args['--http-password'])
+    if args['--http-login'] is None or args['--http-password'] is None:
         print "WARN: due to the lack of http login/password details, results will not be submitted"
         print "WARN: tarballs of results will be kept locally only"
     def sigterm_handler(signum, frame):
         os.killpg(os.getpgid(os.getpid()), signal.SIGTERM)
         sys.exit(1)
     processes = []
-    for i in range(0, ninstances):
-        p = Process(target=run_instance, args=(i, njobs, http_login, http_password, submitter, sysinfo))
+    for i in range(0, int(args['--ninstances'])):
+        p = Process(target=run_instance, args=(i, int(args['--njobs']),
+                args['--http-login'], args['--http-password'],
+                args['--submitter'], sysinfo))
         p.start()
         processes.append(p)
     signal.signal(signal.SIGTERM, sigterm_handler)

index b686a78..8b0091d 100755
@@ -72,6 +72,9 @@ Options:
                                  reasons it is recommended to define this in the
                                  config file instead, with user-read permissions
                                  only)
+  --make-opts OPTSTRING          string of extra options to pass to Buildroot
+                                 make, such as specific command wrappers
+                                 [default: ]
   -c, --config CONFIG            path to configuration file
 
 Format of the configuration file:

@@ -420,7 +423,7 @@ def gen_config(instance, log, sysinfo):
 
     return 0
 
-def do_build(instance, njobs, log):
+def do_build(instance, njobs, log, make_opts):
     """Run the build itself"""
 
     idir = "instance-%d" % instance

@@ -433,8 +436,10 @@ def do_build(instance, njobs, log):
     srcdir = os.path.join(idir, "buildroot")
     f = open(os.path.join(outputdir, "logfile"), "w+")
     log_write(log, "INFO: build started")
-    ret = subprocess.call(["timeout", str(MAX_DURATION), "make", "O=%s" % outputdir, "-C", srcdir,
-                           "BR2_DL_DIR=%s" % dldir, "BR2_JLEVEL=%s" % njobs], stdout=f, stderr=f)
+    cmd = ["timeout", str(MAX_DURATION), "make", "O=%s" % outputdir,
+            "-C", srcdir, "BR2_DL_DIR=%s" % dldir, "BR2_JLEVEL=%s" % njobs] \
+          + make_opts.split()
+    ret = subprocess.call(cmd, stdout=f, stderr=f)
     # 124 is a special error code that indicates we have reached the
     # timeout
     if ret == 124:

@@ -525,7 +530,8 @@ def send_results(instance, http_login, http_password, submitter, log, result):
         os.rename(os.path.join(outputdir, "results.tar.bz2"), resultfilename)
         log_write(log, "INFO: results saved as %s" % resultfilename)
 
-def run_instance(instance, njobs, http_login, http_password, submitter, sysinfo):
+def run_instance(instance, njobs, http_login, http_password, submitter,
+                 make_opts, sysinfo):
     """Main per-instance loop
 
     Prepare the build, generate a configuration, run the build, and submit the

@@ -551,7 +557,7 @@ def run_instance(instance, njobs, http_login, http_password, submitter, sysinfo)
         if ret != 0:
             continue
 
-        ret = do_build(instance, njobs, instance_log)
+        ret = do_build(instance, njobs, instance_log, make_opts)
         send_results(instance, http_login, http_password, submitter, instance_log, ret)
 
 # args / config file merging inspired by:

@@ -606,7 +612,7 @@ def main():
     for i in range(0, int(args['--ninstances'])):
         p = Process(target=run_instance, args=(i, int(args['--njobs']),
                 args['--http-login'], args['--http-password'],
-                args['--submitter'], sysinfo))
+                args['--submitter'], args['--make-opts'], sysinfo))
         p.start()
         processes.append(p)
     signal.signal(signal.SIGTERM, sigterm_handler)

index 8b0091d..4fc883a 100755
@@ -189,7 +189,7 @@ def get_toolchain_configs():
         configs.append(config)
     return configs
 
-def prepare_build(instance, log):
+def prepare_build(**kwargs):
     """Prepare for the next build of the specified instance
 
     This function prepares the build by making sure all the needed

@@ -197,7 +197,8 @@ def prepare_build(instance, log):
     code, and cleaning up remaining stuff from previous builds.
     """
 
-    idir = "instance-%d" % instance
+    idir = "instance-%d" % kwargs['instance']
+    log = kwargs['log']
 
     log_write(log, "INFO: preparing a new build")
 

@@ -246,7 +247,7 @@ def prepare_build(instance, log):
 
     return 0
 
-def fixup_config(instance, sysinfo):
+def fixup_config(**kwargs):
     """Finalize the configuration and reject any problematic combinations
 
     This function returns 'True' when the configuration has been

@@ -255,9 +256,10 @@ def fixup_config(instance, sysinfo):
     generated).
     """
 
-    idir = "instance-%d" % instance
-    outputdir = os.path.join(idir, "output")
+    idir = "instance-%d" % kwargs['instance']
+    sysinfo = kwargs['sysinfo']
 
+    outputdir = os.path.join(idir, "output")
     with open(os.path.join(outputdir, ".config")) as configf:
         configlines = configf.readlines()
 

@@ -347,7 +349,7 @@ def fixup_config(instance, sysinfo):
 
     return True
 
-def gen_config(instance, log, sysinfo):
+def gen_config(**kwargs):
     """Generate a new random configuration
 
     This function generates the configuration, by choosing a random

@@ -355,7 +357,9 @@ def gen_config(instance, log, sysinfo):
     packages.
     """
 
-    idir = "instance-%d" % instance
+    idir = "instance-%d" % kwargs['instance']
+    log = kwargs['log']
+
     dldir = os.path.join(idir, "dl")
     # We need the absolute path to use with O=, because the relative
     # path to the output directory here is not relative to the

@@ -406,7 +410,7 @@ def gen_config(instance, log, sysinfo):
         if ret != 0:
             log_write(log, "ERROR: cannot generate random configuration")
             return -1
-        if fixup_config(instance, sysinfo):
+        if fixup_config(**kwargs):
             break
 
     ret = subprocess.call(["yes '' 2>/dev/null| make O=%s -C %s oldconfig" % \

@@ -423,10 +427,12 @@ def gen_config(instance, log, sysinfo):
 
     return 0
 
-def do_build(instance, njobs, log, make_opts):
+def do_build(**kwargs):
     """Run the build itself"""
 
-    idir = "instance-%d" % instance
+    idir = "instance-%d" % kwargs['instance']
+    log = kwargs['log']
+
     # We need the absolute path to use with O=, because the relative
     # path to the output directory here is not relative to the
     # Buildroot sources, but to the location of the autobuilder

@@ -437,8 +443,9 @@ def do_build(instance, njobs, log, make_opts):
     f = open(os.path.join(outputdir, "logfile"), "w+")
     log_write(log, "INFO: build started")
     cmd = ["timeout", str(MAX_DURATION), "make", "O=%s" % outputdir,
-            "-C", srcdir, "BR2_DL_DIR=%s" % dldir, "BR2_JLEVEL=%s" % njobs] \
-          + make_opts.split()
+            "-C", srcdir, "BR2_DL_DIR=%s" % dldir,
+            "BR2_JLEVEL=%s" % kwargs['njobs']] \
+          + kwargs['make_opts'].split()
     ret = subprocess.call(cmd, stdout=f, stderr=f)
     # 124 is a special error code that indicates we have reached the
     # timeout

@@ -455,7 +462,7 @@ def do_build(instance, njobs, log, make_opts):
     log_write(log, "INFO: build successful")
     return 0
 
-def send_results(instance, http_login, http_password, submitter, log, result):
+def send_results(result, **kwargs):
     """Prepare and store/send tarball with results
 
     This function prepares the tarball with the results, and either

@@ -463,7 +470,9 @@ def send_results(instance, http_login, http_password, submitter, log, result):
     are available) or stores them locally as tarballs.
     """
 
-    idir = "instance-%d" % instance
+    idir = "instance-%d" % kwargs['instance']
+    log = kwargs['log']
+
     outputdir = os.path.abspath(os.path.join(idir, "output"))
     srcdir = os.path.join(idir, "buildroot")
     resultdir = os.path.join(outputdir, "results")

@@ -498,7 +507,7 @@ def send_results(instance, http_login, http_password, submitter, log, result):
     resultf.close()
 
     with open(os.path.join(resultdir, "submitter"), "w+") as submitterf:
-        submitterf.write(submitter)
+        submitterf.write(kwargs['submitter'])
 
     # Yes, shutil.make_archive() would be nice, but it doesn't exist
     # in Python 2.6.

@@ -508,11 +517,12 @@ def send_results(instance, http_login, http_password, submitter, log, result):
         log_write(log, "ERROR: could not make results tarball")
         sys.exit(1)
 
-    if http_login and http_password:
+    if kwargs['http_login'] and kwargs['http_password']:
         # Submit results. Yes, Python has some HTTP libraries, but
         # none of the ones that are part of the standard library can
         # upload a file without writing dozens of lines of code.
-        ret = subprocess.call(["curl", "-u", "%s:%s" % (http_login, http_password),
+        ret = subprocess.call(["curl", "-u",
+                               "%s:%s" % (kwargs['http_login'], kwargs['http_password']),
                                "-H", "Expect:",
                                "-F", "uploadedfile=@%s" % os.path.join(outputdir, "results.tar.bz2"),
                                "-F", "uploadsubmit=1",

@@ -526,39 +536,40 @@ def send_results(instance, http_login, http_password, submitter, log, result):
         # No http login/password, keep tarballs locally
         with open(os.path.join(outputdir, "results.tar.bz2"), 'rb') as f:
             sha1 = hashlib.sha1(f.read()).hexdigest()
-        resultfilename = "instance-%d-%s.tar.bz2" % (instance, sha1)
+        resultfilename = "instance-%d-%s.tar.bz2" % (kwargs['instance'], sha1)
         os.rename(os.path.join(outputdir, "results.tar.bz2"), resultfilename)
         log_write(log, "INFO: results saved as %s" % resultfilename)
 
-def run_instance(instance, njobs, http_login, http_password, submitter,
-                 make_opts, sysinfo):
+def run_instance(**kwargs):
     """Main per-instance loop
 
     Prepare the build, generate a configuration, run the build, and submit the
     results.
     """
 
-    idir = "instance-%d" % instance
+    idir = "instance-%d" % kwargs['instance']
 
     # If it doesn't exist, create the instance directory
     if not os.path.exists(idir):
         os.mkdir(idir)
 
-    instance_log = open(os.path.join(idir, "instance.log"), "a+")
-    log_write(instance_log, "INFO: instance started")
+    kwargs['log'] = open(os.path.join(idir, "instance.log"), "a+")
+    log_write(kwargs['log'], "INFO: instance started")
+
     while True:
         check_version()
 
-        ret = prepare_build(instance, instance_log)
+        ret = prepare_build(**kwargs)
         if ret != 0:
             continue
 
-        ret = gen_config(instance, instance_log, sysinfo)
+        ret = gen_config(**kwargs)
         if ret != 0:
             continue
 
-        ret = do_build(instance, njobs, instance_log, make_opts)
-        send_results(instance, http_login, http_password, submitter, instance_log, ret)
+        ret = do_build(**kwargs)
+
+        send_results(ret, **kwargs)
 
 # args / config file merging inspired by:
 # https://github.com/docopt/docopt/blob/master/examples/config_file_example.py

@@ -610,9 +621,15 @@ def main():
         sys.exit(1)
     processes = []
     for i in range(0, int(args['--ninstances'])):
-        p = Process(target=run_instance, args=(i, int(args['--njobs']),
-                args['--http-login'], args['--http-password'],
-                args['--submitter'], args['--make-opts'], sysinfo))
+        p = Process(target=run_instance, kwargs=dict(
+                instance = i,
+                njobs = args['--njobs'],
+                sysinfo = sysinfo,
+                http_login = args['--http-login'],
+                http_password = args['--http-password'],
+                submitter = args['--submitter'],
+                make_opts = args['--make-opts']
+            ))
         p.start()
         processes.append(p)
     signal.signal(signal.SIGTERM, sigterm_handler)

index 4fc883a..ab53d3f 100755
@@ -120,12 +120,12 @@ def check_version():
         print "ERROR: script version too old, please upgrade."
         sys.exit(1)
 
-def check_requirements(http_login, http_password):
+def check_requirements(upload=False):
     devnull = open(os.devnull, "w")
     needed_progs = ["make", "git", "gcc", "timeout"]
     missing_requirements = False
 
-    if http_login and http_password:
+    if upload:
         needed_progs.append("curl")
 
     for prog in needed_progs:

@@ -517,7 +517,7 @@ def send_results(result, **kwargs):
         log_write(log, "ERROR: could not make results tarball")
         sys.exit(1)
 
-    if kwargs['http_login'] and kwargs['http_password']:
+    if kwargs['upload']:
         # Submit results. Yes, Python has some HTTP libraries, but
         # none of the ones that are part of the standard library can
         # upload a file without writing dozens of lines of code.

@@ -612,10 +612,15 @@ def main():
         # merge config/args, priority given to config
         args = merge(ini_config, args)
 
-    check_requirements(args['--http-login'], args['--http-password'])
-    if args['--http-login'] is None or args['--http-password'] is None:
+    # http_login/password could theoretically be allowed as empty, so check
+    # explicitly on None.
+    upload = (args['--http-login'] is not None) \
+                and (args['--http-password'] is not None)
+    check_requirements(upload)
+    if not upload:
         print "WARN: due to the lack of http login/password details, results will not be submitted"
         print "WARN: tarballs of results will be kept locally only"
+
     def sigterm_handler(signum, frame):
         os.killpg(os.getpgid(os.getpid()), signal.SIGTERM)
         sys.exit(1)

@@ -628,7 +633,8 @@ def main():
                 http_login = args['--http-login'],
                 http_password = args['--http-password'],
                 submitter = args['--submitter'],
-                make_opts = args['--make-opts']
+                make_opts = args['--make-opts'],
+                upload = upload
             ))
         p.start()
         processes.append(p)

index ab53d3f..d085029 100755
@@ -48,8 +48,6 @@
 #   of just using the last 500 lines of the build log, search the
 #   start of the build of the failing package.
 #
-# - Add LC_ALL=C where appropriate.
-#
 # - Include the config.log file (when it exists) in the tarball for
 #   failed builds when the failure occurs on an autotools package.
 #

@@ -602,6 +600,11 @@ def merge(dict_1, dict_2):
                 for key in set(dict_2) | set(dict_1))
 
 def main():
+
+    # Avoid locale settings of autobuilder machine leaking in, for example
+    # showing error messages in another language.
+    os.environ['LC_ALL'] = 'C'
+
     check_version()
     sysinfo = SystemInfo()
 

index d085029..32466e8 100755
@@ -44,10 +44,6 @@
 #
 # TODO:
 #
-# - Improve the logic that generates the 'build-end.log' file. Instead
-#   of just using the last 500 lines of the build log, search the
-#   start of the build of the failing package.
-#
 # - Include the config.log file (when it exists) in the tarball for
 #   failed builds when the failure occurs on an autotools package.
 #

@@ -491,9 +487,52 @@ def send_results(result, **kwargs):
     subprocess.call(["git log master -n 1 --pretty=format:%%H > %s" % \
                      os.path.join(resultdir, "gitid")],
                     shell=True, cwd=srcdir)
-    subprocess.call(["tail -500 %s > %s" % \
-                     (os.path.join(outputdir, "logfile"), os.path.join(resultdir, "build-end.log"))],
-                    shell=True)
+
+    def get_failure_reason():
+        # Output is a tuple (package, version), or None.
+
+        lastlines = subprocess.check_output(["tail", "-n", "3",
+                        os.path.join(outputdir, "logfile")]).splitlines()
+
+        import re
+        regexp = re.compile(r'make: \*\*\* .*/(?:build|toolchain)/([^/]*)/')
+        for line in lastlines:
+            m = regexp.search(line)
+            if m:
+                return m.group(1).rsplit('-', 1)
+
+        # not found
+        return None
+
+    def extract_end_log(resultfile):
+        """Save the last part of the build log, starting from the failed package"""
+
+        def extract_last_500_lines():
+            subprocess.call(["tail -500 %s > %s" % \
+                             (os.path.join(outputdir, "logfile"), resultfile)],
+                            shell=True)
+
+        reason = get_failure_reason()
+        if not reason:
+            extract_last_500_lines()
+        else:
+            import mmap
+            f = open(os.path.join(outputdir, "logfile"), 'r')
+            mf = mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ)
+            mf.seek(0)
+            # Search for first action on the failed package
+            offset = mf.find('>>> %s' % ' '.join(reason))
+            if offset != -1:
+                with open(resultfile, "w") as endlog:
+                    endlog.write(mf[offset:])
+            else:
+                # not found, use last 500 lines as fallback
+                extract_last_500_lines()
+
+            mf.close()
+            f.close()
+
+    extract_end_log(os.path.join(resultdir, "build-end.log"))
 
     resultf = open(os.path.join(resultdir, "status"), "w+")
     if result == 0:

index 32466e8..198c7bc 100755
@@ -44,9 +44,6 @@
 #
 # TODO:
 #
-# - Include the config.log file (when it exists) in the tarball for
-#   failed builds when the failure occurs on an autotools package.
-#
 # - Instead of excluding all configurations that have
 #   BR2_PACKAGE_CLASSPATH=y, improve the script to detect whether the
 #   necessary host machine requirements are there to build classpath.

@@ -534,6 +531,26 @@ def send_results(result, **kwargs):
 
     extract_end_log(os.path.join(resultdir, "build-end.log"))
 
+    def copy_config_log_files():
+        """Recursively copy any config.log files from the failing package"""
+
+        reason = get_failure_reason()
+        if not reason:
+            return
+
+        srcroot = os.path.join(outputdir, "build", '-'.join(reason))
+        destroot = os.path.join(resultdir, '-'.join(reason))
+
+        for root, dirs, files in os.walk(srcroot):
+            dest = os.path.join(destroot, os.path.relpath(root, srcroot))
+
+            for file in files:
+                if file == 'config.log':
+                    os.makedirs(dest)
+                    shutil.copy(os.path.join(root, file), os.path.join(dest, file))
+
+    copy_config_log_files()
+
     resultf = open(os.path.join(resultdir, "status"), "w+")
     if result == 0:
         resultf.write("OK")

index 198c7bc..237d443 100755
@@ -47,6 +47,17 @@
 # - Instead of excluding all configurations that have
 #   BR2_PACKAGE_CLASSPATH=y, improve the script to detect whether the
 #   necessary host machine requirements are there to build classpath.
+# - Integrate method check-requirements with the SysInfo class, distinghuishing
+#   between required and optional dependencies.
+# - Extend required dependencies to subversion, mercurial, cpio, wget, python,
+#   etc.
+# - Detect selection of multiple virtual package providers and don't consider it
+#   a failure
+# - Properly handle kill of main script: stop all running children and do not
+#   print out stacktraces
+# - Fix problem in removal of output directory: sometimes this fails with
+#   message 'directory not empty' which suggests that someone is writing to the
+#   directory at the time of removal.
 
 """autobuild-run - run Buildroot autobuilder
 

index 237d443..3c448bd 100755
@@ -97,9 +97,10 @@ import urllib2
 import csv
 from random import randint
 import subprocess
-from multiprocessing import Process
+import multiprocessing
 import signal
 import os
+import errno
 import shutil
 from time import localtime, strftime
 import sys

@@ -444,11 +445,16 @@ def do_build(**kwargs):
     srcdir = os.path.join(idir, "buildroot")
     f = open(os.path.join(outputdir, "logfile"), "w+")
     log_write(log, "INFO: build started")
+
     cmd = ["timeout", str(MAX_DURATION), "make", "O=%s" % outputdir,
             "-C", srcdir, "BR2_DL_DIR=%s" % dldir,
             "BR2_JLEVEL=%s" % kwargs['njobs']] \
           + kwargs['make_opts'].split()
-    ret = subprocess.call(cmd, stdout=f, stderr=f)
+    sub = subprocess.Popen(cmd, stdout=f, stderr=f)
+    kwargs['buildpid'][kwargs['instance']] = sub.pid
+    ret = sub.wait()
+    kwargs['buildpid'][kwargs['instance']] = 0
+
     # 124 is a special error code that indicates we have reached the
     # timeout
     if ret == 124:

@@ -692,8 +698,32 @@ def main():
         print "WARN: tarballs of results will be kept locally only"
 
     def sigterm_handler(signum, frame):
+        """Kill all children"""
+
+        # uninstall signal handler to prevent being called for all subprocesses
+        signal.signal(signal.SIGTERM, signal.SIG_DFL)
+
+        # stop all instances to prevent new children to be spawned
+        for p in processes:
+            p.terminate()
+
+        # kill build processes started with timeout (that puts its children
+        # explicitly in a separate process group)
+        for pid in buildpid:
+            if pid == 0:
+                continue
+            try:
+                os.kill(pid, signal.SIGTERM)
+            except OSError as e:
+                if e.errno != errno.ESRCH: # No such process, ignore
+                    raise
+
+        # kill any remaining children in our process group
         os.killpg(os.getpgid(os.getpid()), signal.SIGTERM)
+
         sys.exit(1)
+
+    buildpid = multiprocessing.Array('i', int(args['--ninstances']))
     processes = []
     for i in range(0, int(args['--ninstances'])):
         p = Process(target=run_instance, kwargs=dict(

@@ -704,11 +734,14 @@ def main():
                 http_password = args['--http-password'],
                 submitter = args['--submitter'],
                 make_opts = args['--make-opts'],
-                upload = upload
+                upload = upload,
+                buildpid = buildpid
             ))
         p.start()
         processes.append(p)
+
     signal.signal(signal.SIGTERM, sigterm_handler)
+
     for p in processes:
         p.join()
 

index 3c448bd..c8ce918 100755
@@ -701,6 +701,7 @@ def main():
         """Kill all children"""
 
         # uninstall signal handler to prevent being called for all subprocesses
+        signal.signal(signal.SIGINT, signal.SIG_IGN)
         signal.signal(signal.SIGTERM, signal.SIG_DFL)
 
         # stop all instances to prevent new children to be spawned

@@ -740,6 +741,7 @@ def main():
         p.start()
         processes.append(p)
 
+    signal.signal(signal.SIGINT, sigterm_handler)
     signal.signal(signal.SIGTERM, sigterm_handler)
 
     for p in processes: