autobuild-run improvements

index 7497001..282f15d 100755
@@ -85,12 +85,12 @@ def check_version():
         print "ERROR: script version too old, please upgrade."
         sys.exit(1)
 
-def check_requirements(http_login, http_password):
+def check_requirements(do_send_results=False):
     devnull = open(os.devnull, "w")
     needed_progs = ["make", "git", "gcc", "timeout"]
     missing_requirements = False
 
-    if http_login and http_password:
+    if do_send_results:
         needed_progs.append("curl")
 
     for prog in needed_progs:

@@ -553,10 +553,15 @@ if __name__ == '__main__':
     check_version()
     sysinfo = SystemInfo()
     (ninstances, njobs, http_login, http_password, submitter) = config_get()
-    check_requirements(http_login, http_password)
-    if http_login is None or http_password is None:
+
+    # http_login/password could theoretically be allowed as empty, so check
+    # explicitly on None.
+    do_send_results = (http_login is not None) and (http_password is not None)
+    check_requirements(do_send_results)
+    if not do_send_results:
         print "WARN: due to the lack of http login/password details, results will not be submitted"
         print "WARN: tarballs of results will be kept locally only"
+
     def sigterm_handler(signum, frame):
         os.killpg(os.getpgid(os.getpid()), signal.SIGTERM)
         sys.exit(1)

index 282f15d..66f6e1b 100755
@@ -124,13 +124,16 @@ class SystemInfo:
         self.has_jar = \
           subprocess.call(["which", "jar"], stdout=devnull, stderr=devnull) == 0
 
-# This function fetches the possible toolchain configurations, and
-# returns an array of dictionaries, with for each toolchain:
-#  - url: the URL of the toolchain defconfig
-#  - libc: the C library used by the toolchain
-#  - hostarch: the host architecture for which the toolchain is built
-#  - contents: an array of lines of the defconfig
 def get_toolchain_configs():
+    """Fetch and return the possible toolchain configurations
+
+    This function returns an array of dictionaries, with for each toolchain:
+        - url: the URL of the toolchain defconfig
+        - libc: the C library used by the toolchain
+        - hostarch: the host architecture for which the toolchain is built
+        - contents: an array of lines of the defconfig
+    """
+
     r = urllib2.urlopen('http://autobuild.buildroot.org/toolchains/configs/toolchain-configs.csv')
     l = r.readlines()
     configs = []

@@ -151,10 +154,14 @@ def get_toolchain_configs():
         configs.append(config)
     return configs
 
-# This function prepares the build by making sure all the needed
-# directories are created, cloning or updating the Buildroot source
-# code, and cleaning up remaining stuff from previous builds.
 def prepare_build(instance, log):
+    """Prepare for the next build of the specified instance
+
+    This function prepares the build by making sure all the needed
+    directories are created, cloning or updating the Buildroot source
+    code, and cleaning up remaining stuff from previous builds.
+    """
+
     idir = "instance-%d" % instance
 
     log_write(log, "INFO: preparing a new build")

@@ -204,14 +211,15 @@ def prepare_build(instance, log):
 
     return 0
 
-# This function makes adjustments to the configuration, as well as
-# additional validation to avoid cases that are known not to work.
-#
-# This function returns 'True' when the configuration has been
-# accepted, and 'False' when the configuration has not been accepted
-# (in which case another random configuration will be generated).
-
 def fixup_config(instance, sysinfo):
+    """Finalize the configuration and reject any problematic combinations
+
+    This function returns 'True' when the configuration has been
+    accepted, and 'False' when the configuration has not been accepted because
+    it is known to fail (in which case another random configuration will be
+    generated).
+    """
+
     idir = "instance-%d" % instance
     outputdir = os.path.join(idir, "output")
 

@@ -290,10 +298,14 @@ def fixup_config(instance, sysinfo):
 
     return True
 
-# This function generates the configuration, by choosing a random
-# toolchain configuration and then generating a random selection of
-# packages.
 def gen_config(instance, log, sysinfo):
+    """Generate a new random configuration
+
+    This function generates the configuration, by choosing a random
+    toolchain configuration and then generating a random selection of
+    packages.
+    """
+
     idir = "instance-%d" % instance
     dldir = os.path.join(idir, "dl")
     # We need the absolute path to use with O=, because the relative

@@ -362,8 +374,9 @@ def gen_config(instance, log, sysinfo):
 
     return 0
 
-# Run the build itself
 def do_build(instance, njobs, log):
+    """Run the build itself"""
+
     idir = "instance-%d" % instance
     # We need the absolute path to use with O=, because the relative
     # path to the output directory here is not relative to the

@@ -391,10 +404,14 @@ def do_build(instance, njobs, log):
     log_write(log, "INFO: build successful")
     return 0
 
-# This function prepares the tarball with the results, and either
-# submits them to the official server (if the appropriate credentials
-# are available) or stores them locally as tarballs.
 def send_results(instance, http_login, http_password, submitter, log, result):
+    """Prepare and store/send tarball with results
+
+    This function prepares the tarball with the results, and either
+    submits them to the official server (if the appropriate credentials
+    are available) or stores them locally as tarballs.
+    """
+
     idir = "instance-%d" % instance
     outputdir = os.path.abspath(os.path.join(idir, "output"))
     srcdir = os.path.join(idir, "buildroot")

@@ -462,10 +479,13 @@ def send_results(instance, http_login, http_password, submitter, log, result):
         os.rename(os.path.join(outputdir, "results.tar.bz2"), resultfilename)
         log_write(log, "INFO: results saved as %s" % resultfilename)
 
-# This function implements the main per-instance loop, which prepares
-# the build, generate a configuration, runs the build, and submits the
-# results.
 def run_instance(instance, njobs, http_login, http_password, submitter, sysinfo):
+    """Main per-instance loop
+
+    Prepare the build, generate a configuration, run the build, and submit the
+    results.
+    """
+
     idir = "instance-%d" % instance
 
     # If it doesn't exist, create the instance directory

@@ -488,9 +508,9 @@ def run_instance(instance, njobs, http_login, http_password, submitter, sysinfo)
         ret = do_build(instance, njobs, instance_log)
         send_results(instance, http_login, http_password, submitter, instance_log, ret)
 
-# Function to get the configuration parameters, either from the
-# command line, or through a configuration file.
 def config_get():
+    """Get configuration parameters, either from the command line or the config file"""
+
     epilog_text = """
 Format of the configuration file:
 

index 66f6e1b..589b1d1 100755
@@ -569,7 +569,7 @@ Format of the configuration file:
 
     return (ninstances, njobs, http_login, http_password, submitter)
 
-if __name__ == '__main__':
+def main():
     check_version()
     sysinfo = SystemInfo()
     (ninstances, njobs, http_login, http_password, submitter) = config_get()

@@ -593,3 +593,6 @@ if __name__ == '__main__':
     signal.signal(signal.SIGTERM, sigterm_handler)
     for p in processes:
         p.join()
+
+if __name__ == '__main__':
+    main()

new file mode 100644
index 0000000..58ff1bc
@@ -0,0 +1,23 @@
+Copyright (c) 2012 Vladimir Keleshev, <[email protected]>
+
+Permission is hereby granted, free of charge, to any person
+obtaining a copy of this software and associated
+documentation files (the "Software"), to deal in the Software
+without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense,
+and/or sell copies of the Software, and to permit persons to
+whom the Software is furnished to do so, subject to the
+following conditions:
+
+The above copyright notice and this permission notice shall
+be included in all copies or substantial portions of the
+Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
+KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
+WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

581 lines not on this page · raw message

index 589b1d1..061ae31 100755
@@ -57,6 +57,37 @@
 #   BR2_PACKAGE_CLASSPATH=y, improve the script to detect whether the
 #   necessary host machine requirements are there to build classpath.
 
+"""autobuild-run - run Buildroot autobuilder
+
+Usage: autobuild-run [options]
+
+Options:
+  -h, --help                     show this help message and exit
+  -V, --version                  show version
+  -n, --ninstances NINSTANCES    number of parallel instances [default: 1]
+  -j, --njobs NJOBS              number of parallel jobs [default: 1]
+  -s, --submitter SUBMITTER      name/machine of submitter [default: N/A]
+  --http-login LOGIN             username to send results with
+  --http-password PASSWORD       password to send results with (for security
+                                 reasons it is recommended to define this in the
+                                 config file instead, with user-read permissions
+                                 only)
+  -c, --config CONFIG            path to configuration file
+
+Format of the configuration file:
+
+  All arguments can also be specified in the configuration file specified with
+  --config, using 'key = value' format (not including the leading --
+  characters). For example:
+
+   [main]
+   ninstances = <value>
+   njobs = <value>
+   http-login = <value>
+   http-password = <value>
+   submitter = <value>
+"""
+
 import urllib2
 import csv
 from random import randint

@@ -70,6 +101,7 @@ import sys
 import hashlib
 import argparse
 import ConfigParser
+import docopt
 
 MAX_DURATION = 60 * 60 * 4
 VERSION = 1

@@ -508,75 +540,51 @@ def run_instance(instance, njobs, http_login, http_password, submitter, sysinfo)
         ret = do_build(instance, njobs, instance_log)
         send_results(instance, http_login, http_password, submitter, instance_log, ret)
 
-def config_get():
-    """Get configuration parameters, either from the command line or the config file"""
+# args / config file merging inspired by:
+# https://github.com/docopt/docopt/blob/master/examples/config_file_example.py
 
-    epilog_text = """
-Format of the configuration file:
+def load_ini_config(configfile):
+    """Load configuration from file, returning a docopt-like dictionary"""
 
-   [main]
-   ninstances = <value>
-   njobs = <value>
-   http-login = <value>
-   http-password = <value>
-   submitter = <value>
-"""
+    if not os.path.exists(configfile):
+        print "ERROR: configuration file %s does not exist" % configfile
+        sys.exit(1)
+
+    config = ConfigParser.RawConfigParser()
+    if not config.read(configfile):
+        print "ERROR: cannot parse configuration file %s" % configfile
+        sys.exit(1)
 
-    parser = argparse.ArgumentParser(description='Run Buildroot autobuilder',
-                                     formatter_class=argparse.RawDescriptionHelpFormatter,
-                                     epilog=epilog_text)
-    parser.add_argument("--ninstances", '-n', metavar="NINSTANCES",
-                        help="Number of parallel instances", default=None)
-    parser.add_argument("--njobs", '-j', metavar="NJOBS",
-                        help="Number of parallel jobs", default=None)
-    parser.add_argument("--submitter", '-s', metavar="SUBMITTER",
-                        help="Name/machine of submitter")
-    parser.add_argument("--config", '-c', metavar="CONFIG",
-                        help="Path to configuration file")
-    args = parser.parse_args()
-
-    ninstances = 1
-    njobs = 1
-    http_login = None
-    http_password = None
-    submitter = "N/A"
-
-    if args.config:
-        if not os.path.exists(args.config):
-            print "ERROR: configuration file %s does not exist" % args.config
-            sys.exit(1)
-        parser = ConfigParser.RawConfigParser()
-        if not parser.read([args.config]):
-            print "ERROR: cannot parse configuration file %s" % args.config
-            sys.exit(1)
-        if parser.has_option('main', 'ninstances'):
-            ninstances = parser.getint('main', 'ninstances')
-        if parser.has_option('main', 'njobs'):
-            njobs = parser.getint('main', 'njobs')
-        if parser.has_option('main', 'http-login'):
-            http_login = parser.get('main', 'http-login')
-        if parser.has_option('main', 'http-password'):
-            http_password = parser.get('main', 'http-password')
-        if parser.has_option('main', 'submitter'):
-            submitter = parser.get('main', 'submitter')
-
-    if args.njobs:
-        njobs = int(args.njobs)
-    if args.ninstances:
-        ninstances = int(args.ninstances)
-    if args.submitter:
-        submitter = args.submitter
-
-    return (ninstances, njobs, http_login, http_password, submitter)
+    # Prepend '--' to options specified in the config file, so they can be
+    # merged with those given on the command-line
+    return dict(('--%s' % key, value) for key, value in config.items('main'))
+
+
+def merge(dict_1, dict_2):
+    """Merge two dictionaries.
+
+    Values that evaluate to true take priority over falsy values.
+    `dict_1` takes priority over `dict_2`.
+
+    """
+    return dict((str(key), dict_1.get(key) or dict_2.get(key))
+                for key in set(dict_2) | set(dict_1))
 
 def main():
     check_version()
     sysinfo = SystemInfo()
-    (ninstances, njobs, http_login, http_password, submitter) = config_get()
+
+    args = docopt.docopt(__doc__, version=VERSION)
+
+    if args['--config']:
+        ini_config = load_ini_config(args['--config'])
+        # merge config/args, priority given to config
+        args = merge(ini_config, args)
 
     # http_login/password could theoretically be allowed as empty, so check
     # explicitly on None.
-    do_send_results = (http_login is not None) and (http_password is not None)
+    do_send_results = (args['--http-login'] is not None) \
+                        and (args['--http-password'] is not None)
     check_requirements(do_send_results)
     if not do_send_results:
         print "WARN: due to the lack of http login/password details, results will not be submitted"

@@ -586,8 +594,10 @@ def main():
         os.killpg(os.getpgid(os.getpid()), signal.SIGTERM)
         sys.exit(1)
     processes = []
-    for i in range(0, ninstances):
-        p = Process(target=run_instance, args=(i, njobs, http_login, http_password, submitter, sysinfo))
+    for i in range(0, int(args['--ninstances'])):
+        p = Process(target=run_instance, args=(i, int(args['--njobs']),
+                args['--http-login'], args['--http-password'],
+                args['--submitter'], sysinfo))
         p.start()
         processes.append(p)
     signal.signal(signal.SIGTERM, sigterm_handler)

index 061ae31..40bcaa0 100755
@@ -72,6 +72,9 @@ Options:
                                  reasons it is recommended to define this in the
                                  config file instead, with user-read permissions
                                  only)
+  --make-opts OPTSTRING          string of extra options to pass to Buildroot
+                                 make, such as specific command wrappers
+                                 [default: ]
   -c, --config CONFIG            path to configuration file
 
 Format of the configuration file:

@@ -406,7 +409,7 @@ def gen_config(instance, log, sysinfo):
 
     return 0
 
-def do_build(instance, njobs, log):
+def do_build(instance, njobs, log, make_opts):
     """Run the build itself"""
 
     idir = "instance-%d" % instance

@@ -419,8 +422,10 @@ def do_build(instance, njobs, log):
     srcdir = os.path.join(idir, "buildroot")
     f = open(os.path.join(outputdir, "logfile"), "w+")
     log_write(log, "INFO: build started")
-    ret = subprocess.call(["timeout", str(MAX_DURATION), "make", "O=%s" % outputdir, "-C", srcdir,
-                           "BR2_DL_DIR=%s" % dldir, "BR2_JLEVEL=%s" % njobs], stdout=f, stderr=f)
+    cmd = ["timeout", str(MAX_DURATION), "make", "O=%s" % outputdir,
+            "-C", srcdir, "BR2_DL_DIR=%s" % dldir, "BR2_JLEVEL=%s" % njobs] \
+          + make_opts.split()
+    ret = subprocess.call(cmd, stdout=f, stderr=f)
     # 124 is a special error code that indicates we have reached the
     # timeout
     if ret == 124:

@@ -511,7 +516,8 @@ def send_results(instance, http_login, http_password, submitter, log, result):
         os.rename(os.path.join(outputdir, "results.tar.bz2"), resultfilename)
         log_write(log, "INFO: results saved as %s" % resultfilename)
 
-def run_instance(instance, njobs, http_login, http_password, submitter, sysinfo):
+def run_instance(instance, njobs, http_login, http_password, submitter,
+                 make_opts, sysinfo):
     """Main per-instance loop
 
     Prepare the build, generate a configuration, run the build, and submit the

@@ -537,7 +543,7 @@ def run_instance(instance, njobs, http_login, http_password, submitter, sysinfo)
         if ret != 0:
             continue
 
-        ret = do_build(instance, njobs, instance_log)
+        ret = do_build(instance, njobs, instance_log, make_opts)
         send_results(instance, http_login, http_password, submitter, instance_log, ret)
 
 # args / config file merging inspired by:

@@ -597,7 +603,7 @@ def main():
     for i in range(0, int(args['--ninstances'])):
         p = Process(target=run_instance, args=(i, int(args['--njobs']),
                 args['--http-login'], args['--http-password'],
-                args['--submitter'], sysinfo))
+                args['--submitter'], args['--make-opts'], sysinfo))
         p.start()
         processes.append(p)
     signal.signal(signal.SIGTERM, sigterm_handler)

index 40bcaa0..2f83696 100755
@@ -189,7 +189,7 @@ def get_toolchain_configs():
         configs.append(config)
     return configs
 
-def prepare_build(instance, log):
+def prepare_build(**kwargs):
     """Prepare for the next build of the specified instance
 
     This function prepares the build by making sure all the needed

@@ -197,7 +197,8 @@ def prepare_build(instance, log):
     code, and cleaning up remaining stuff from previous builds.
     """
 
-    idir = "instance-%d" % instance
+    idir = "instance-%d" % kwargs['instance']
+    log = kwargs['log']
 
     log_write(log, "INFO: preparing a new build")
 

@@ -246,7 +247,7 @@ def prepare_build(instance, log):
 
     return 0
 
-def fixup_config(instance, sysinfo):
+def fixup_config(**kwargs):
     """Finalize the configuration and reject any problematic combinations
 
     This function returns 'True' when the configuration has been

@@ -255,9 +256,10 @@ def fixup_config(instance, sysinfo):
     generated).
     """
 
-    idir = "instance-%d" % instance
-    outputdir = os.path.join(idir, "output")
+    idir = "instance-%d" % kwargs['instance']
+    sysinfo = kwargs['sysinfo']
 
+    outputdir = os.path.join(idir, "output")
     with open(os.path.join(outputdir, ".config")) as configf:
         configlines = configf.readlines()
 

@@ -333,7 +335,7 @@ def fixup_config(instance, sysinfo):
 
     return True
 
-def gen_config(instance, log, sysinfo):
+def gen_config(**kwargs):
     """Generate a new random configuration
 
     This function generates the configuration, by choosing a random

@@ -341,7 +343,9 @@ def gen_config(instance, log, sysinfo):
     packages.
     """
 
-    idir = "instance-%d" % instance
+    idir = "instance-%d" % kwargs['instance']
+    log = kwargs['log']
+
     dldir = os.path.join(idir, "dl")
     # We need the absolute path to use with O=, because the relative
     # path to the output directory here is not relative to the

@@ -392,7 +396,7 @@ def gen_config(instance, log, sysinfo):
         if ret != 0:
             log_write(log, "ERROR: cannot generate random configuration")
             return -1
-        if fixup_config(instance, sysinfo):
+        if fixup_config(**kwargs):
             break
 
     ret = subprocess.call(["yes '' 2>/dev/null| make O=%s -C %s oldconfig" % \

@@ -409,10 +413,12 @@ def gen_config(instance, log, sysinfo):
 
     return 0
 
-def do_build(instance, njobs, log, make_opts):
+def do_build(**kwargs):
     """Run the build itself"""
 
-    idir = "instance-%d" % instance
+    idir = "instance-%d" % kwargs['instance']
+    log = kwargs['log']
+
     # We need the absolute path to use with O=, because the relative
     # path to the output directory here is not relative to the
     # Buildroot sources, but to the location of the autobuilder

@@ -423,8 +429,9 @@ def do_build(instance, njobs, log, make_opts):
     f = open(os.path.join(outputdir, "logfile"), "w+")
     log_write(log, "INFO: build started")
     cmd = ["timeout", str(MAX_DURATION), "make", "O=%s" % outputdir,
-            "-C", srcdir, "BR2_DL_DIR=%s" % dldir, "BR2_JLEVEL=%s" % njobs] \
-          + make_opts.split()
+            "-C", srcdir, "BR2_DL_DIR=%s" % dldir,
+            "BR2_JLEVEL=%s" % kwargs['njobs']] \
+          + kwargs['make_opts'].split()
     ret = subprocess.call(cmd, stdout=f, stderr=f)
     # 124 is a special error code that indicates we have reached the
     # timeout

@@ -441,7 +448,7 @@ def do_build(instance, njobs, log, make_opts):
     log_write(log, "INFO: build successful")
     return 0
 
-def send_results(instance, http_login, http_password, submitter, log, result):
+def send_results(result, **kwargs):
     """Prepare and store/send tarball with results
 
     This function prepares the tarball with the results, and either

@@ -449,7 +456,9 @@ def send_results(instance, http_login, http_password, submitter, log, result):
     are available) or stores them locally as tarballs.
     """
 
-    idir = "instance-%d" % instance
+    idir = "instance-%d" % kwargs['instance']
+    log = kwargs['log']
+
     outputdir = os.path.abspath(os.path.join(idir, "output"))
     srcdir = os.path.join(idir, "buildroot")
     resultdir = os.path.join(outputdir, "results")

@@ -484,7 +493,7 @@ def send_results(instance, http_login, http_password, submitter, log, result):
     resultf.close()
 
     with open(os.path.join(resultdir, "submitter"), "w+") as submitterf:
-        submitterf.write(submitter)
+        submitterf.write(kwargs['submitter'])
 
     # Yes, shutil.make_archive() would be nice, but it doesn't exist
     # in Python 2.6.

@@ -494,11 +503,12 @@ def send_results(instance, http_login, http_password, submitter, log, result):
         log_write(log, "ERROR: could not make results tarball")
         sys.exit(1)
 
-    if http_login and http_password:
+    if kwargs['http_login'] and kwargs['http_password']:
         # Submit results. Yes, Python has some HTTP libraries, but
         # none of the ones that are part of the standard library can
         # upload a file without writing dozens of lines of code.
-        ret = subprocess.call(["curl", "-u", "%s:%s" % (http_login, http_password),
+        ret = subprocess.call(["curl", "-u",
+                               "%s:%s" % (kwargs['http_login'], kwargs['http_password']),
                                "-H", "Expect:",
                                "-F", "uploadedfile=@%s" % os.path.join(outputdir, "results.tar.bz2"),
                                "-F", "uploadsubmit=1",

@@ -512,39 +522,40 @@ def send_results(instance, http_login, http_password, submitter, log, result):
         # No http login/password, keep tarballs locally
         with open(os.path.join(outputdir, "results.tar.bz2"), 'rb') as f:
             sha1 = hashlib.sha1(f.read()).hexdigest()
-        resultfilename = "instance-%d-%s.tar.bz2" % (instance, sha1)
+        resultfilename = "instance-%d-%s.tar.bz2" % (kwargs['instance'], sha1)
         os.rename(os.path.join(outputdir, "results.tar.bz2"), resultfilename)
         log_write(log, "INFO: results saved as %s" % resultfilename)
 
-def run_instance(instance, njobs, http_login, http_password, submitter,
-                 make_opts, sysinfo):
+def run_instance(**kwargs):
     """Main per-instance loop
 
     Prepare the build, generate a configuration, run the build, and submit the
     results.
     """
 
-    idir = "instance-%d" % instance
+    idir = "instance-%d" % kwargs['instance']
 
     # If it doesn't exist, create the instance directory
     if not os.path.exists(idir):
         os.mkdir(idir)
 
-    instance_log = open(os.path.join(idir, "instance.log"), "a+")
-    log_write(instance_log, "INFO: instance started")
+    kwargs['log'] = open(os.path.join(idir, "instance.log"), "a+")
+    log_write(kwargs['log'], "INFO: instance started")
+
     while True:
         check_version()
 
-        ret = prepare_build(instance, instance_log)
+        ret = prepare_build(**kwargs)
         if ret != 0:
             continue
 
-        ret = gen_config(instance, instance_log, sysinfo)
+        ret = gen_config(**kwargs)
         if ret != 0:
             continue
 
-        ret = do_build(instance, njobs, instance_log, make_opts)
-        send_results(instance, http_login, http_password, submitter, instance_log, ret)
+        ret = do_build(**kwargs)
+
+        send_results(ret, **kwargs)
 
 # args / config file merging inspired by:
 # https://github.com/docopt/docopt/blob/master/examples/config_file_example.py

@@ -601,9 +612,11 @@ def main():
         sys.exit(1)
     processes = []
     for i in range(0, int(args['--ninstances'])):
-        p = Process(target=run_instance, args=(i, int(args['--njobs']),
-                args['--http-login'], args['--http-password'],
-                args['--submitter'], args['--make-opts'], sysinfo))
+        p = Process(target=run_instance, kwargs={
+            'instance': i, 'njobs': args['--njobs'], 'sysinfo': sysinfo,
+            'http_login': args['--http-login'],
+            'http_password': args['--http-password'],
+            'submitter': args['--submitter'], 'make_opts': args['--make-opts']})
         p.start()
         processes.append(p)
     signal.signal(signal.SIGTERM, sigterm_handler)

index 2f83696..c3abe62 100755
@@ -48,8 +48,6 @@
 #   of just using the last 500 lines of the build log, search the
 #   start of the build of the failing package.
 #
-# - Add LC_ALL=C where appropriate.
-#
 # - Include the config.log file (when it exists) in the tarball for
 #   failed builds when the failure occurs on an autotools package.
 #

@@ -588,6 +586,11 @@ def merge(dict_1, dict_2):
                 for key in set(dict_2) | set(dict_1))
 
 def main():
+
+    # Avoid locale settings of autobuilder machine leaking in, for example
+    # showing error messages in another language.
+    os.environ['LC_ALL'] = 'C'
+
     check_version()
     sysinfo = SystemInfo()
 

index c3abe62..1960f53 100755
@@ -44,10 +44,6 @@
 #
 # TODO:
 #
-# - Improve the logic that generates the 'build-end.log' file. Instead
-#   of just using the last 500 lines of the build log, search the
-#   start of the build of the failing package.
-#
 # - Include the config.log file (when it exists) in the tarball for
 #   failed builds when the failure occurs on an autotools package.
 #

@@ -477,9 +473,52 @@ def send_results(result, **kwargs):
     subprocess.call(["git log master -n 1 --pretty=format:%%H > %s" % \
                      os.path.join(resultdir, "gitid")],
                     shell=True, cwd=srcdir)
-    subprocess.call(["tail -500 %s > %s" % \
-                     (os.path.join(outputdir, "logfile"), os.path.join(resultdir, "build-end.log"))],
-                    shell=True)
+
+    def get_failure_reason():
+        # Output is a tuple (package, version), or None.
+
+        lastlines = subprocess.check_output(["tail", "-n", "3",
+                        os.path.join(outputdir, "logfile")]).splitlines()
+
+        import re
+        regexp = re.compile(r'make: \*\*\* .*/(?:build|toolchain)/([^/]*)/')
+        for line in lastlines:
+            m = regexp.search(line)
+            if m:
+                return m.group(1).rsplit('-', 1)
+
+        # not found
+        return None
+
+    def extract_end_log(resultfile):
+        """Save the last part of the build log, starting from the failed package"""
+
+        def extract_last_500_lines():
+            subprocess.call(["tail -500 %s > %s" % \
+                             (os.path.join(outputdir, "logfile"), resultfile)],
+                            shell=True)
+
+        reason = get_failure_reason()
+        if not reason:
+            extract_last_500_lines()
+        else:
+            import mmap
+            f = open(os.path.join(outputdir, "logfile"), 'r')
+            mf = mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ)
+            mf.seek(0)
+            # Search for first action on the failed package
+            offset = mf.find('>>> %s' % ' '.join(reason))
+            if offset != -1:
+                with open(resultfile, "w") as endlog:
+                    endlog.write(mf[offset:])
+            else:
+                # not found, use last 500 lines as fallback
+                extract_last_500_lines()
+
+            mf.close()
+            f.close()
+
+    extract_end_log(os.path.join(resultdir, "build-end.log"))
 
     resultf = open(os.path.join(resultdir, "status"), "w+")
     if result == 0:

index 1960f53..4803fa1 100755
@@ -44,9 +44,6 @@
 #
 # TODO:
 #
-# - Include the config.log file (when it exists) in the tarball for
-#   failed builds when the failure occurs on an autotools package.
-#
 # - Instead of excluding all configurations that have
 #   BR2_PACKAGE_CLASSPATH=y, improve the script to detect whether the
 #   necessary host machine requirements are there to build classpath.

@@ -520,6 +517,26 @@ def send_results(result, **kwargs):
 
     extract_end_log(os.path.join(resultdir, "build-end.log"))
 
+    def copy_config_log_files():
+        """Recursively copy any config.log files from the failing package"""
+
+        reason = get_failure_reason()
+        if not reason:
+            return
+
+        srcroot = os.path.join(outputdir, "build", '-'.join(reason))
+        destroot = os.path.join(resultdir, '-'.join(reason))
+
+        for root, dirs, files in os.walk(srcroot):
+            dest = os.path.join(destroot, os.path.relpath(root, srcroot))
+
+            for file in files:
+                if file == 'config.log':
+                    os.makedirs(dest)
+                    shutil.copy(os.path.join(root, file), os.path.join(dest, file))
+
+    copy_config_log_files()
+
     resultf = open(os.path.join(resultdir, "status"), "w+")
     if result == 0:
         resultf.write("OK")

index 4803fa1..053b488 100755
@@ -47,6 +47,14 @@
 # - Instead of excluding all configurations that have
 #   BR2_PACKAGE_CLASSPATH=y, improve the script to detect whether the
 #   necessary host machine requirements are there to build classpath.
+# - Integrate method check-requirements with the SysInfo class, distinghuishing
+#   between required and optional dependencies.
+# - Extend required dependencies to subversion, mercurial, cpio, wget, python,
+#   etc.
+# - Detect selection of multiple virtual package providers and don't consider it
+#   a failure
+# - Properly handle kill of main script: stop all running children and do not
+#   print out stacktraces
 
 """autobuild-run - run Buildroot autobuilder