scripts/autobuild-run: fix and default to python3
PATCH v2 1/2 scripts/autobuild-run: fix and default to python3
--- scripts/autobuild-run | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)
scripts/autobuild-run +2 -2
@@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (C) 2014 by Thomas Petazzoni <[email protected]> #
@@ -794,7 +794,7 @@ class Builder: while True: try: self.run_one_build() - except URLError, e: + except URLError as e: sleep(30) # args / config file merging inspired by:
Signature · 6 lines
2.25.1 _______________________________________________ buildroot mailing list [email protected] https://lists.buildroot.org/mailman/listinfo/buildroot
PATCH v2 2/2 scripts/autobuild-run: add --no-toolchains-csv option and drop docopt
Since docopt is unmaintained and less flexible than argparse lets drop it and migrate to argparse.
---
Changes v1 -> v2:
- fix configparser
--- scripts/autobuild-run | 201 ++++++------- scripts/docopt.LICENSE-MIT | 23 -- scripts/docopt.py | 581 ------------------------------------- 3 files changed, 92 insertions(+), 713 deletions(-) delete mode 100644 scripts/docopt.LICENSE-MIT delete mode 100644 scripts/docopt.py
scripts/autobuild-run +92 -109
@@ -57,56 +57,7 @@ from __future__ import print_function -# Don't tell docopt about the defaults, as it would not allow the following -# priority hierarchy for arguments: command-line > config file > defaults -defaults = { - '--ninstances': '1', - '--njobs': '1', - '--submitter': 'N/A', - '--make-opts': '', - '--nice': 0, - '--pid-file': '/tmp/buildroot-autobuild.pid', - '--http-url': 'http://autobuild.buildroot.org/', - '--toolchains-csv': 'support/config-fragments/autobuild/toolchain-configs.csv', - '--repo': 'https://github.com/buildroot/buildroot.git', -} - -doc = """autobuild-run - run Buildroot autobuilder - -Usage: autobuild-run [options] - -Options: - -h, --help show this help message and exit - -V, --version show version - -n, --ninstances NINSTANCES number of parallel instances - Defaults to %(--ninstances)s. - -j, --njobs NJOBS number of parallel jobs - Defaults to %(--njobs)s. - --nice N Niceness, positive number - Defaults to %(--nice)s. - -s, --submitter SUBMITTER name/machine of submitter - Defaults to %(--submitter)s. - --http-url URL URL of resource to submit your results. - Defaults to %(--http-url)s. - --http-login LOGIN username to send results with - Not set by default. - --http-password PASSWORD password to send results with (for security - reasons it is recommended to define this in the - config file instead, with user-read permissions - only) - Not set by default. - --make-opts OPTSTRING string of extra options to pass to Buildroot - make, such as specific command wrappers - Empty by default. - --pid-file PATH path to a file where to store the PID - Defaults to %(--pid-file)s. - -c, --config CONFIG path to configuration file - Not set by default. - -d, --debug Send log output to stdout instead of log file - --toolchains-csv CSVFILE Toolchain configuration file - -r, --repo URL URL of Buildroot repository to clone - Defaults to %(--repo)s - +epilog = """ Format of the configuration file: All arguments can also be specified in the configuration file specified with
@@ -120,20 +71,18 @@ Format of the configuration file: http-login = <value> http-password = <value> submitter = <value> - - -""" % defaults - -__doc__ = doc + no-toolchains-csv +""" import contextlib import csv -import docopt +import argparse import errno import hashlib import mmap import multiprocessing import os +import pathlib from random import randint import re import shutil
@@ -423,6 +372,8 @@ class Builder: if not os.path.isabs(toolchains_csv): toolchains_csv = os.path.join(self.srcdir, toolchains_csv) args.extend(["--toolchains-csv", toolchains_csv]) + else: + args.extend(["--no-toolchains-csv"]) ret = subprocess.call(args, stdout=devnull, stderr=self.log) return ret
@@ -797,35 +748,20 @@ class Builder: except URLError as e: sleep(30) -# args / config file merging inspired by: -# https://github.com/docopt/docopt/blob/master/examples/config_file_example.py - -def load_ini_config(configfile): - """Load configuration from file, returning a docopt-like dictionary""" - - if not os.path.exists(configfile): - print("ERROR: configuration file %s does not exist" % configfile) - sys.exit(1) - - config = configparser.RawConfigParser() - if not config.read(configfile): - print("ERROR: cannot parse configuration file %s" % configfile) - sys.exit(1) - - # Prepend '--' to options specified in the config file, so they can be - # merged with those given on the command-line - return dict(('--%s' % key, value) for key, value in config.items('main')) - - -def merge(dict_1, dict_2): - """Merge two dictionaries. - - Values that evaluate to true take priority over falsy values. - `dict_1` takes priority over `dict_2`. - - """ - return dict((str(key), dict_1.get(key) or dict_2.get(key)) - for key in set(dict_2) | set(dict_1)) +class Formatter(argparse.ArgumentDefaultsHelpFormatter, argparse.RawDescriptionHelpFormatter): + pass + +class LoadConfigFile(argparse.Action): + def __call__(self, parser, namespace, values, option_string=None): + config = configparser.RawConfigParser(allow_no_value=True) + config.read_file(values) + for k, v in config.items('main'): + key = k.replace("-", "_") + value = v + if key.startswith("no_") and value is None: + key = key[3:] + value = False + setattr(namespace, key, value) def main():
@@ -835,24 +771,71 @@ def main(): sysinfo = SystemInfo() - args = docopt.docopt(doc, version=VERSION) - - if args['--config']: - ini_config = load_ini_config(args['--config']) - # merge config/args, priority given to args - args = merge(args, ini_config) - - # load in defaults at lowest priority - args = merge(args, defaults) + parser = argparse.ArgumentParser(description="Buildroot autobuilder", + epilog=epilog, + formatter_class=Formatter) + parser.add_argument("--ninstances", "-n", + help="number of parallel instances", + type=int, default=1) + parser.add_argument("--njobs", "-j", + help="number of parallel jobs", + type=int, default=1) + parser.add_argument("--nice", + help="Niceness, positive number", + type=int, default=0) + parser.add_argument("--submitter", "-s", + help="name/machine of submitter", + type=str, default='N/A') + parser.add_argument("--http-url", + help="URL of resource to submit your results.", + type=str, default='http://autobuild.buildroot.org/') + parser.add_argument("--http-login", + help="username to send results with", + type=str) + parser.add_argument("--http-password", + help="password to send results with (for security " + "reasons it is recommended to define this in the " + "config file instead, with user-read permissions " + "only)", + type=str) + parser.add_argument("--make-opts", + help="string of extra options to pass to Buildroot " + "make, such as specific command wrappers", + type=str) + parser.add_argument("--pid-file", + help="path to a file where to store the PID", + type=pathlib.Path, default=pathlib.Path("/tmp/buildroot-autobuild.pid")) + parser.add_argument("--config", "-c", + help="path to configuration file", + type=open, action=LoadConfigFile) + parser.add_argument("--debug", "-d", + help="Send log output to stdout instead of log file", + type=str) + toolchains_csv = parser.add_mutually_exclusive_group(required=False) + toolchains_csv.add_argument("--toolchains-csv", + dest="toolchains_csv", + help="Toolchain configuration file", + type=pathlib.Path) + toolchains_csv.add_argument("--no-toolchains-csv", + dest="toolchains_csv", + help="Generate random toolchain configuration", + action='store_false') + parser.set_defaults(toolchains_csv=pathlib.Path("support/config-fragments/autobuild/toolchain-configs.csv")) + parser.add_argument("--repo", "-r", + help="URL of Buildroot repository to clone", + type=str, + default="https://github.com/buildroot/buildroot.git") + + args = parser.parse_args() # Save our PID very early, so we can be stopped - with open(args['--pid-file'], "w+") as pidf: + with args.pid_file.open("w+") as pidf: pidf.write("%d" % os.getpid()) # http_login/password could theoretically be allowed as empty, so check # explicitly on None. - upload = (args['--http-login'] is not None) \ - and (args['--http-password'] is not None) + upload = (args.http_login is not None) \ + and (args.http_password is not None) if upload: sysinfo.needed_progs.append("curl") else:
@@ -894,24 +877,24 @@ def main(): sys.exit(1) - buildpid = multiprocessing.Array('i', int(args['--ninstances'])) + buildpid = multiprocessing.Array('i', int(args.ninstances)) processes = [] - for i in range(0, int(args['--ninstances'])): + for i in range(0, int(args.ninstances)): builder = Builder( instance = i, - njobs = args['--njobs'], + njobs = args.njobs, sysinfo = sysinfo, - http_url = args['--http-url'], - http_login = args['--http-login'], - http_password = args['--http-password'], - submitter = args['--submitter'], - make_opts = (args['--make-opts'] or ''), - nice = (args['--nice'] or 0), - toolchains_csv = args['--toolchains-csv'], - repo = args['--repo'], + http_url = args.http_url, + http_login = args.http_login, + http_password = args.http_password, + submitter = args.submitter, + make_opts = (args.make_opts or ''), + nice = (args.nice or 0), + toolchains_csv = args.toolchains_csv, + repo = args.repo, upload = upload, buildpid = buildpid, - debug = args['--debug']) + debug = args.debug) p = multiprocessing.Process(target=builder.run_instance) p.start() processes.append(p)
scripts/docopt.LICENSE-MIT deleted +0 -23
@@ -1,23 +0,0 @@ -Copyright (c) 2012 Vladimir Keleshev, <[email protected]> - -Permission is hereby granted, free of charge, to any person -obtaining a copy of this software and associated -documentation files (the "Software"), to deal in the Software -without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, -and/or sell copies of the Software, and to permit persons to -whom the Software is furnished to do so, subject to the -following conditions: - -The above copyright notice and this permission notice shall -be included in all copies or substantial portions of the -Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY -KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE -WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR -PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR -COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR -OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE -SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
scripts/docopt.py deleted +0 -581
581 lines not on this page · raw message
Signature · 6 lines
2.25.1 _______________________________________________ buildroot mailing list [email protected] https://lists.buildroot.org/mailman/listinfo/buildroot
Re: [Buildroot] [PATCH v2 2/2] scripts/autobuild-run: add --no-toolchains-csv option and drop docopt
Quoted 4 lines · On Sat, 9 Apr 2022 22:32:27 -0600 James Hilliard <[email protected]>…
Since docopt is unmaintained and less flexible than argparse lets drop it and migrate to argparse.
Signed-off-by James Hilliard <[email protected]>
Thanks! I think it would have been great to split the conversion from docopt to argparse from the addition of no-toolchains-csv. Regardless of that, I wanted to apply, but spotted something incorrect in the help text, showing that the argparse logic is probably wrong. Look:
--toolchains-csv TOOLCHAINS_CSV
Toolchain configuration file (default: support/config-fragments/autobuild/toolchain- configs.csv)
--no-toolchains-csv Generate random toolchain configuration (default: support/config-
fragments/autobuild/toolchain-configs.csv)
See how --no-toolchains-csv option has a default value... that doesn't make any sense?
scripts/autobuild-run def main(): · line 823
Quoted 10 lines
scripts/autobuild-run +10
-0+ toolchains_csv = parser.add_mutually_exclusive_group(required=False) + toolchains_csv.add_argument("--toolchains-csv", + dest="toolchains_csv", + help="Toolchain configuration file", + type=pathlib.Path) + toolchains_csv.add_argument("--no-toolchains-csv", + dest="toolchains_csv", + help="Generate random toolchain configuration", + action='store_false') + parser.set_defaults(toolchains_csv=pathlib.Path("support/config-fragments/autobuild/toolchain-configs.csv"))
I suppose this is what makes it have this incorrect "default value".
Thomas
Signature · 7 lines
Thomas Petazzoni, co-owner and CEO, Bootlin Embedded Linux and Kernel engineering and training https://bootlin.com _______________________________________________ buildroot mailing list [email protected] https://lists.buildroot.org/mailman/listinfo/buildroot
Re: [Buildroot] [PATCH v2 2/2] scripts/autobuild-run: add --no-toolchains-csv option and drop docopt
Quoted 14 lines · Thomas Petazzoni
Quoted 4 lines · On Sat, 9 Apr 2022 22:32:27 -0600 James Hilliard <[email protected]>…
Since docopt is unmaintained and less flexible than argparse lets drop it and migrate to argparse.
Signed-off-by James Hilliard <[email protected]>Thanks! I think it would have been great to split the conversion from docopt to argparse from the addition of no-toolchains-csv. Regardless of that, I wanted to apply, but spotted something incorrect in the help text, showing that the argparse logic is probably wrong. Look:
Quoted 8 lines
--toolchains-csv
TOOLCHAINS_CSV
Toolchain configuration file (default: support/config-fragments/autobuild/toolchain- configs.csv)
--no-toolchains-csv Generate random toolchain configuration (default: support/config-
fragments/autobuild/toolchain-configs.csv)See how --no-toolchains-csv option has a default value... that doesn't make any sense?
It's due to both those options mapping to the same variable.
Quoted 12 lines
scripts/autobuild-run def main(): · line 823
Quoted 11 lines
scripts/autobuild-run +10
-0+ toolchains_csv = parser.add_mutually_exclusive_group(required=False) + toolchains_csv.add_argument("--toolchains-csv", + dest="toolchains_csv", + help="Toolchain configuration file", + type=pathlib.Path) + toolchains_csv.add_argument("--no-toolchains-csv", + dest="toolchains_csv", + help="Generate random toolchain configuration", + action='store_false') + parser.set_defaults(toolchains_csv=pathlib.Path("support/config-fragments/autobuild/toolchain-configs.csv"))
Quoted 1 line
I suppose this is what makes it have this incorrect "default value".
Yeah, should be fixed here:
https://patchwork.ozlabs.org/project/buildroot/patch/[email protected]/
Quoted 6 lines
Thomas
--
Thomas Petazzoni, co-owner and CEO, Bootlin Embedded Linux and Kernel engineering and training
https://bootlin.com
_______________________________________________
buildroot mailing list
[email protected]
https://lists.buildroot.org/mailman/listinfo/buildroot
Quoted 4 lines · On Sat, 9 Apr 2022 22:32:26 -0600 James Hilliard <[email protected]>…
Signed-off-by James Hilliard <[email protected]>--- scripts/autobuild-run | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)
Applied to buildroot-test, thanks.
Thomas
Signature · 7 lines
Thomas Petazzoni, CTO, Bootlin Embedded Linux and Kernel engineering https://bootlin.com _______________________________________________ buildroot mailing list [email protected] https://lists.buildroot.org/mailman/listinfo/buildroot