scripts/autobuild-run: make the HTTP URL really configurable
PATCH 1/2 scripts/autobuild-run: make the HTTP URL really configurable
The --http-url option allowed to customize the URL at which build results are submitted. However, there were two other places in the script where autobuild.buildroot.org was hardcoded: when checking the script version, and when getting the list of branches to build.
This commit changes the --http-url to be the base URL of the autobuild server, and it is used everywhere instead of hardcoding autobuild.buildroot.org.
Note: users of autobuild-run that were passing a custom --http-url option such as http://foo.com/submit/ should change it to just
http://foo.com/
--- scripts/autobuild-run | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-)
scripts/autobuild-run +10 -10
@@ -66,7 +66,7 @@ defaults = { '--make-opts': '', '--nice': 0, '--pid-file': '/tmp/buildroot-autobuild.pid', - '--http-url': 'http://autobuild.buildroot.org/submit/', + '--http-url': 'http://autobuild.buildroot.org/', '--toolchains-csv': 'support/config-fragments/autobuild/toolchain-configs.csv', '--repo': 'https://github.com/buildroot/buildroot.git', }
@@ -145,6 +145,7 @@ from distutils.version import StrictVersion import platform from threading import Thread, Event import datetime +import urlparse if sys.hexversion >= 0x3000000: import configparser
@@ -178,23 +179,23 @@ def log_write(logf, msg): logf.write("[%s] %s\n" % (strftime("%a, %d %b %Y %H:%M:%S", localtime()), msg)) logf.flush() -def check_version(): - with urlopen_closing('http://autobuild.buildroot.org/version') as r: +def check_version(**kwargs): + with urlopen_closing(urlparse.urljoin(kwargs['http_url'], 'version')) as r: version = int(decode_bytes(r.readline()).strip()) if version > VERSION: print("ERROR: script version too old, please upgrade.") sys.exit(1) -def get_branch(): +def get_branch(**kwargs): """Returns the branch that should be built. It fetches a CSV file from - autobuild.buildroot.org that provides the list of branches to test + the autobuild server that provides the list of branches to test (first field) and their weight (second field). We build a list of branches, where the branch name is repeated N times, with N being the weight of the branch. We then pick a random branch in this list. This way, branches with a higher weight are more likely to be selected. """ - with urlopen_closing('http://autobuild.buildroot.org/branches') as r: + with urlopen_closing(urlparse.urljoin(kwargs['http_url'], 'branches')) as r: csv_branches = r.readlines() branches = [] for branch in csv.reader(csv_branches):
@@ -314,7 +315,7 @@ def prepare_build(**kwargs): os.path.relpath(f, dldir)) os.remove(f) - branch = get_branch() + branch = get_branch(**kwargs) log_write(log, "INFO: testing branch '%s'" % branch) # Clone Buildroot. This only happens if the source directory
@@ -679,7 +680,7 @@ def send_results(result, **kwargs): "-H", "Expect:", "-F", "uploadedfile=@%s" % os.path.join(outputdir, "results.tar.bz2"), "-F", "uploadsubmit=1", - kwargs['http_url']], + urlparse.urljoin(kwargs['http_url'], 'submit/')], stdout=log, stderr=log) if ret != 0: log_write(log, "INFO: results could not be submitted, %d" % ret)
@@ -713,7 +714,7 @@ def run_instance(**kwargs): log_write(kwargs['log'], "INFO: instance started") while True: - check_version() + check_version(**kwargs) ret = prepare_build(**kwargs) if ret != 0:
@@ -774,7 +775,6 @@ def main(): # showing error messages in another language. os.environ['LC_ALL'] = 'C' - check_version() sysinfo = SystemInfo() args = docopt.docopt(doc, version=VERSION)
Signature · 1 line
2.21.0
PATCH 2/2 scripts/autobuild-run: support changing repo
The current logic in prepare_build() assumes that the "origin" repo never changes. However, if one regularly changes his autobuild-run configuration, switching being repository, this is not true. Currently, it requires manually wiping out the Buildroot clone in every autobuild instance when changing the repository to pull from.
So instead, use:
git fetch <repo> <branch>
git checkout FETCH_HEAD
which will easily allow switching from one repo to the other.
--- scripts/autobuild-run | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)
scripts/autobuild-run +2 -2
@@ -330,12 +330,12 @@ def prepare_build(**kwargs): # Update the Buildroot sources. abssrcdir = os.path.abspath(srcdir) - ret = subprocess.call(["git", "fetch", "origin"], cwd=abssrcdir, stdout=log, stderr=log) + ret = subprocess.call(["git", "fetch", kwargs['repo'], branch], cwd=abssrcdir, stdout=log, stderr=log) if ret != 0: log_write(log, "ERROR: could not fetch Buildroot sources") return -1 - ret = subprocess.call(["git", "checkout", "--detach", "origin/%s" % branch], cwd=abssrcdir, stdout=log, stderr=log) + ret = subprocess.call(["git", "checkout", "FETCH_HEAD"], cwd=abssrcdir, stdout=log, stderr=log) if ret != 0: log_write(log, "ERROR: could not check out Buildroot sources") return -1
Signature · 1 line
2.21.0
PATCH 1/2 scripts/autobuild-run: make the HTTP URL really configurable
In reply to Thomas PetazzoniThomas, All,
On 2019-06-17 20:22 +0200, Thomas Petazzoni spake thusly:
Quoted 9 lines
The --http-url option allowed to customize the URL at which build results are submitted. However, there were two other places in the script where autobuild.buildroot.org was hardcoded: when checking the script version, and when getting the list of branches to build.
This commit changes the --http-url to be the base URL of the autobuild server, and it is used everywhere instead of hardcoding autobuild.buildroot.org.
Quoted 6 lines
Note: users of autobuild-run that were passing a custom --http-url option such as http://foo.com/submit/ should change it to just
http://foo.com/Signed-off-by Thomas Petazzoni <[email protected]>---
[--SNIP--]
scripts/autobuild-run def log_write(logf, msg): · line 182
Quoted 7 lines
scripts/autobuild-run +1
-2@@ -178,23 +179,23 @@ def log_write(logf, msg): logf.write("[%s] %s\n" % (strftime("%a, %d %b %Y %H:%M:%S", localtime()), msg)) logf.flush() -def check_version(): - with urlopen_closing('http://autobuild.buildroot.org/version') as r: +def check_version(**kwargs):
There is a current work by Atharva to actually remove usage of kwargs elsewhere in the code.
I think it would be nice to have a new object that contains the server config:
class ServerConfig():
def __init__(self, base_url):
self.base_url = base_url
def branches_url(self):
return urlparse.urljoin(self.base_url, 'branches')
def submit_url(self):
return urlparse.urljoin(self.base_url, 'submit')
And then in main (or elsewhere it is meaningful):
server_cfg = ServerConfig(args['http_url'])
and pass that object to run_instance().
Then the series from Atharva can more easily account for this new variable.
Regards,
Yann E. MORIN.
Quoted 64 lines
scripts/autobuild-run +7
-7+ with urlopen_closing(urlparse.urljoin(kwargs['http_url'], 'version')) as r: version = int(decode_bytes(r.readline()).strip()) if version > VERSION: print("ERROR: script version too old, please upgrade.") sys.exit(1) -def get_branch(): +def get_branch(**kwargs): """Returns the branch that should be built. It fetches a CSV file from - autobuild.buildroot.org that provides the list of branches to test + the autobuild server that provides the list of branches to test (first field) and their weight (second field). We build a list of branches, where the branch name is repeated N times, with N being the weight of the branch. We then pick a random branch in this list. This way, branches with a higher weight are more likely to be selected. """ - with urlopen_closing('http://autobuild.buildroot.org/branches') as r: + with urlopen_closing(urlparse.urljoin(kwargs['http_url'], 'branches')) as r: csv_branches = r.readlines() branches = [] for branch in csv.reader(csv_branches):@@ -314,7 +315,7 @@ def prepare_build(**kwargs): os.path.relpath(f, dldir)) os.remove(f) - branch = get_branch() + branch = get_branch(**kwargs) log_write(log, "INFO: testing branch '%s'" % branch) # Clone Buildroot. This only happens if the source directory@@ -679,7 +680,7 @@ def send_results(result, **kwargs): "-H", "Expect:", "-F", "uploadedfile=@%s" % os.path.join(outputdir, "results.tar.bz2"), "-F", "uploadsubmit=1", - kwargs['http_url']], + urlparse.urljoin(kwargs['http_url'], 'submit/')], stdout=log, stderr=log) if ret != 0: log_write(log, "INFO: results could not be submitted, %d" % ret)@@ -713,7 +714,7 @@ def run_instance(**kwargs): log_write(kwargs['log'], "INFO: instance started") while True: - check_version() + check_version(**kwargs) ret = prepare_build(**kwargs) if ret != 0:@@ -774,7 +775,6 @@ def main(): # showing error messages in another language. os.environ['LC_ALL'] = 'C' - check_version() sysinfo = SystemInfo() args = docopt.docopt(doc, version=VERSION)Signature · 6 lines
2.21.0 _______________________________________________ buildroot mailing list buildroot at busybox.net http://lists.busybox.net/mailman/listinfo/buildroot
Signature · 6 lines
.-----------------.--------------------.------------------.--------------------. | Yann E. MORIN | Real-Time Embedded | /"\ ASCII RIBBON | Erics' conspiracy: | | +33 662 376 056 | Software Designer | \ / CAMPAIGN | ___ | | +33 561 099 427 `------------.-------: X AGAINST | \e/ There is no | | http://ymorin.is-a-geek.org/ | _/*\_ | / \ HTML MAIL | v conspiracy. | '------------------------------^-------^------------------^--------------------'