buildman: A few more tweaks
cover buildman: A few more tweaks
A few problems have been found with recent buildman changes. This series tidies these up:
- Fixes lack of build output with -w
- Fix operation of -A
- Handle the updated kernel.org toolchain version
Simon Glass (5):
buildman: Fix test for new 9.2 kernel
buildman: Correct operation of -A flag
buildman: Make sure that -o is given with -w
buildman: Use out-env for environment output
buildman: Write output files when using -w
tools/buildman/README | 3 ++- tools/buildman/builder.py | 5 +++++ tools/buildman/builderthread.py | 33 ++++++++++++++++++--------------- tools/buildman/cmdline.py | 3 +-- tools/buildman/control.py | 6 +++++- tools/buildman/func_test.py | 17 +++++++++++++++++ tools/buildman/test.py | 2 +- 7 files changed, 49 insertions(+), 20 deletions(-)
Signature · 1 line
2.26.1.301.g55bc3eb7cb9-goog
PATCH 1/5 buildman: Fix test for new 9.2 kernel
The naming is slightly different on kernel.org now. Update the regex so that the test still passes.
---
tools/buildman/test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
tools/buildman/test.py +1 -1
@@ -583,7 +583,7 @@ class TestBuild(unittest.TestCase): url = self.toolchains.LocateArchUrl('arm') self.assertRegexpMatches(url, 'https://www.kernel.org/pub/tools/' 'crosstool/files/bin/x86_64/.*/' - 'x86_64-gcc-.*-nolibc_arm-.*linux-gnueabi.tar.xz') + 'x86_64-gcc-.*-nolibc[-_]arm-.*linux-gnueabi.tar.xz') def testGetEnvArgs(self): """Test the GetEnvArgs() function"""
Signature · 1 line
2.26.1.301.g55bc3eb7cb9-goog
PATCH 1/5 buildman: Fix test for new 9.2 kernel
The naming is slightly different on kernel.org now. Update the regex so that the test still passes.
---
tools/buildman/test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
Applied to u-boot-dm, thanks!
PATCH 2/5 buildman: Correct operation of -A flag
In reply to Simon GlassThis was broken when -a was removed and unfortunately there are no tests for this. Fix it.
---
tools/buildman/control.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
tools/buildman/control.py +1 -1
@@ -207,7 +207,7 @@ def DoBuildman(options, args, toolchains=None, make_func=None, boards=None, sys.exit(col.Color(col.RED, 'No matching boards found')) if options.print_prefix: - err = ShowToolchainInfo(boards, toolchains) + err = ShowToolchainPrefix(boards, toolchains) if err: sys.exit(col.Color(col.RED, err)) return 0
Signature · 1 line
2.26.1.301.g55bc3eb7cb9-goog
PATCH 2/5 buildman: Correct operation of -A flag
This was broken when -a was removed and unfortunately there are no tests for this. Fix it.
---
tools/buildman/control.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
Applied to u-boot-dm, thanks!
PATCH 3/5 buildman: Make sure that -o is given with -w
In reply to Simon GlassIt is a bad idea to use the default output directory ('..') with -w since it does a build in that directory and writes various files these.
Require that -o is given to avoid this problem.
---
tools/buildman/README | 3 ++- tools/buildman/cmdline.py | 3 +-- tools/buildman/control.py | 4 ++++ tools/buildman/func_test.py | 6 ++++++ 4 files changed, 13 insertions(+), 3 deletions(-)
tools/buildman/README +2 -1
@@ -1091,7 +1091,8 @@ the -w option, for example: buildman -o /tmp/build --board sandbox -w -This will write the full build into /tmp/build including object files. +This will write the full build into /tmp/build including object files. You must +specify the output directory with -o when using -w. Other options
tools/buildman/cmdline.py +1 -2
@@ -76,8 +76,7 @@ def ParseArgs(): default=False, help="Do a dry run (describe actions, but do nothing)") parser.add_option('-N', '--no-subdirs', action='store_true', dest='no_subdirs', default=False, help="Don't create subdirectories when building current source for a single board") - parser.add_option('-o', '--output-dir', type='string', - dest='output_dir', default='..', + parser.add_option('-o', '--output-dir', type='string', dest='output_dir', help='Directory where all builds happen and buildman has its workspace (default is ../)') parser.add_option('-O', '--override-toolchain', type='string', help="Override host toochain to use for sandbox (e.g. 'clang-7')")
tools/buildman/control.py +4 -0
@@ -175,6 +175,10 @@ def DoBuildman(options, args, toolchains=None, make_func=None, boards=None, if options.incremental: print(col.Color(col.RED, 'Warning: -I has been removed. See documentation')) + if not options.output_dir: + if options.work_in_output: + sys.exit(col.Color(col.RED, '-w requires that you specify -o')) + options.output_dir = '..' # Work out what subset of the boards we are building if not boards:
tools/buildman/func_test.py +6 -0
@@ -569,3 +569,9 @@ class TestFunctional(unittest.TestCase): self._RunControl('-b', self._test_branch, '-o', self._output_dir, '-w', clean_dir=False, boards=board_list) self.assertIn("single commit", str(e.exception)) + + board_list = board.Boards() + board_list.AddBoard(board.Board(*boards[0])) + with self.assertRaises(SystemExit) as e: + self._RunControl('-w', clean_dir=False) + self.assertIn("specify -o", str(e.exception))
Signature · 1 line
2.26.1.301.g55bc3eb7cb9-goog
PATCH 3/5 buildman: Make sure that -o is given with -w
It is a bad idea to use the default output directory ('..') with -w since it does a build in that directory and writes various files these.
Require that -o is given to avoid this problem.
---
tools/buildman/README | 3 ++- tools/buildman/cmdline.py | 3 +-- tools/buildman/control.py | 4 ++++ tools/buildman/func_test.py | 6 ++++++ 4 files changed, 13 insertions(+), 3 deletions(-)
Applied to u-boot-dm, thanks!
PATCH 4/5 buildman: Use out-env for environment output
In reply to Simon GlassAt present the environment used by U-Boot is written to the 'env' directory. This is fine when the output directory is not the same as the source directory, but when it is (as with -w) it conflicts with the source directory of the same name.
Rename 'env' to 'out-env' to fix this.
---
tools/buildman/builderthread.py | 2 +- tools/buildman/func_test.py | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-)
tools/buildman/builderthread.py +1 -1
@@ -333,7 +333,7 @@ class BuilderThread(threading.Thread): # Write out the image and function size information and an objdump env = result.toolchain.MakeEnvironment(self.builder.full_path) - with open(os.path.join(build_dir, 'env'), 'w') as fd: + with open(os.path.join(build_dir, 'out-env'), 'w') as fd: for var in sorted(env.keys()): print('%s="%s"' % (var, env[var]), file=fd) lines = []
tools/buildman/func_test.py +7 -0
@@ -546,6 +546,13 @@ class TestFunctional(unittest.TestCase): self.assertEqual(self._builder.count, self._total_builds) self.assertEqual(self._builder.fail, 0) + def testEnvironment(self): + """Test that the done and environment files are written to out-env""" + self._RunControl('-o', self._output_dir) + board0_dir = os.path.join(self._output_dir, 'current', 'board0') + self.assertTrue(os.path.exists(os.path.join(board0_dir, 'done'))) + self.assertTrue(os.path.exists(os.path.join(board0_dir, 'out-env'))) + def testWorkInOutput(self): """Test the -w option which should write directly to the output dir""" board_list = board.Boards()
Signature · 1 line
2.26.1.301.g55bc3eb7cb9-goog
PATCH 4/5 buildman: Use out-env for environment output
At present the environment used by U-Boot is written to the 'env' directory. This is fine when the output directory is not the same as the source directory, but when it is (as with -w) it conflicts with the source directory of the same name.
Rename 'env' to 'out-env' to fix this.
---
tools/buildman/builderthread.py | 2 +- tools/buildman/func_test.py | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-)
Applied to u-boot-dm, thanks!
PATCH 5/5 buildman: Write output files when using -w
In reply to Simon GlassAt present buildman does not write its own output files (err, done, the environment) when using -w. However this is useful for when the build is run with -s to check it.
In fact ProduceResultSummary() reads the result from those files rather than using the 'result' info directly. So ProcessResult() does not work with -w at present. It does not print any output.
Fix this by writing output files even when -w is used.
---
tools/buildman/builder.py | 5 +++++ tools/buildman/builderthread.py | 31 +++++++++++++++++-------------- tools/buildman/func_test.py | 4 ++++ 3 files changed, 26 insertions(+), 14 deletions(-)
tools/buildman/builder.py +5 -0
@@ -479,6 +479,9 @@ class Builder: Args: commit_upto: Commit number to use (0..self.count-1) """ + if self.work_in_output: + return self._working_dir + commit_dir = None if self.commits: commit = self.commits[commit_upto]
@@ -502,6 +505,8 @@ class Builder: target: Target name """ output_dir = self._GetOutputDir(commit_upto) + if self.work_in_output: + return output_dir return os.path.join(output_dir, target) def GetDoneFile(self, commit_upto, target):
tools/buildman/builderthread.py +17 -14
@@ -280,8 +280,6 @@ class BuilderThread(threading.Thread): work_in_output: Use the output directory as the work directory and don't write to a separate output directory. """ - if work_in_output: - return # Fatal error if result.return_code < 0: return
@@ -379,7 +377,8 @@ class BuilderThread(threading.Thread): capture_stderr=True, cwd=result.out_dir, raise_on_error=False, env=env) ubootenv = os.path.join(result.out_dir, 'uboot.env') - self.CopyFiles(result.out_dir, build_dir, '', ['uboot.env']) + if not work_in_output: + self.CopyFiles(result.out_dir, build_dir, '', ['uboot.env']) # Write out the image sizes file. This is similar to the output # of binutil's 'size' utility, but it omits the header line and
@@ -391,17 +390,21 @@ class BuilderThread(threading.Thread): with open(sizes, 'w') as fd: print('\n'.join(lines), file=fd) - # Write out the configuration files, with a special case for SPL - for dirname in ['', 'spl', 'tpl']: - self.CopyFiles(result.out_dir, build_dir, dirname, ['u-boot.cfg', - 'spl/u-boot-spl.cfg', 'tpl/u-boot-tpl.cfg', '.config', - 'include/autoconf.mk', 'include/generated/autoconf.h']) - - # Now write the actual build output - if keep_outputs: - self.CopyFiles(result.out_dir, build_dir, '', ['u-boot*', '*.bin', - '*.map', '*.img', 'MLO', 'SPL', 'include/autoconf.mk', - 'spl/u-boot-spl*']) + if not work_in_output: + # Write out the configuration files, with a special case for SPL + for dirname in ['', 'spl', 'tpl']: + self.CopyFiles( + result.out_dir, build_dir, dirname, + ['u-boot.cfg', 'spl/u-boot-spl.cfg', 'tpl/u-boot-tpl.cfg', + '.config', 'include/autoconf.mk', + 'include/generated/autoconf.h']) + + # Now write the actual build output + if keep_outputs: + self.CopyFiles( + result.out_dir, build_dir, '', + ['u-boot*', '*.bin', '*.map', '*.img', 'MLO', 'SPL', + 'include/autoconf.mk', 'spl/u-boot-spl*']) def CopyFiles(self, out_dir, build_dir, dirname, patterns): """Copy files from the build directory to the output.
tools/buildman/func_test.py +4 -0
@@ -561,6 +561,10 @@ class TestFunctional(unittest.TestCase): boards=board_list) self.assertTrue( os.path.exists(os.path.join(self._output_dir, 'u-boot'))) + self.assertTrue( + os.path.exists(os.path.join(self._output_dir, 'done'))) + self.assertTrue( + os.path.exists(os.path.join(self._output_dir, 'out-env'))) def testWorkInOutputFail(self): """Test the -w option failures"""
Signature · 1 line
2.26.1.301.g55bc3eb7cb9-goog
PATCH 5/5 buildman: Write output files when using -w
At present buildman does not write its own output files (err, done, the environment) when using -w. However this is useful for when the build is run with -s to check it.
In fact ProduceResultSummary() reads the result from those files rather than using the 'result' info directly. So ProcessResult() does not work with -w at present. It does not print any output.
Fix this by writing output files even when -w is used.
---
tools/buildman/builder.py | 5 +++++ tools/buildman/builderthread.py | 31 +++++++++++++++++-------------- tools/buildman/func_test.py | 4 ++++ 3 files changed, 26 insertions(+), 14 deletions(-)
Applied to u-boot-dm, thanks!