Avoid shell use
cover Avoid shell use
This series removes shell use from a few backend paths where Buildbot already has structured command arguments or generated file contents.
The changes are intended as hardening/cleanup with no intentional behaviour change.
Testing:
* Metadata writers:
- qemuarm-oecore build wrote layerinfo.json and bbsetup-overrides.json via StringDownload
- both files parsed correctly and preserved repo, branch and revision values literally
- whitebox comparison showed byte-for-byte identical output for qemuarm-oecore, a-quick, docs and default-repo fallback cases.
* getproperties.py argv change:
- live qemuarm-oecore build reached and passed "Load build revisions"
- the step set 15 properties
- Buildbot log showed direct argv execution:
argv: [b'.../getproperties.py', b'.../build', b'qemuarm-oecore']
- direct replay on a preserved qemuarm-oecore build tree matched the old shell-string output exactly.
* Disk-space check:
- live builds passed the canStartBuild disk gate and started normally
- local comparison showed the old findmnt --df fifth field matched the new findmnt -o AVAIL output for ., /tmp and the worker tree
Anders Heimer (3):
steps: avoid shell when writing repo metadata
builders: pass getproperties command as argv list
builders: avoid shell pipeline for disk space check
builders.py | 16 +++++++++++++--- steps/writebbsetupjson.py | 19 +++++++------------ steps/writelayerinfo.py | 19 +++++++------------ 3 files changed, 27 insertions(+), 27 deletions(-)
PATCH 1/3 steps: avoid shell when writing repo metadata
Use Buildbot StringDownload for repo metadata JSON instead of shell printf. This transfers scheduler-supplied repo values as file content without shell interpretation.
AI-Generated: Claude Opus 4.6 <[email protected]>
--- steps/writebbsetupjson.py | 19 +++++++------------ steps/writelayerinfo.py | 19 +++++++------------ 2 files changed, 14 insertions(+), 24 deletions(-)
steps/writebbsetupjson.py +7 -12
@@ -5,9 +5,8 @@ # from twisted.internet import defer -from buildbot.process import buildstep +from buildbot.plugins import steps import json -import os from yoctoabb import config
@@ -15,11 +14,11 @@ mappings = { "oecore" : "openembedded-core", } -class WriteBBSetupJson(buildstep.ShellMixin, buildstep.BuildStep): +class WriteBBSetupJson(steps.StringDownload): name = "WriteBBSetupJson" def __init__(self, **kwargs): - buildstep.BuildStep.__init__(self, **kwargs) + super().__init__(s="", workerdest="bbsetup-overrides.json", **kwargs) def generateSetupJson(self): setupdata = {}
@@ -66,14 +65,10 @@ class WriteBBSetupJson(buildstep.ShellMixin, buildstep.BuildStep): @defer.inlineCallbacks def run(self): - repojson = self.generateSetupJson() - layerinfo = os.path.join(self.getProperty("builddir"), - "bbsetup-overrides.json") - writerepos = "printf '%s' > %s" % (repojson, layerinfo) - cmd = yield self.makeRemoteShellCommand( - command=writerepos) - yield self.runCommand(cmd) - return cmd.results() + self.s = self.generateSetupJson() + self.workdir = self.getProperty("builddir") + result = yield super().run() + return result @defer.inlineCallbacks
steps/writelayerinfo.py +7 -12
@@ -3,18 +3,17 @@ # from twisted.internet import defer -from buildbot.process import buildstep +from buildbot.plugins import steps import json -import os from yoctoabb import config -class WriteLayerInfo(buildstep.ShellMixin, buildstep.BuildStep): +class WriteLayerInfo(steps.StringDownload): name = "WriteLayerInfo" def __init__(self, **kwargs): - buildstep.BuildStep.__init__(self, **kwargs) + super().__init__(s="", workerdest="layerinfo.json", **kwargs) def generateLayerInfo(self): layerinfo = {}
@@ -34,14 +33,10 @@ class WriteLayerInfo(buildstep.ShellMixin, buildstep.BuildStep): @defer.inlineCallbacks def run(self): - repojson = self.generateLayerInfo() - layerinfo = os.path.join(self.getProperty("builddir"), - "layerinfo.json") - writerepos = "printf '%s' > %s" % (repojson, layerinfo) - cmd = yield self.makeRemoteShellCommand( - command=writerepos) - yield self.runCommand(cmd) - return cmd.results() + self.s = self.generateLayerInfo() + self.workdir = self.getProperty("builddir") + result = yield super().run() + return result @defer.inlineCallbacks
PATCH 2/3 builders: pass getproperties command as argv list
In reply to Anders HeimerString commands are processed by the worker shell, which is fragile for quoting and unnecessary when the command and arguments are already known. Pass getproperties.py as an argv list so builddir and buildername values are delivered as arguments without shell interpretation.
AI-Generated: Claude Opus 4.6 <[email protected]>
--- builders.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-)
builders.py +10 -2
@@ -141,7 +141,11 @@ def create_builder_factory(): f.addStep(steps.JSONPropertiesDownload(workerdest="build-properties.json")) f.addStep(steps.SetPropertyFromCommand( - command=util.Interpolate("%(prop:builddir)s/yocto-autobuilder-helper/scripts/getproperties.py %(prop:builddir)s/build %(prop:buildername)s"), + command=[ + util.Interpolate("%(prop:builddir)s/yocto-autobuilder-helper/scripts/getproperties.py"), + util.Interpolate("%(prop:builddir)s/build"), + util.Property("buildername"), + ], extract_fn=extract_json_props, name='Load build revisions', haltOnFailure=True))
@@ -364,7 +368,11 @@ def create_parent_builder_factory(buildername, waitname): factory.addStep(steps.JSONPropertiesDownload(workerdest="build-properties.json")) factory.addStep(steps.SetPropertyFromCommand( - command=util.Interpolate("%(prop:builddir)s/yocto-autobuilder-helper/scripts/getproperties.py %(prop:builddir)s/build %(prop:buildername)s"), + command=[ + util.Interpolate("%(prop:builddir)s/yocto-autobuilder-helper/scripts/getproperties.py"), + util.Interpolate("%(prop:builddir)s/build"), + util.Property("buildername"), + ], extract_fn=extract_json_props, name='Load build revisions', haltOnFailure=True))
PATCH 3/3 builders: avoid shell pipeline for disk space check
In reply to Anders HeimerThe disk space probe used a shell pipeline to run findmnt and awk. Ask findmnt for the available-byte field directly and pass the command as argv, avoiding shell interpretation.
AI-Generated: Claude Opus 4.6 <[email protected]>
--- builders.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-)
builders.py +3 -1
@@ -81,7 +81,9 @@ def canStartBuild(builder, wfb, request): bytes_in_gb = 1024 * 1024 * 1024 - cmd = yield shell("findmnt -T %s --df -n --bytes | awk '{print $5}'" % mountpoint, wfb.worker, builder) + cmd = yield shell(["findmnt", "-T", mountpoint, "--df", "-n", "--bytes", + "-o", "AVAIL"], + wfb.worker, builder) space = int(cmd.stdout.strip()) if space < threshold * bytes_in_gb: log.msg("Detected {0} GB of space available on {1}, less than threshold of {2} GB. Can't start build".format(space / bytes_in_gb, name, threshold))