Skip to contentSkip to conversation

Avoid shell use

PATCH New Anders Heimer 1 participant 4 messages Jun 11

lore
Share & tools
lore
#0 lore raw

cover Avoid shell use

This series removes shell use from a few backend paths where Buildbot already has structured command arguments or generated file contents.

The changes are intended as hardening/cleanup with no intentional behaviour change.

Testing:
* Metadata writers:
- qemuarm-oecore build wrote layerinfo.json and bbsetup-overrides.json via StringDownload
- both files parsed correctly and preserved repo, branch and revision values literally
- whitebox comparison showed byte-for-byte identical output for qemuarm-oecore, a-quick, docs and default-repo fallback cases.

* getproperties.py argv change:
- live qemuarm-oecore build reached and passed "Load build revisions"
- the step set 15 properties
- Buildbot log showed direct argv execution:
argv: [b'.../getproperties.py', b'.../build', b'qemuarm-oecore']
- direct replay on a preserved qemuarm-oecore build tree matched the old shell-string output exactly.

* Disk-space check:
- live builds passed the canStartBuild disk gate and started normally
- local comparison showed the old findmnt --df fifth field matched the new findmnt -o AVAIL output for ., /tmp and the worker tree

Anders Heimer (3):
steps: avoid shell when writing repo metadata
builders: pass getproperties command as argv list
builders: avoid shell pipeline for disk space check

 builders.py               | 16 +++++++++++++---
 steps/writebbsetupjson.py | 19 +++++++------------
 steps/writelayerinfo.py   | 19 +++++++------------
 3 files changed, 27 insertions(+), 27 deletions(-)
#1 lore raw

PATCH 1/3 steps: avoid shell when writing repo metadata

Use Buildbot StringDownload for repo metadata JSON instead of shell printf. This transfers scheduler-supplied repo values as file content without shell interpretation.

AI-Generated: Claude Opus 4.6 <[email protected]>

---
 steps/writebbsetupjson.py | 19 +++++++------------
 steps/writelayerinfo.py   | 19 +++++++------------
 2 files changed, 14 insertions(+), 24 deletions(-)
steps/writebbsetupjson.py +7 -12
index da8d434..e1f1463 100644
@@ -5,9 +5,8 @@
 #
 
 from twisted.internet import defer
-from buildbot.process import buildstep
+from buildbot.plugins import steps
 import json
-import os
 
 from yoctoabb import config
 
@@ -15,11 +14,11 @@ mappings = {
     "oecore" : "openembedded-core",
 }
 
-class WriteBBSetupJson(buildstep.ShellMixin, buildstep.BuildStep):
+class WriteBBSetupJson(steps.StringDownload):
     name = "WriteBBSetupJson"
 
     def __init__(self, **kwargs):
-        buildstep.BuildStep.__init__(self, **kwargs)
+        super().__init__(s="", workerdest="bbsetup-overrides.json", **kwargs)
 
     def generateSetupJson(self):
         setupdata = {}
@@ -66,14 +65,10 @@ class WriteBBSetupJson(buildstep.ShellMixin, buildstep.BuildStep):
 
     @defer.inlineCallbacks
     def run(self):
-        repojson = self.generateSetupJson()
-        layerinfo = os.path.join(self.getProperty("builddir"),
-                                 "bbsetup-overrides.json")
-        writerepos = "printf '%s' > %s" % (repojson, layerinfo)
-        cmd = yield self.makeRemoteShellCommand(
-            command=writerepos)
-        yield self.runCommand(cmd)
-        return cmd.results()
+        self.s = self.generateSetupJson()
+        self.workdir = self.getProperty("builddir")
+        result = yield super().run()
+        return result
 
 
 @defer.inlineCallbacks
steps/writelayerinfo.py +7 -12
index a0669bd..009e9fc 100644
@@ -3,18 +3,17 @@
 #
 
 from twisted.internet import defer
-from buildbot.process import buildstep
+from buildbot.plugins import steps
 import json
-import os
 
 from yoctoabb import config
 
 
-class WriteLayerInfo(buildstep.ShellMixin, buildstep.BuildStep):
+class WriteLayerInfo(steps.StringDownload):
     name = "WriteLayerInfo"
 
     def __init__(self, **kwargs):
-        buildstep.BuildStep.__init__(self, **kwargs)
+        super().__init__(s="", workerdest="layerinfo.json", **kwargs)
 
     def generateLayerInfo(self):
         layerinfo = {}
@@ -34,14 +33,10 @@ class WriteLayerInfo(buildstep.ShellMixin, buildstep.BuildStep):
 
     @defer.inlineCallbacks
     def run(self):
-        repojson = self.generateLayerInfo()
-        layerinfo = os.path.join(self.getProperty("builddir"),
-                                 "layerinfo.json")
-        writerepos = "printf '%s' > %s" % (repojson, layerinfo)
-        cmd = yield self.makeRemoteShellCommand(
-            command=writerepos)
-        yield self.runCommand(cmd)
-        return cmd.results()
+        self.s = self.generateLayerInfo()
+        self.workdir = self.getProperty("builddir")
+        result = yield super().run()
+        return result
 
 
 @defer.inlineCallbacks
#2 lore raw

PATCH 2/3 builders: pass getproperties command as argv list

In reply to Anders Heimer

String commands are processed by the worker shell, which is fragile for quoting and unnecessary when the command and arguments are already known. Pass getproperties.py as an argv list so builddir and buildername values are delivered as arguments without shell interpretation.

AI-Generated: Claude Opus 4.6 <[email protected]>

---
 builders.py | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)
builders.py +10 -2
index 55f0769..418ef1b 100644
@@ -141,7 +141,11 @@ def create_builder_factory():
     f.addStep(steps.JSONPropertiesDownload(workerdest="build-properties.json"))
 
     f.addStep(steps.SetPropertyFromCommand(
-        command=util.Interpolate("%(prop:builddir)s/yocto-autobuilder-helper/scripts/getproperties.py %(prop:builddir)s/build %(prop:buildername)s"),
+        command=[
+            util.Interpolate("%(prop:builddir)s/yocto-autobuilder-helper/scripts/getproperties.py"),
+            util.Interpolate("%(prop:builddir)s/build"),
+            util.Property("buildername"),
+        ],
         extract_fn=extract_json_props,
         name='Load build revisions',
         haltOnFailure=True))
@@ -364,7 +368,11 @@ def create_parent_builder_factory(buildername, waitname):
     factory.addStep(steps.JSONPropertiesDownload(workerdest="build-properties.json"))
 
     factory.addStep(steps.SetPropertyFromCommand(
-        command=util.Interpolate("%(prop:builddir)s/yocto-autobuilder-helper/scripts/getproperties.py %(prop:builddir)s/build %(prop:buildername)s"),
+        command=[
+            util.Interpolate("%(prop:builddir)s/yocto-autobuilder-helper/scripts/getproperties.py"),
+            util.Interpolate("%(prop:builddir)s/build"),
+            util.Property("buildername"),
+        ],
         extract_fn=extract_json_props,
         name='Load build revisions',
         haltOnFailure=True))
#3 lore raw

PATCH 3/3 builders: avoid shell pipeline for disk space check

In reply to Anders Heimer

The disk space probe used a shell pipeline to run findmnt and awk. Ask findmnt for the available-byte field directly and pass the command as argv, avoiding shell interpretation.

AI-Generated: Claude Opus 4.6 <[email protected]>

---
 builders.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)
builders.py +3 -1
index 418ef1b..1b65e55 100644
@@ -81,7 +81,9 @@ def canStartBuild(builder, wfb, request):
 
         bytes_in_gb = 1024 * 1024 * 1024
 
-        cmd = yield shell("findmnt -T %s --df -n --bytes | awk '{print $5}'" % mountpoint, wfb.worker, builder)
+        cmd = yield shell(["findmnt", "-T", mountpoint, "--df", "-n", "--bytes",
+                           "-o", "AVAIL"],
+                          wfb.worker, builder)
         space = int(cmd.stdout.strip())
         if space < threshold * bytes_in_gb:
             log.msg("Detected {0} GB of space available on {1}, less than threshold of {2} GB. Can't start build".format(space / bytes_in_gb, name, threshold))