Skip to contentSkip to conversation

lib/buildsteps.py: Add BitbakeShellCommand class for dump bitbake logs

PATCH In review Aníbal Limón 2 participants 3 messages Mar 1, 2016

lore
Share & tools
lore

Aníbal Limón

author
#0 lore raw

PATCH 1/1 lib/buildsteps.py: Add BitbakeShellCommand class for dump bitbake logs

The BitbakeShellCommand is a new class for store common methods when is aim to run bitbake inside an step.

This commit overrides commandComplete method for search in bitbake stdout if ERROR exists and if found then open the bitbake log and dumps into new bitbake_logs output.

Signed-off-by Aníbal Limón <[email protected]>
---
 .../autobuilder/buildsteps/BuildImages.py          |  4 +++-
 .../autobuilder/buildsteps/BuildToolchainImages.py |  4 +++-
 .../autobuilder/buildsteps/GetBitbakeVersion.py    |  4 +++-
 .../autobuilder/buildsteps/RunESDKSanityTests.py   |  4 +++-
 .../autobuilder/buildsteps/RunSDKSanityTests.py    |  4 +++-
 .../autobuilder/buildsteps/RunSanityTests.py       |  4 +++-
 .../site-packages/autobuilder/lib/buildsteps.py    | 28 ++++++++++++++++++++++
 7 files changed, 46 insertions(+), 6 deletions(-)
lib/python2.7/site-packages/autobuilder/buildsteps/BuildImages.py +3 -1
index 4987ef3..5ed6e1a 100644
@@ -17,7 +17,9 @@ from distutils.version import StrictVersion
 from buildbot.status.results import SUCCESS
 import os
 
-class BuildImages(ShellCommand):
+from lib.buildsteps import BitbakeShellCommand
+
+class BuildImages(BitbakeShellCommand):
     haltOnFailure = False
     flunkOnFailure = True
     name = "BuildImages"
lib/python2.7/site-packages/autobuilder/buildsteps/BuildToolchainImages.py +3 -1
index 51069c2..0594ac2 100644
@@ -15,7 +15,9 @@ from buildbot.steps.shell import ShellCommand
 from buildbot.process.buildstep import LogLineObserver
 import os
 
-class BuildToolchainImages(ShellCommand):
+from lib.buildsteps import BitbakeShellCommand
+
+class BuildToolchainImages(BitbakeShellCommand):
     haltOnFailure = False
     flunkOnFailure = True
     name = "Building Toolchain Images"
lib/python2.7/site-packages/autobuilder/buildsteps/GetBitbakeVersion.py +3 -1
index dc6c25b..94b5efe 100644
@@ -16,7 +16,9 @@ from buildbot.process.properties import WithProperties
 from twisted.python import log
 from autobuilder.config import *
 
-class GetBitbakeVersion(ShellCommand):
+from lib.buildsteps import BitbakeShellCommand
+
+class GetBitbakeVersion(BitbakeShellCommand):
     haltOnFailure = False
     flunkOnFailure = False
     name = "GetBitbakeVersion"
lib/python2.7/site-packages/autobuilder/buildsteps/RunESDKSanityTests.py +3 -1
index e3df6c8..f2817d4 100644
@@ -16,7 +16,9 @@ from buildbot.status.results import SUCCESS, FAILURE
 from twisted.python import log as tlog
 import os, re
 
-class RunESDKSanityTests(ShellCommand):
+from lib.buildsteps import BitbakeShellCommand
+
+class RunESDKSanityTests(BitbakeShellCommand):
 
     haltOnFailure = False
     flunkOnFailure = True
lib/python2.7/site-packages/autobuilder/buildsteps/RunSDKSanityTests.py +3 -1
index cc0321f..1eeb4d3 100644
@@ -16,7 +16,9 @@ from buildbot.status.results import SUCCESS, FAILURE
 from twisted.python import log as tlog
 import os, re
 
-class RunSDKSanityTests(ShellCommand):
+from lib.buildsteps import BitbakeShellCommand
+
+class RunSDKSanityTests(BitbakeShellCommand):
 
     haltOnFailure = False
     flunkOnFailure = True
lib/python2.7/site-packages/autobuilder/buildsteps/RunSanityTests.py +3 -1
index 62f04af..15a714c 100644
@@ -16,7 +16,9 @@ from buildbot.status.results import SUCCESS, FAILURE
 from twisted.python import log as tlog
 import os, re
 
-class RunSanityTests(ShellCommand):
+from lib.buildsteps import BitbakeShellCommand
+
+class RunSanityTests(BitbakeShellCommand):
 
     haltOnFailure = False
     flunkOnFailure = True
lib/python2.7/site-packages/autobuilder/lib/buildsteps.py +28 -0
index 3693a7a..4d2787d 100644
@@ -11,6 +11,8 @@ __email__ = "[email protected]"
 '''
 
 import os
+import re
+
 from buildbot.steps.shell import ShellCommand
 
 DEFAULT_SHELL = 'bash'

@@ -40,3 +42,29 @@ class ShellCommandCleanEnv(ShellCommand):
             pe_cmd += "%s=\"$%s\" " % (pe, pe)
 
         return "env -i %s %s -c " % (pe_cmd, shell)
+
+class BitbakeShellCommand(ShellCommand):
+    def commandComplete(self, cmd):
+        if cmd.didFail():
+            log = self.addLog('bitbake_logs')
+            paths_displayed = []
+
+            rexp = re.compile("^ERROR: Logfile of failure stored in: (.*)$")
+            out = cmd.logs['stdio'].getText()
+            for line in out.split('\n'):
+                m = rexp.match(line)
+                if m:
+                    path = m.group(1)
+
+                    if path in paths_displayed:
+                        continue
+
+                    paths_displayed.append(path)
+
+                    log.addStdout("Output of: %s\n" % path)
+                    with open(path, "r") as f:
+                        log.addStdout(f.read())
+                    log.addStdout("End output of: %s\n" % path)
+                    log.addStdout("\n\n\n\n")
+            log.finish()
+
Signature · 1 line
2.1.4
#1 lore raw
Quoted 6 lines · Aníbal Limón

The BitbakeShellCommand is a new class for store common methods when is aim to run bitbake inside an step.

This commit overrides commandComplete method for search in bitbake stdout if ERROR exists and if found then open the bitbake log and dumps into new bitbake_logs output.

Yes, I like this a lot. I'm going to pull it into master-next and test it for a bit. Expect it in master/production in a few days.

-b

Quoted 148 lines
Signed-off-by Aníbal Limón <[email protected]>
---
 .../autobuilder/buildsteps/BuildImages.py          |  4 +++-
 .../autobuilder/buildsteps/BuildToolchainImages.py |  4 +++-
 .../autobuilder/buildsteps/GetBitbakeVersion.py    |  4 +++-
 .../autobuilder/buildsteps/RunESDKSanityTests.py   |  4 +++-
 .../autobuilder/buildsteps/RunSDKSanityTests.py    |  4 +++-
 .../autobuilder/buildsteps/RunSanityTests.py       |  4 +++-
 .../site-packages/autobuilder/lib/buildsteps.py    | 28 ++++++++++++++++++++++
 7 files changed, 46 insertions(+), 6 deletions(-)
lib/python2.7/site-packages/autobuilder/buildsteps/BuildImages.py +3 -1
index 4987ef3..5ed6e1a 100644
@@ -17,7 +17,9 @@ from distutils.version import StrictVersion
 from buildbot.status.results import SUCCESS
 import os
 
-class BuildImages(ShellCommand):
+from lib.buildsteps import BitbakeShellCommand
+
+class BuildImages(BitbakeShellCommand):
     haltOnFailure = False
     flunkOnFailure = True
     name = "BuildImages"
lib/python2.7/site-packages/autobuilder/buildsteps/BuildToolchainImages.py +3 -1
index 51069c2..0594ac2 100644
@@ -15,7 +15,9 @@ from buildbot.steps.shell import ShellCommand
 from buildbot.process.buildstep import LogLineObserver
 import os
 
-class BuildToolchainImages(ShellCommand):
+from lib.buildsteps import BitbakeShellCommand
+
+class BuildToolchainImages(BitbakeShellCommand):
     haltOnFailure = False
     flunkOnFailure = True
     name = "Building Toolchain Images"
lib/python2.7/site-packages/autobuilder/buildsteps/GetBitbakeVersion.py +3 -1
index dc6c25b..94b5efe 100644
@@ -16,7 +16,9 @@ from buildbot.process.properties import WithProperties
 from twisted.python import log
 from autobuilder.config import *
 
-class GetBitbakeVersion(ShellCommand):
+from lib.buildsteps import BitbakeShellCommand
+
+class GetBitbakeVersion(BitbakeShellCommand):
     haltOnFailure = False
     flunkOnFailure = False
     name = "GetBitbakeVersion"
lib/python2.7/site-packages/autobuilder/buildsteps/RunESDKSanityTests.py +3 -1
index e3df6c8..f2817d4 100644
@@ -16,7 +16,9 @@ from buildbot.status.results import SUCCESS, FAILURE
 from twisted.python import log as tlog
 import os, re
 
-class RunESDKSanityTests(ShellCommand):
+from lib.buildsteps import BitbakeShellCommand
+
+class RunESDKSanityTests(BitbakeShellCommand):
 
     haltOnFailure = False
     flunkOnFailure = True
lib/python2.7/site-packages/autobuilder/buildsteps/RunSDKSanityTests.py +3 -1
index cc0321f..1eeb4d3 100644
@@ -16,7 +16,9 @@ from buildbot.status.results import SUCCESS, FAILURE
 from twisted.python import log as tlog
 import os, re
 
-class RunSDKSanityTests(ShellCommand):
+from lib.buildsteps import BitbakeShellCommand
+
+class RunSDKSanityTests(BitbakeShellCommand):
 
     haltOnFailure = False
     flunkOnFailure = True
lib/python2.7/site-packages/autobuilder/buildsteps/RunSanityTests.py +3 -1
index 62f04af..15a714c 100644
@@ -16,7 +16,9 @@ from buildbot.status.results import SUCCESS, FAILURE
 from twisted.python import log as tlog
 import os, re
 
-class RunSanityTests(ShellCommand):
+from lib.buildsteps import BitbakeShellCommand
+
+class RunSanityTests(BitbakeShellCommand):
 
     haltOnFailure = False
     flunkOnFailure = True
lib/python2.7/site-packages/autobuilder/lib/buildsteps.py +28 -1
index 3693a7a..4d2787d 100644
@@ -11,6 +11,8 @@ __email__ = "[email protected]"
 '''
 
 import os
+import re
+
 from buildbot.steps.shell import ShellCommand
 
 DEFAULT_SHELL = 'bash'

@@ -40,3 +42,29 @@ class ShellCommandCleanEnv(ShellCommand):
             pe_cmd += "%s=\"$%s\" " % (pe, pe)
 
         return "env -i %s %s -c " % (pe_cmd, shell)
+
+class BitbakeShellCommand(ShellCommand):
+    def commandComplete(self, cmd):
+        if cmd.didFail():
+            log = self.addLog('bitbake_logs')
+            paths_displayed = []
+
+            rexp = re.compile("^ERROR: Logfile of failure stored in: (.*)$")
+            out = cmd.logs['stdio'].getText()
+            for line in out.split('\n'):
+                m = rexp.match(line)
+                if m:
+                    path = m.group(1)
+
+                    if path in paths_displayed:
+                        continue
+
+                    paths_displayed.append(path)
+
+                    log.addStdout("Output of: %s\n" % path)
+                    with open(path, "r") as f:
+                        log.addStdout(f.read())
+                    log.addStdout("End output of: %s\n" % path)
+                    log.addStdout("\n\n\n\n")
+            log.finish()
+
--

2.1.4

Signature · 3 lines
Elizabeth Flanagan
Yocto Project
Build and Release
#2 lore raw
Quoted 7 lines · Elizabeth
Quoted 6 lines · Aníbal Limón

The BitbakeShellCommand is a new class for store common methods when is aim to run bitbake inside an step.

This commit overrides commandComplete method for search in bitbake stdout if ERROR exists and if found then open the bitbake log and dumps into new bitbake_logs output.

So, obviously some issues here with using open (controller doesn't have access to the worker filesystem).

A thought. Scroll the logs, setting a property with the failures and log location and use that at the end to have a PublishBuildbotErrorLogs to DEST?

-b

Quoted 161 lines

Yes, I like this a lot. I'm going to pull it into master-next and test it for a bit. Expect it in master/production in a few days.

-b

Quoted 146 lines
Signed-off-by Aníbal Limón <[email protected]>
---
 .../autobuilder/buildsteps/BuildImages.py          |  4 +++-
 .../autobuilder/buildsteps/BuildToolchainImages.py |  4 +++-
 .../autobuilder/buildsteps/GetBitbakeVersion.py    |  4 +++-
 .../autobuilder/buildsteps/RunESDKSanityTests.py   |  4 +++-
 .../autobuilder/buildsteps/RunSDKSanityTests.py    |  4 +++-
 .../autobuilder/buildsteps/RunSanityTests.py       |  4 +++-
 .../site-packages/autobuilder/lib/buildsteps.py    | 28 ++++++++++++++++++++++
 7 files changed, 46 insertions(+), 6 deletions(-)
lib/python2.7/site-packages/autobuilder/buildsteps/BuildImages.py +3 -1
index 4987ef3..5ed6e1a 100644
@@ -17,7 +17,9 @@ from distutils.version import StrictVersion
 from buildbot.status.results import SUCCESS
 import os
 
-class BuildImages(ShellCommand):
+from lib.buildsteps import BitbakeShellCommand
+
+class BuildImages(BitbakeShellCommand):
     haltOnFailure = False
     flunkOnFailure = True
     name = "BuildImages"
lib/python2.7/site-packages/autobuilder/buildsteps/BuildToolchainImages.py +3 -1
index 51069c2..0594ac2 100644
@@ -15,7 +15,9 @@ from buildbot.steps.shell import ShellCommand
 from buildbot.process.buildstep import LogLineObserver
 import os
 
-class BuildToolchainImages(ShellCommand):
+from lib.buildsteps import BitbakeShellCommand
+
+class BuildToolchainImages(BitbakeShellCommand):
     haltOnFailure = False
     flunkOnFailure = True
     name = "Building Toolchain Images"
lib/python2.7/site-packages/autobuilder/buildsteps/GetBitbakeVersion.py +3 -1
index dc6c25b..94b5efe 100644
@@ -16,7 +16,9 @@ from buildbot.process.properties import WithProperties
 from twisted.python import log
 from autobuilder.config import *
 
-class GetBitbakeVersion(ShellCommand):
+from lib.buildsteps import BitbakeShellCommand
+
+class GetBitbakeVersion(BitbakeShellCommand):
     haltOnFailure = False
     flunkOnFailure = False
     name = "GetBitbakeVersion"
lib/python2.7/site-packages/autobuilder/buildsteps/RunESDKSanityTests.py +3 -1
index e3df6c8..f2817d4 100644
@@ -16,7 +16,9 @@ from buildbot.status.results import SUCCESS, FAILURE
 from twisted.python import log as tlog
 import os, re
 
-class RunESDKSanityTests(ShellCommand):
+from lib.buildsteps import BitbakeShellCommand
+
+class RunESDKSanityTests(BitbakeShellCommand):
 
     haltOnFailure = False
     flunkOnFailure = True
lib/python2.7/site-packages/autobuilder/buildsteps/RunSDKSanityTests.py +3 -1
index cc0321f..1eeb4d3 100644
@@ -16,7 +16,9 @@ from buildbot.status.results import SUCCESS, FAILURE
 from twisted.python import log as tlog
 import os, re
 
-class RunSDKSanityTests(ShellCommand):
+from lib.buildsteps import BitbakeShellCommand
+
+class RunSDKSanityTests(BitbakeShellCommand):
 
     haltOnFailure = False
     flunkOnFailure = True
lib/python2.7/site-packages/autobuilder/buildsteps/RunSanityTests.py +3 -1
index 62f04af..15a714c 100644
@@ -16,7 +16,9 @@ from buildbot.status.results import SUCCESS, FAILURE
 from twisted.python import log as tlog
 import os, re
 
-class RunSanityTests(ShellCommand):
+from lib.buildsteps import BitbakeShellCommand
+
+class RunSanityTests(BitbakeShellCommand):
 
     haltOnFailure = False
     flunkOnFailure = True
lib/python2.7/site-packages/autobuilder/lib/buildsteps.py +28 -1
index 3693a7a..4d2787d 100644
@@ -11,6 +11,8 @@ __email__ = "[email protected]"
 '''
 
 import os
+import re
+
 from buildbot.steps.shell import ShellCommand
 
 DEFAULT_SHELL = 'bash'

@@ -40,3 +42,29 @@ class ShellCommandCleanEnv(ShellCommand):
             pe_cmd += "%s=\"$%s\" " % (pe, pe)
 
         return "env -i %s %s -c " % (pe_cmd, shell)
+
+class BitbakeShellCommand(ShellCommand):
+    def commandComplete(self, cmd):
+        if cmd.didFail():
+            log = self.addLog('bitbake_logs')
+            paths_displayed = []
+
+            rexp = re.compile("^ERROR: Logfile of failure stored in: (.*)$")
+            out = cmd.logs['stdio'].getText()
+            for line in out.split('\n'):
+                m = rexp.match(line)
+                if m:
+                    path = m.group(1)
+
+                    if path in paths_displayed:
+                        continue
+
+                    paths_displayed.append(path)
+
+                    log.addStdout("Output of: %s\n" % path)
+                    with open(path, "r") as f:
+                        log.addStdout(f.read())
+                    log.addStdout("End output of: %s\n" % path)
+                    log.addStdout("\n\n\n\n")
+            log.finish()
+
--
Quoted 2 lines

2.1.4

--
Elizabeth Flanagan
Yocto Project
Build and Release

Signature · 3 lines
Elizabeth Flanagan
Yocto Project
Build and Release