Skip to contentSkip to conversation

Toaster improvements (OE-Core part)

PATCH New Paul Eggleton 1 participant 4 messages Dec 9, 2013

lore
Share & tools
lore
#0 lore raw

cover Toaster improvements (OE-Core part)

Note: this should be merged into OE-Core at the same time that the corresponding changes to BitBake are merged.

The following changes since commit 9f69e00200cdbd5ba2e46a54f33c29797816e43f:

runqemu: remove core-image-* whitelist (2013-12-09 18:01:39 +0000)

are available in the git repository at:

git://git.openembedded.org/openembedded-core-contrib paule/toaster-oecore
http://cgit.openembedded.org/cgit.cgi/openembedded-core-contrib/log/?h=paule/toaster-oecore

Alexandru DAMIAN (3):
toaster.bbclass: read layer information toaster.bbclass: read build stats toaster.bbclass: read package and image information

 meta/classes/toaster.bbclass | 230 +++++++++++++++++++++++++++++++++++++++----
 1 file changed, 213 insertions(+), 17 deletions(-)
Signature · 1 line
1.8.1.2
#1 lore raw

PATCH 1/3 toaster.bbclass: read layer information

From: Alexandru DAMIAN <[email protected]>

In the process of removing the local system accesses from toaster UI (which must be able to run remotely), the code to read layer information is moved from Bitbake Toaster UI
to the server-side toaster.bbclass

    [YOCTO #5604]
---
 meta/classes/toaster.bbclass | 68 +++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 67 insertions(+), 1 deletion(-)
meta/classes/toaster.bbclass +67 -1
index 8dc1663..ec9b6c5 100644
@@ -24,7 +24,73 @@
 #
 #
 
-# 1. Dump package file info data
+# Find and dump layer info when we got the layers parsed
+
+
+
+python toaster_layerinfo_dumpdata() {
+    import subprocess
+
+    def _get_git_branch(layer_path):
+        branch = subprocess.Popen("git symbolic-ref HEAD 2>/dev/null ", cwd=layer_path, shell=True, stdout=subprocess.PIPE).communicate()[0]
+        branch = branch.replace('refs/heads/', '').rstrip()
+        return branch
+
+    def _get_git_revision(layer_path):
+        revision = subprocess.Popen("git rev-parse HEAD 2>/dev/null ", cwd=layer_path, shell=True, stdout=subprocess.PIPE).communicate()[0].rstrip()
+        return revision
+
+    def _get_url_map_name(layer_name):
+        """ Some layers have a different name on openembedded.org site,
+            this method returns the correct name to use in the URL
+        """
+
+        url_name = layer_name
+        url_mapping = {'meta': 'openembedded-core'}
+
+        for key in url_mapping.keys():
+            if key == layer_name:
+                url_name = url_mapping[key]
+
+        return url_name
+
+    def _get_layer_version_information(layer_path):
+
+        layer_version_info = {}
+        layer_version_info['branch'] = _get_git_branch(layer_path)
+        layer_version_info['commit'] = _get_git_revision(layer_path)
+        layer_version_info['priority'] = 0
+
+        return layer_version_info
+
+
+    def _get_layer_dict(layer_path):
+
+        layer_info = {}
+        layer_name = layer_path.split('/')[-1]
+        layer_url = 'http://layers.openembedded.org/layerindex/layer/{layer}/'
+        layer_url_name = _get_url_map_name(layer_name)
+
+        layer_info['name'] = layer_name
+        layer_info['local_path'] = layer_path
+        layer_info['layer_index_url'] = layer_url.format(layer=layer_url_name)
+        layer_info['version'] = _get_layer_version_information(layer_path)
+
+        return layer_info
+
+
+    bblayers = e.data.getVar("BBLAYERS", True)
+
+    llayerinfo = {}
+
+    for layer in { l for l in bblayers.strip().split(" ") if len(l) }:
+        llayerinfo[layer] = _get_layer_dict(layer)
+
+
+    bb.event.fire(bb.event.MetadataEvent("LayerInfo", llayerinfo), e.data)
+}
+
+# Dump package file info data
 
 python toaster_package_dumpdata() {
     """
Signature · 1 line
1.8.1.2
#2 lore raw

PATCH 2/3 toaster.bbclass: read build stats

In reply to Paul Eggleton

From: Alexandru DAMIAN <[email protected]>

In the process of removing the local system accesses from toaster UI (which must be able to run remotely), the code to read build stats is moved from Bitbake Toaster UI
to the server-side toaster.bbclass

The code will accumulate a list of stat files to be read at build completion. When the build completes, the whole data list is read and sent through in a single event.

    [YOCTO #5604]
---
 meta/classes/toaster.bbclass | 73 ++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 73 insertions(+)
meta/classes/toaster.bbclass +73 -0
index ec9b6c5..59c61c5 100644
@@ -153,6 +153,79 @@ python toaster_image_dumpdata() {
 }
 
 
+
+# collect list of buildstats files based on fired events; when the build completes, collect all stats and fire an event with collected data
+
+python toaster_collect_task_stats() {
+    import bb.build
+    import bb.event
+    import bb.data
+    import bb.utils
+    import os
+
+    def _append_read_list(v):
+        lock = bb.utils.lockfile(e.data.expand("${TOPDIR}/toaster.lock"), False, True)
+
+        with open(os.path.join(e.data.getVar('BUILDSTATS_BASE', True), "toasterstatlist"), "a") as fout:
+            bn = get_bn(e)
+            bsdir = os.path.join(e.data.getVar('BUILDSTATS_BASE', True), bn)
+            taskdir = os.path.join(bsdir, e.data.expand("${PF}"))
+            fout.write("%s:%s:%s\n" % (e.taskfile, e.taskname, os.path.join(taskdir, e.task)))
+
+        bb.utils.unlockfile(lock)
+
+    def _read_stats(filename):
+        cpu_usage = 0
+        disk_io = 0
+        startio = ''
+        endio = ''
+        pn = ''
+        taskname = ''
+        statinfo = {}
+
+        with open(filename, 'r') as task_bs:
+            for line in task_bs.readlines():
+                k,v = line.strip().split(": ", 1)
+                statinfo[k] = v
+
+        try:
+            cpu_usage = statinfo["CPU usage"]
+            endio = statinfo["EndTimeIO"]
+            startio = statinfo["StartTimeIO"]
+        except KeyError:
+            pass    # we may have incomplete data here
+
+        if startio and endio:
+            disk_io = int(endio.strip('\n ')) - int(startio.strip('\n '))
+
+        if cpu_usage:
+            cpu_usage = float(cpu_usage.strip('% \n'))
+
+        return {'cpu_usage': cpu_usage, 'disk_io': disk_io}
+
+
+    if isinstance(e, (bb.build.TaskSucceeded, bb.build.TaskFailed)):
+        _append_read_list(e)
+        pass
+
+
+    if isinstance(e, bb.event.BuildCompleted):
+        events = []
+        with open(os.path.join(e.data.getVar('BUILDSTATS_BASE', True), "toasterstatlist"), "r") as fin:
+            for line in fin:
+                (taskfile, taskname, filename) = line.strip().split(":")
+                events.append((taskfile, taskname, _read_stats(filename)))
+        bb.event.fire(bb.event.MetadataEvent("BuildStatsList", events), e.data)
+        os.unlink(os.path.join(e.data.getVar('BUILDSTATS_BASE', True), "toasterstatlist"))
+}
+
+
+# set event handlers
+addhandler toaster_layerinfo_dumpdata
+toaster_layerinfo_dumpdata[eventmask] = "bb.event.TreeDataPreparationCompleted"
+
+addhandler toaster_collect_task_stats
+toaster_collect_task_stats[eventmask] = "bb.event.BuildCompleted bb.build.TaskSucceeded bb.build.TaskFailed"
 do_package[postfuncs] += "toaster_package_dumpdata "
 
 do_rootfs[postfuncs] += "toaster_image_dumpdata "
Signature · 1 line
1.8.1.2
#3 lore raw

PATCH 3/3 toaster.bbclass: read package and image information

In reply to Paul Eggleton

From: Alexandru DAMIAN <[email protected]>

In the process of removing the local system accesses from toaster UI (which must be able to run remotely), the code to read package information is moved from Bitbake Toaster UI to the server-side toaster.bbclass

     [YOCTO #5604]
---
 meta/classes/toaster.bbclass | 91 +++++++++++++++++++++++++++++++++++---------
 1 file changed, 74 insertions(+), 17 deletions(-)
meta/classes/toaster.bbclass +74 -17
index 59c61c5..e1a93b5 100644
@@ -92,6 +92,23 @@ python toaster_layerinfo_dumpdata() {
 
 # Dump package file info data
 
+def _toaster_load_pkgdatafile(dirpath, filepath):
+    import json
+    pkgdata = {}
+    with open(os.path.join(dirpath, filepath), "r") as fin:
+        for line in fin:
+            try:
+                kn, kv = line.strip().split(": ", 1)
+                kn = "_".join([x for x in kn.split("_") if x.isupper()])
+                pkgdata[kn] = kv.strip()
+                if kn == 'FILES_INFO':
+                    pkgdata[kn] = json.loads(kv)
+
+            except ValueError:
+                pass    # ignore lines without valid key: value pairs
+    return pkgdata
+
+
 python toaster_package_dumpdata() {
     """
     Dumps the data created by emit_pkgdata
@@ -103,27 +120,12 @@ python toaster_package_dumpdata() {
 
     pkgdatadir = d.getVar('PKGDESTWORK', True)
 
-
     # scan and send data for each package
-    import json
 
     lpkgdata = {}
     for pkg in packages.split():
 
-        subdata_file = pkgdatadir + "/runtime/%s" % pkg
-        lpkgdata = {}
-
-        sf = open(subdata_file, "r")
-        line = sf.readline()
-        while line:
-            (n, v) = line.rstrip().split(":", 1)
-            if pkg in n:
-                n = n.replace("_" + pkg, "")
-            if n == 'FILES_INFO':
-                lpkgdata[n] = json.loads(v)
-            else:
-                lpkgdata[n] = v.strip()
-            line = sf.readline()
+        lpkgdata = _toaster_load_pkgdatafile(pkgdatadir + "/runtime/", pkg)
 
         # Fire an event containing the pkg data
         bb.event.fire(bb.event.MetadataEvent("SinglePackageInfo", lpkgdata), d)
@@ -209,7 +211,7 @@ python toaster_collect_task_stats() {
         pass
 
 
-    if isinstance(e, bb.event.BuildCompleted):
+    if isinstance(e, bb.event.BuildCompleted) and os.path.exists(os.path.join(e.data.getVar('BUILDSTATS_BASE', True), "toasterstatlist")):
         events = []
         with open(os.path.join(e.data.getVar('BUILDSTATS_BASE', True), "toasterstatlist"), "r") as fin:
             for line in fin:
@@ -219,6 +221,58 @@ python toaster_collect_task_stats() {
         os.unlink(os.path.join(e.data.getVar('BUILDSTATS_BASE', True), "toasterstatlist"))
 }
 
+# dump relevant build history data as an event when the build is completed
+
+python toaster_buildhistory_dump() {
+    import re
+    BUILDHISTORY_DIR = e.data.expand("${TOPDIR}/buildhistory")
+    BUILDHISTORY_DIR_IMAGE_BASE = e.data.expand("%s/images/${MACHINE_ARCH}/${TCLIBC}/"% BUILDHISTORY_DIR)
+    pkgdata_dir = e.data.getVar("PKGDATA_DIR", True)
+
+
+    # scan the build targets for this build
+    images = {}
+    allpkgs = {}
+    for target in e._pkgs:
+        installed_img_path = e.data.expand(os.path.join(BUILDHISTORY_DIR_IMAGE_BASE, target))
+        if os.path.exists(installed_img_path):
+            images[target] = {}
+            with open("%s/installed-package-sizes.txt" % installed_img_path, "r") as fin:
+                for line in fin:
+                    line = line.rstrip(";")
+                    psize, px = line.split("\t")
+                    punit, pname = px.split(" ")
+                    # this size is "installed-size" as it measures how much space it takes on disk
+                    images[target][pname.strip()] = {'size':int(psize)*1024, 'depends' : []}
+
+            with open("%s/depends.dot" % installed_img_path, "r") as fin:
+                p = re.compile(r' -> ')
+                dot = re.compile(r'.*style=dotted')
+                for line in fin:
+                    line = line.rstrip(';')
+                    linesplit = p.split(line)
+                    if len(linesplit) == 2:
+                        pname = linesplit[0].rstrip('"').strip('"')
+                        dependsname = linesplit[1].split(" ")[0].strip().strip(";").strip('"').rstrip('"')
+                        deptype = "depends"
+                        if dot.match(line):
+                            deptype = "recommends"
+                        if not pname in images[target]:
+                            images[target][pname] = {'size': 0, 'depends' : []}
+                        if not dependsname in images[target]:
+                            images[target][dependsname] = {'size': 0, 'depends' : []}
+                        images[target][pname]['depends'].append((dependsname, deptype))
+
+            for pname in images[target]:
+                if not pname in allpkgs:
+                    allpkgs[pname] = _toaster_load_pkgdatafile("%s/runtime-reverse/" % pkgdata_dir, pname)
+
+
+    data = { 'pkgdata' : allpkgs, 'imgdata' : images }
+
+    bb.event.fire(bb.event.MetadataEvent("ImagePkgList", data), e.data)
+
+}
 
 # set event handlers
 addhandler toaster_layerinfo_dumpdata
@@ -226,6 +280,9 @@ toaster_layerinfo_dumpdata[eventmask] = "bb.event.TreeDataPreparationCompleted"
 
 addhandler toaster_collect_task_stats
 toaster_collect_task_stats[eventmask] = "bb.event.BuildCompleted bb.build.TaskSucceeded bb.build.TaskFailed"
+
+addhandler toaster_buildhistory_dump
+toaster_buildhistory_dump[eventmask] = "bb.event.BuildCompleted"
 do_package[postfuncs] += "toaster_package_dumpdata "
 
 do_rootfs[postfuncs] += "toaster_image_dumpdata "
Signature · 1 line
1.8.1.2