From: Elliot Smith <elliot.smith@intel.com>
To: toaster@yoctoproject.org
Date: Mon,  7 Mar 2016 12:21:39 +0000
Message-Id: <1457353301-698-2-git-send-email-elliot.smith@intel.com>
In-Reply-To: <1457353301-698-1-git-send-email-elliot.smith@intel.com>
References: <1457353301-698-1-git-send-email-elliot.smith@intel.com>
Subject: [PATCH 1/3] toaster.bbclass: improve how we gather buildstats for Toaster
Precedence: list

Clean up the code which gathers buildstats for Toaster,
and modify the field names so that the correct parts of the
buildstats files are used to derive the CPU usage values.

Also derive elapsed time for the build here, rather than in
Toaster, as we have ready access to the data in the correct
format.

[YOCTO #8842]

Signed-off-by: Elliot Smith <elliot.smith@intel.com>
---
 meta/classes/toaster.bbclass | 70 ++++++++++++++++++++++++++++----------------
 1 file changed, 45 insertions(+), 25 deletions(-)

diff --git a/meta/classes/toaster.bbclass b/meta/classes/toaster.bbclass
index 51a4c74..b7e8773 100644
--- a/meta/classes/toaster.bbclass
+++ b/meta/classes/toaster.bbclass
@@ -202,25 +202,37 @@ python toaster_collect_task_stats() {
     import bb.utils
     import os
 
+    toaster_statlist_file = os.path.join(e.data.getVar('BUILDSTATS_BASE', True), "toasterstatlist")
+
     if not e.data.getVar('BUILDSTATS_BASE', True):
         return  # if we don't have buildstats, we cannot collect stats
 
+    def stat_to_float(value):
+        return float(value.strip('% \n\r'))
+
     def _append_read_list(v):
         lock = bb.utils.lockfile(e.data.expand("${TOPDIR}/toaster.lock"), False, True)
 
-        with open(os.path.join(e.data.getVar('BUILDSTATS_BASE', True), "toasterstatlist"), "a") as fout:
+        with open(toaster_statlist_file, "a") as fout:
             taskdir = e.data.expand("${BUILDSTATS_BASE}/${BUILDNAME}/${PF}")
             fout.write("%s::%s::%s::%s\n" % (e.taskfile, e.taskname, os.path.join(taskdir, e.task), e.data.expand("${PN}")))
 
         bb.utils.unlockfile(lock)
 
     def _read_stats(filename):
-        cpu_usage = 0
-        disk_io = 0
-        started = '0'
-        ended = '0'
-        pn = ''
+        # seconds
+        cpu_time_user = 0
+        cpu_time_system = 0
+
+        # bytes
+        disk_io_read = 0
+        disk_io_write = 0
+
+        started = 0
+        ended = 0
+
         taskname = ''
+
         statinfo = {}
 
         with open(filename, 'r') as task_bs:
@@ -228,41 +240,49 @@ python toaster_collect_task_stats() {
                 k,v = line.strip().split(": ", 1)
                 statinfo[k] = v
 
-        if "CPU usage" in statinfo:
-            cpu_usage = str(statinfo["CPU usage"]).strip('% \n\r')
-
-        if "IO write_bytes" in statinfo:
-            disk_io = disk_io + int(statinfo["IO write_bytes"].strip('% \n\r'))
-
-        if "IO read_bytes" in statinfo:
-            disk_io = disk_io + int(statinfo["IO read_bytes"].strip('% \n\r'))
-
         if "Started" in statinfo:
-            started = str(statinfo["Started"]).strip('% \n\r')
+            started = stat_to_float(statinfo["Started"])
 
         if "Ended" in statinfo:
-            ended = str(statinfo["Ended"]).strip('% \n\r')
+            ended = stat_to_float(statinfo["Ended"])
 
-        elapsed_time = float(ended) - float(started)
+        if "Child rusage ru_utime" in statinfo:
+            cpu_time_user = cpu_time_user + stat_to_float(statinfo["Child rusage ru_utime"])
 
-        cpu_usage = float(cpu_usage)
+        if "Child rusage ru_stime" in statinfo:
+            cpu_time_system = cpu_time_system + stat_to_float(statinfo["Child rusage ru_stime"])
 
-        return {'cpu_usage': cpu_usage, 'disk_io': disk_io, 'elapsed_time': elapsed_time}
+        if "IO write_bytes" in statinfo:
+            write_bytes = int(statinfo["IO write_bytes"].strip('% \n\r'))
+            disk_io_write = disk_io_write + write_bytes
 
+        if "IO read_bytes" in statinfo:
+            read_bytes = int(statinfo["IO read_bytes"].strip('% \n\r'))
+            disk_io_read = disk_io_read + read_bytes
+
+        return {
+            'stat_file': filename,
+            'cpu_time_user': cpu_time_user,
+            'cpu_time_system': cpu_time_system,
+            'disk_io_read': disk_io_read,
+            'disk_io_write': disk_io_write,
+            'started': started,
+            'ended': ended
+        }
 
     if isinstance(e, (bb.build.TaskSucceeded, bb.build.TaskFailed)):
         _append_read_list(e)
         pass
 
-
-    if isinstance(e, bb.event.BuildCompleted) and os.path.exists(os.path.join(e.data.getVar('BUILDSTATS_BASE', True), "toasterstatlist")):
+    if isinstance(e, bb.event.BuildCompleted) and os.path.exists(toaster_statlist_file):
         events = []
-        with open(os.path.join(e.data.getVar('BUILDSTATS_BASE', True), "toasterstatlist"), "r") as fin:
+        with open(toaster_statlist_file, "r") as fin:
             for line in fin:
                 (taskfile, taskname, filename, recipename) = line.strip().split("::")
-                events.append((taskfile, taskname, _read_stats(filename), recipename))
+                stats = _read_stats(filename)
+                events.append((taskfile, taskname, stats, recipename))
         bb.event.fire(bb.event.MetadataEvent("BuildStatsList", events), e.data)
-        os.unlink(os.path.join(e.data.getVar('BUILDSTATS_BASE', True), "toasterstatlist"))
+        #os.unlink(toaster_statlist_file)
 }
 
 # dump relevant build history data as an event when the build is completed
-- 
Elliot Smith
Software Engineer
Intel OTC

---------------------------------------------------------------------
Intel Corporation (UK) Limited
Registered No. 1134945 (England)
Registered Office: Pipers Way, Swindon SN3 1RJ
VAT No: 860 2173 47

This e-mail and any attachments may contain confidential material for
the sole use of the intended recipient(s). Any review or distribution
by others is strictly prohibited. If you are not the intended
recipient, please contact the sender and delete all copies.