Fix task buildstats gathering

Clean up the code which gathers buildstats for Toaster, and modify the field names so that the correct parts of the buildstats files are used to derive the CPU usage values.

Also derive elapsed time for the build here, rather than in Toaster, as we have ready access to the data in the correct format.

[YOCTO #8842]

Signed-off-by Elliot Smith <[email protected]>

---
  meta/classes/toaster.bbclass | 70 ++++++++++++++++++++++++++++----------------
  1 file changed, 45 insertions(+), 25 deletions(-)

meta/classes/toaster.bbclass +41 -22

index 51a4c74..b7e8773 100644
@@ -202,25 +202,37 @@ python toaster_collect_task_stats() {
      import bb.utils
      import os
 
+    toaster_statlist_file = os.path.join(e.data.getVar('BUILDSTATS_BASE', True), "toasterstatlist")
+
      if not e.data.getVar('BUILDSTATS_BASE', True):
          return  # if we don't have buildstats, we cannot collect stats
 
+    def stat_to_float(value):
+        return float(value.strip('% \n\r'))
+
      def _append_read_list(v):
          lock = bb.utils.lockfile(e.data.expand("${TOPDIR}/toaster.lock"), False, True)
 
-        with open(os.path.join(e.data.getVar('BUILDSTATS_BASE', True), "toasterstatlist"), "a") as fout:
+        with open(toaster_statlist_file, "a") as fout:
              taskdir = e.data.expand("${BUILDSTATS_BASE}/${BUILDNAME}/${PF}")
              fout.write("%s::%s::%s::%s\n" % (e.taskfile, e.taskname, os.path.join(taskdir, e.task), e.data.expand("${PN}")))
 
          bb.utils.unlockfile(lock)
 
      def _read_stats(filename):
-        cpu_usage = 0
-        disk_io = 0
-        started = '0'
-        ended = '0'
-        pn = ''
+        # seconds
+        cpu_time_user = 0
+        cpu_time_system = 0
+
+        # bytes
+        disk_io_read = 0
+        disk_io_write = 0
+
+        started = 0
+        ended = 0
+
          taskname = ''
+
          statinfo = {}
 
          with open(filename, 'r') as task_bs:

@@ -228,41 +240,49 @@ python toaster_collect_task_stats() {
                  k,v = line.strip().split(": ", 1)
                  statinfo[k] = v
 
-        if "CPU usage" in statinfo:
-            cpu_usage = str(statinfo["CPU usage"]).strip('% \n\r')
-
-        if "IO write_bytes" in statinfo:
-            disk_io = disk_io + int(statinfo["IO write_bytes"].strip('% \n\r'))
-
-        if "IO read_bytes" in statinfo:
-            disk_io = disk_io + int(statinfo["IO read_bytes"].strip('% \n\r'))
-
          if "Started" in statinfo:
-            started = str(statinfo["Started"]).strip('% \n\r')
+            started = stat_to_float(statinfo["Started"])
 
          if "Ended" in statinfo:
-            ended = str(statinfo["Ended"]).strip('% \n\r')
+            ended = stat_to_float(statinfo["Ended"])
 
-        elapsed_time = float(ended) - float(started)
+        if "Child rusage ru_utime" in statinfo:
+            cpu_time_user = cpu_time_user + stat_to_float(statinfo["Child rusage ru_utime"])
 
-        cpu_usage = float(cpu_usage)
+        if "Child rusage ru_stime" in statinfo:
+            cpu_time_system = cpu_time_system + stat_to_float(statinfo["Child rusage ru_stime"])
 
-        return {'cpu_usage': cpu_usage, 'disk_io': disk_io, 'elapsed_time': elapsed_time}
+        if "IO write_bytes" in statinfo:
+            write_bytes = int(statinfo["IO write_bytes"].strip('% \n\r'))
+            disk_io_write = disk_io_write + write_bytes
 
+        if "IO read_bytes" in statinfo:
+            read_bytes = int(statinfo["IO read_bytes"].strip('% \n\r'))
+            disk_io_read = disk_io_read + read_bytes
+
+        return {
+            'stat_file': filename,
+            'cpu_time_user': cpu_time_user,
+            'cpu_time_system': cpu_time_system,
+            'disk_io_read': disk_io_read,
+            'disk_io_write': disk_io_write,
+            'started': started,
+            'ended': ended
+        }
 
      if isinstance(e, (bb.build.TaskSucceeded, bb.build.TaskFailed)):
          _append_read_list(e)
          pass
 
-
-    if isinstance(e, bb.event.BuildCompleted) and os.path.exists(os.path.join(e.data.getVar('BUILDSTATS_BASE', True), "toasterstatlist")):
+    if isinstance(e, bb.event.BuildCompleted) and os.path.exists(toaster_statlist_file):
          events = []

Quoted 130 lines · Elliot Smith

Clean up the code which gathers buildstats for Toaster, and modify the field names so that the correct parts of the buildstats files are used to derive the CPU usage values.

Also derive elapsed time for the build here, rather than in Toaster, as we have ready access to the data in the correct format.

[YOCTO #8842]

Signed-off-by Elliot Smith <[email protected]>

---
  meta/classes/toaster.bbclass | 70
++++++++++++++++++++++++++++----------------
  1 file changed, 45 insertions(+), 25 deletions(-)

meta/classes/toaster.bbclass +3 -0

index 51a4c74..b7e8773 100644
@@ -202,25 +202,37 @@ python toaster_collect_task_stats() {
      import bb.utils
      import os
  +    toaster_statlist_file =
 os.path.join(e.data.getVar('BUILDSTATS_BASE', True), "toasterstatlist")
+
      if not e.data.getVar('BUILDSTATS_BASE', True):
          return  # if we don't have buildstats, we cannot collect stats
  +    def stat_to_float(value):
+        return float(value.strip('% \n\r'))
+
      def _append_read_list(v):
          lock =
 bb.utils.lockfile(e.data.expand("${TOPDIR}/toaster.lock"), False, True)
  -        with open(os.path.join(e.data.getVar('BUILDSTATS_BASE', True),

"toasterstatlist"), "a") as fout:
+ with open(toaster_statlist_file, "a") as fout:
taskdir =
e.data.expand("${BUILDSTATS_BASE}/${BUILDNAME}/${PF}")

              fout.write("%s::%s::%s::%s\n" % (e.taskfile, e.taskname,

os.path.join(taskdir, e.task), e.data.expand("${PN}")))
bb.utils.unlockfile(lock)
def _read_stats(filename):

meta/classes/toaster.bbclass +33 -16

-        cpu_usage = 0
-        disk_io = 0
-        started = '0'
-        ended = '0'
-        pn = ''
+        # seconds
+        cpu_time_user = 0
+        cpu_time_system = 0
+
+        # bytes
+        disk_io_read = 0
+        disk_io_write = 0
+
+        started = 0
+        ended = 0
+
          taskname = ''
+
          statinfo = {}
            with open(filename, 'r') as task_bs:

@@ -228,41 +240,49 @@ python toaster_collect_task_stats() {
                  k,v = line.strip().split(": ", 1)
                  statinfo[k] = v
  -        if "CPU usage" in statinfo:
-            cpu_usage = str(statinfo["CPU usage"]).strip('% \n\r')
-
-        if "IO write_bytes" in statinfo:
-            disk_io = disk_io + int(statinfo["IO write_bytes"].strip('%
 \n\r'))
-
-        if "IO read_bytes" in statinfo:
-            disk_io = disk_io + int(statinfo["IO read_bytes"].strip('%
 \n\r'))
-
          if "Started" in statinfo:
-            started = str(statinfo["Started"]).strip('% \n\r')
+            started = stat_to_float(statinfo["Started"])
            if "Ended" in statinfo:
-            ended = str(statinfo["Ended"]).strip('% \n\r')
+            ended = stat_to_float(statinfo["Ended"])
  -        elapsed_time = float(ended) - float(started)
+        if "Child rusage ru_utime" in statinfo:
+            cpu_time_user = cpu_time_user +
 stat_to_float(statinfo["Child rusage ru_utime"])
  -        cpu_usage = float(cpu_usage)
+        if "Child rusage ru_stime" in statinfo:
+            cpu_time_system = cpu_time_system +
 stat_to_float(statinfo["Child rusage ru_stime"])
  -        return {'cpu_usage': cpu_usage, 'disk_io': disk_io,
 'elapsed_time': elapsed_time}
+        if "IO write_bytes" in statinfo:
+            write_bytes = int(statinfo["IO write_bytes"].strip('% \n\r'))
+            disk_io_write = disk_io_write + write_bytes
  +        if "IO read_bytes" in statinfo:
+            read_bytes = int(statinfo["IO read_bytes"].strip('% \n\r'))
+            disk_io_read = disk_io_read + read_bytes
+
+        return {
+            'stat_file': filename,
+            'cpu_time_user': cpu_time_user,
+            'cpu_time_system': cpu_time_system,
+            'disk_io_read': disk_io_read,
+            'disk_io_write': disk_io_write,
+            'started': started,
+            'ended': ended
+        }
        if isinstance(e, (bb.build.TaskSucceeded, bb.build.TaskFailed)):
          _append_read_list(e)
          pass
  -
-    if isinstance(e, bb.event.BuildCompleted) and
 os.path.exists(os.path.join(e.data.getVar('BUILDSTATS_BASE', True),

"toasterstatlist")):
+ if isinstance(e, bb.event.BuildCompleted) and
os.path.exists(toaster_statlist_file):

          events = []

- with open(os.path.join(e.data.getVar('BUILDSTATS_BASE', True),
"toasterstatlist"), "r") as fin:
+ with open(toaster_statlist_file, "r") as fin:
for line in fin:
(taskfile, taskname, filename, recipename) =
line.strip().split("::")
- events.append((taskfile, taskname,
_read_stats(filename), recipename))

meta/classes/toaster.bbclass +2 -0

+                stats = _read_stats(filename)
+                events.append((taskfile, taskname, stats, recipename))
          bb.event.fire(bb.event.MetadataEvent("BuildStatsList", events),

Quoted 5 lines

e.data)
- os.unlink(os.path.join(e.data.getVar('BUILDSTATS_BASE', True),
"toasterstatlist"))
+ #os.unlink(toaster_statlist_file)

Did you mean to leave that 'unlink' commented out?

Quoted 4 lines

}

    # dump relevant build history data as an event when the build is
completed

---------------------------------------------------------------------
Intel Corporation (UK) Limited
Registered No. 1134945 (England)
Registered Office: Pipers Way, Swindon SN3 1RJ
VAT No: 860 2173 47

This e-mail and any attachments may contain confidential material for the sole use of the intended recipient(s). Any review or distribution by others is strictly prohibited. If you are not the intended recipient, please contact the sender and delete all copies.

--
_______________________________________________
toaster mailing list
[email protected]
https://lists.yoctoproject.org/listinfo/toaster

The data available from buildstats is now more fine grained than previously, so take advantage of that to enrich the data we save against tasks:

* Store the CPU usage for user and system separately, and display them separately.
* Disk IO is now measured in bytes, not ms. Also store the read/write bytes separately.
* Store started and ended times, as well as elapsed_time. This will enable future features such as showing which tasks were running at a particular point in the build.

There was also a problem with how we were looking up the Task object, which meant that the buildstats were being added to new tasks which weren't correctly associated with the build. Fix how we look up the Task (only looking for tasks which match the build, and the task and recipe names in the build stats data) so the build stats are associated with the correct task.

[YOCTO #8842]

Signed-off-by Elliot Smith <[email protected]>

---
  bitbake/lib/bb/ui/buildinfohelper.py               | 72 ++++++++++------------
  .../orm/migrations/0005_task_field_separation.py   | 48 +++++++++++++++
  bitbake/lib/toaster/orm/models.py                  | 18 +++++-
  .../toastergui/templates/basebuildpage.html        |  4 +-
  bitbake/lib/toaster/toastergui/templates/task.html | 23 ++++---
  .../lib/toaster/toastergui/templates/tasks.html    | 20 +++---
  bitbake/lib/toaster/toastergui/urls.py             |  2 +-
  bitbake/lib/toaster/toastergui/views.py            | 55 +++++++++++------
  8 files changed, 163 insertions(+), 79 deletions(-)

create mode 100644 bitbake/lib/toaster/orm/migrations/0005_task_field_separation.py

bitbake/lib/bb/ui/buildinfohelper.py +1 -0

index 81abede..7fedb76 100644
@@ -50,6 +50,7 @@ from bb.msg import BBLogFormatter as formatter
  from django.db import models
  from pprint import pformat
  import logging
+from datetime import datetime, timedelta
 
  from django.db import transaction, connection
 

bitbake/lib/bb/ui/buildinfohelper.py +27 -39

+
      # pylint: disable=no-self-use
      # we disable detection of no self use in functions because the methods actually work on the object
      # even if they don't touch self anywhere

@@ -223,6 +230,28 @@ class ORMWrapper(object):
          target.license_manifest_path = license_manifest_path
          target.save()
 
+    def update_task_object(self, build, task_name, recipe_name, task_stats):
+        """
+        Find the task for build which matches the recipe and task name
+        to be stored
+        """
+        task_to_update = Task.objects.get(
+            build = build,
+            task_name = task_name,
+            recipe__name = recipe_name
+        )
+
+        task_to_update.started = self._timestamp_to_datetime(task_stats['started'])
+        task_to_update.ended = self._timestamp_to_datetime(task_stats['ended'])
+        task_to_update.elapsed_time = (task_stats['ended'] - task_stats['started'])
+        task_to_update.cpu_time_user = task_stats['cpu_time_user']
+        task_to_update.cpu_time_system = task_stats['cpu_time_system']
+        task_to_update.disk_io_read = task_stats['disk_io_read']
+        task_to_update.disk_io_write = task_stats['disk_io_write']
+        task_to_update.disk_io = task_stats['disk_io_read'] + task_stats['disk_io_write']
+
+        task_to_update.save()
+
      def get_update_task_object(self, task_information, must_exist = False):
          assert 'build' in task_information
          assert 'recipe' in task_information

@@ -259,14 +288,6 @@ class ORMWrapper(object):
                  task_object.sstate_result = Task.SSTATE_FAILED
                  object_changed = True
 
-        # mark down duration if we have a start time and a current time
-        if 'start_time' in task_information.keys() and 'end_time' in task_information.keys():
-            duration = task_information['end_time'] - task_information['start_time']
-            task_object.elapsed_time = duration
-            object_changed = True
-            del task_information['start_time']
-            del task_information['end_time']
-
          if object_changed:
              task_object.save()
          return task_object

@@ -1091,31 +1112,11 @@ class BuildInfoHelper(object):
 
 
      def store_tasks_stats(self, event):
-        for (taskfile, taskname, taskstats, recipename) in BuildInfoHelper._get_data_from_event(event):
-            localfilepath = taskfile.split(":")[-1]
-            assert localfilepath.startswith("/")
+        task_data = BuildInfoHelper._get_data_from_event(event)
 
-            recipe_information = self._get_recipe_information_from_taskfile(taskfile)
-            try:
-                if recipe_information['file_path'].startswith(recipe_information['layer_version'].local_path):
-                    recipe_information['file_path'] = recipe_information['file_path'][len(recipe_information['layer_version'].local_path):].lstrip("/")
-
-                recipe_object = Recipe.objects.get(layer_version = recipe_information['layer_version'],
-                            file_path__endswith = recipe_information['file_path'],
-                            name = recipename)
-            except Recipe.DoesNotExist:
-                logger.error("Could not find recipe for recipe_information %s name %s" , pformat(recipe_information), recipename)
-                raise
-
-            task_information = {}
-            task_information['build'] = self.internal_state['build']
-            task_information['recipe'] = recipe_object
-            task_information['task_name'] = taskname
-            task_information['cpu_usage'] = taskstats['cpu_usage']
-            task_information['disk_io'] = taskstats['disk_io']
-            if 'elapsed_time' in taskstats:
-                task_information['elapsed_time'] = taskstats['elapsed_time']
-            self.orm_wrapper.get_update_task_object(task_information)
+        for (task_file, task_name, task_stats, recipe_name) in task_data:
+            build = self.internal_state['build']
+            self.orm_wrapper.update_task_object(build, task_name, recipe_name, task_stats)
 
      def update_and_store_task(self, event):
          assert 'taskfile' in vars(event)

@@ -1137,13 +1138,6 @@ class BuildInfoHelper(object):
          recipe = self.orm_wrapper.get_update_recipe_object(recipe_information, True)
          task_information = self._get_task_information(event,recipe)
 
-        if 'time' in vars(event):
-            if not 'start_time' in self.internal_state['taskdata'][identifier]:
-                self.internal_state['taskdata'][identifier]['start_time'] = event.time
-            else:
-                task_information['end_time'] = event.time
-                task_information['start_time'] = self.internal_state['taskdata'][identifier]['start_time']
-
          task_information['outcome'] = self.internal_state['taskdata'][identifier]['outcome']
 
          if 'logfile' in vars(event):

bitbake/lib/toaster/orm/migrations/0005_task_field_separation.py new file +48 -0

new file mode 100644
index 0000000..fb1196b
@@ -0,0 +1,48 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('orm', '0004_provides'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='task',
+            name='cpu_usage',
+        ),
+        migrations.AddField(
+            model_name='task',
+            name='cpu_time_system',
+            field=models.DecimalField(null=True, max_digits=8, decimal_places=2),
+        ),
+        migrations.AddField(
+            model_name='task',
+            name='cpu_time_user',
+            field=models.DecimalField(null=True, max_digits=8, decimal_places=2),
+        ),
+        migrations.AddField(
+            model_name='task',
+            name='disk_io_read',
+            field=models.IntegerField(null=True),
+        ),
+        migrations.AddField(
+            model_name='task',
+            name='disk_io_write',
+            field=models.IntegerField(null=True),
+        ),
+        migrations.AddField(
+            model_name='task',
+            name='ended',
+            field=models.DateTimeField(null=True),
+        ),
+        migrations.AddField(
+            model_name='task',
+            name='started',
+            field=models.DateTimeField(null=True),
+        ),
+    ]

bitbake/lib/toaster/orm/models.py +16 -2

index 93b5df3..182d355 100644
@@ -720,9 +720,23 @@ class Task(models.Model):
      work_directory = models.FilePathField(max_length=255, blank=True)
      script_type = models.IntegerField(choices=TASK_CODING, default=CODING_NA)
      line_number = models.IntegerField(default=0)
-    disk_io = models.IntegerField(null=True)
-    cpu_usage = models.DecimalField(max_digits=8, decimal_places=2, null=True)
+
+    # start/end times
+    started = models.DateTimeField(null=True)
+    ended = models.DateTimeField(null=True)
+
+    # in seconds; this is stored to enable sorting
      elapsed_time = models.DecimalField(max_digits=8, decimal_places=2, null=True)
+
+    # in bytes; note that disk_io is stored to enable sorting
+    disk_io = models.IntegerField(null=True)
+    disk_io_read = models.IntegerField(null=True)
+    disk_io_write = models.IntegerField(null=True)
+
+    # in seconds
+    cpu_time_user = models.DecimalField(max_digits=8, decimal_places=2, null=True)
+    cpu_time_system = models.DecimalField(max_digits=8, decimal_places=2, null=True)
+
      sstate_result = models.IntegerField(choices=SSTATE_RESULT, default=SSTATE_NA)
      message = models.CharField(max_length=240)
      logfile = models.FilePathField(max_length=255, blank=True)

bitbake/lib/toaster/toastergui/templates/basebuildpage.html +2 -2

index 0dc71f5..ff9433e 100644
@@ -67,8 +67,8 @@
                {% block nav-buildtime %}
                    <li><a href="{% url 'buildtime' build.pk %}">Time</a></li>
                {% endblock %}
-              {% block nav-cpuusage %}
-                  <li><a href="{% url 'cpuusage' build.pk %}">CPU usage</a></li>
+              {% block nav-cputime %}
+                  <li><a href="{% url 'cputime' build.pk %}">CPU time</a></li>
                {% endblock %}
                {% block nav-diskio %}
                    <li><a href="{% url 'diskio' build.pk %}">Disk I/O</a></li>

bitbake/lib/toaster/toastergui/templates/task.html +15 -8

index ef628d9..5768262 100644
@@ -238,7 +238,7 @@
  </dl>
 
  {# Performance section - shown only for executed tasks #}
-{%if task.elapsed_time or task.cpu_usage or task.disk_io %}
+{%if task.elapsed_time or task.cpu_time_user or task.cpu_time_system or task.disk_io %}
      <h2 class="details">Performance</h2>
  {% endif %}
      <dl class="dl-horizontal">

@@ -249,19 +249,26 @@
          </dt>
          <dd>{{task.elapsed_time|format_none_and_zero|floatformat:2}}</dd>
          {% endif %}
-        {% if task.cpu_usage > 0 %}
+        {% if task.cpu_time_user > 0 %}
          <dt>
-            <i class="icon-question-sign get-help" title="The percentage of task CPU utilization"></i>
-            CPU usage
+            <i class="icon-question-sign get-help" title="Total amount of time spent executing in user mode, in seconds. Note that this time can be greater than the task time due to parallel execution."></i>
+            User CPU time (secs)
          </dt>
-        <dd>{{task.cpu_usage|format_none_and_zero|floatformat:2}}%</dd>
+        <dd>{{task.cpu_time_user|format_none_and_zero|floatformat:2}}</dd>
+        {% endif %}
+        {% if task.cpu_time_system > 0 %}
+        <dt>
+            <i class="icon-question-sign get-help" title="Total amount of time spent executing in kernel mode, in seconds. Note that this time can be greater than the task time due to parallel execution."></i>
+            System CPU time (secs)
+        </dt>
+        <dd>{{task.cpu_time_system|format_none_and_zero|floatformat:2}}</dd>
          {% endif %}
          {% if task.disk_io > 0 %}
          <dt>
-            <i class="icon-question-sign get-help" title="Number of miliseconds the task spent doing disk input and output"></i>
-            Disk I/O (ms)
+            <i class="icon-question-sign get-help" title="Number of bytes written to and read from the disk during the task"></i>
+            Disk I/O (bytes)
          </dt>
-        <dd>{{task.disk_io|format_none_and_zero}}</dd>
+        <dd>{{task.disk_io|format_none_and_zero|intcomma}}</dd>
          {% endif %}
      </dl>
 

bitbake/lib/toaster/toastergui/templates/tasks.html +13 -7

index 353410f..23eb957 100644
@@ -1,4 +1,5 @@
  {% extends "basebuildpage.html" %}
+{% load humanize %}
  {% load projecttags %}
 
  {% block title %} {{mainheading}} - {{build.target_set.all|dictsort:"target"|join:", "}} {{build.machine}} - {{build.project.name}} - Toaster{% endblock %}

@@ -20,13 +21,15 @@
      <li><a href="{% url 'buildtime' build.pk %}">Time</a></li>
    {% endif %}
  {% endblock %}
-{% block nav-cpuusage %}
-  {% if 'CPU usage' == mainheading %}
-    <li class="active"><a href="{% url 'cpuusage' build.pk %}">CPU usage</a></li>
+
+{% block nav-cputime %}
+  {% if 'CPU time' == mainheading %}
+    <li class="active"><a href="{% url 'cputime' build.pk %}">CPU time</a></li>
    {% else %}
-    <li><a href="{% url 'cpuusage' build.pk %}">CPU usage</a></li>
+    <li><a href="{% url 'cputime' build.pk %}">CPU time</a></li>
    {% endif %}
  {% endblock %}
+
  {% block nav-diskio %}
    {% if 'Disk I/O' == mainheading %}
      <li class="active"><a href="{% url 'diskio' build.pk %}">Disk I/O</a></li>

@@ -107,11 +110,14 @@
              <td class="time_taken">
                  {{task.elapsed_time|format_none_and_zero|floatformat:2}}
              </td>
-            <td class="cpu_used">
-                {{task.cpu_usage|format_none_and_zero|floatformat:2}}{% if task.cpu_usage %}%{% endif %}
+            <td class="cpu_time_system">
+                {{task.cpu_time_system|format_none_and_zero|floatformat:2}}
+            </td>
+            <td class="cpu_time_user">
+                {{task.cpu_time_user|format_none_and_zero|floatformat:2}}
              </td>
              <td class="disk_io">
-                {{task.disk_io|format_none_and_zero}}
+                {{task.disk_io|format_none_and_zero|intcomma}}
              </td>
 
          </tr>

bitbake/lib/toaster/toastergui/urls.py +1 -1

index 4aa6488..400580a 100644
@@ -64,7 +64,7 @@ urlpatterns = patterns('toastergui.views',
          url(r'^build/(?P<build_id>\d+)/configuration$', 'configuration', name='configuration'),
          url(r'^build/(?P<build_id>\d+)/configvars$', 'configvars', name='configvars'),
          url(r'^build/(?P<build_id>\d+)/buildtime$', 'buildtime', name='buildtime'),
-        url(r'^build/(?P<build_id>\d+)/cpuusage$', 'cpuusage', name='cpuusage'),
+        url(r'^build/(?P<build_id>\d+)/cputime$', 'cputime', name='cputime'),
          url(r'^build/(?P<build_id>\d+)/diskio$', 'diskio', name='diskio'),
 
          # image information dir

bitbake/lib/toaster/toastergui/views.py +33 -17

index bd11892..85ca9be 100755
@@ -1005,11 +1005,11 @@ def tasks_common(request, build_id, variant, task_anchor):
          object_search_display="disk I/O data"
          filter_search_display="tasks"
          (pagesize, orderby) = _get_parameters_values(request, 25, 'disk_io:-')
-    elif 'cpuusage'  == variant:
-        title_variant='CPU usage'
-        object_search_display="CPU usage data"
+    elif 'cputime'  == variant:
+        title_variant='CPU time'
+        object_search_display="CPU time data"
          filter_search_display="tasks"
-        (pagesize, orderby) = _get_parameters_values(request, 25, 'cpu_usage:-')
+        (pagesize, orderby) = _get_parameters_values(request, 25, 'cpu_time_system:-')
      else :
          title_variant='Tasks'
          object_search_display="tasks"

@@ -1161,23 +1161,38 @@ def tasks_common(request, build_id, variant, task_anchor):
          del tc_time['clclass']
          tc_cache['hidden']='1'
 
-    tc_cpu={
-        'name':'CPU usage',
-        'qhelp':'The percentage of task CPU utilization',
-        'orderfield': _get_toggle_order(request, "cpu_usage", True),
-        'ordericon':_get_toggle_order_icon(request, "cpu_usage"),
-        'orderkey' : 'cpu_usage',
-        'clclass': 'cpu_used', 'hidden' : 1,
+    tc_cpu_time_system={
+        'name':'System CPU time (secs)',
+        'qhelp':'Total amount of time spent executing in kernel mode, in ' +
+                'seconds. Note that this time can be greater than the task ' +
+                'time due to parallel execution.',
+        'orderfield': _get_toggle_order(request, "cpu_time_system", True),
+        'ordericon':_get_toggle_order_icon(request, "cpu_time_system"),
+        'orderkey' : 'cpu_time_system',
+        'clclass': 'cpu_time_system', 'hidden' : 1,
      }
 
-    if  'cpuusage' == variant:
-        tc_cpu['hidden']='0'
-        del tc_cpu['clclass']
+    tc_cpu_time_user={
+        'name':'User CPU time (secs)',
+        'qhelp':'Total amount of time spent executing in user mode, in seconds. ' +
+                'Note that this time can be greater than the task time due to ' +
+                'parallel execution.',
+        'orderfield': _get_toggle_order(request, "cpu_time_user", True),
+        'ordericon':_get_toggle_order_icon(request, "cpu_time_user"),
+        'orderkey' : 'cpu_time_user',
+        'clclass': 'cpu_time_user', 'hidden' : 1,
+    }
+
+    if 'cputime' == variant:
+        tc_cpu_time_system['hidden']='0'
+        tc_cpu_time_user['hidden']='0'
+        del tc_cpu_time_system['clclass']
+        del tc_cpu_time_user['clclass']
          tc_cache['hidden']='1'
 
      tc_diskio={
-        'name':'Disk I/O (ms)',
-        'qhelp':'Number of miliseconds the task spent doing disk input and output',
+        'name':'Disk I/O (bytes)',
+        'qhelp':'Number of bytes written to and read from the disk during the task',
          'orderfield': _get_toggle_order(request, "disk_io", True),
          'ordericon':_get_toggle_order_icon(request, "disk_io"),
          'orderkey' : 'disk_io',

@@ -1208,7 +1223,8 @@ def tasks_common(request, build_id, variant, task_anchor):
                      tc_outcome,
                      tc_cache,
                      tc_time,
-                    tc_cpu,
+                    tc_cpu_time_system,
+                    tc_cpu_time_user,
                      tc_diskio,
                  ]}
 

@@ -1229,9 +1245,8 @@ def buildtime(request, build_id):
  def diskio(request, build_id):

Quoted 52 lines · Elliot Smith

The data available from buildstats is now more fine grained than previously, so take advantage of that to enrich the data we save against tasks:

* Store the CPU usage for user and system separately, and display them separately.
* Disk IO is now measured in bytes, not ms. Also store the read/write bytes separately.
* Store started and ended times, as well as elapsed_time. This will enable future features such as showing which tasks were running at a particular point in the build.

There was also a problem with how we were looking up the Task object, which meant that the buildstats were being added to new tasks which weren't correctly associated with the build. Fix how we look up the Task (only looking for tasks which match the build, and the task and recipe names in the build stats data) so the build stats are associated with the correct task.

[YOCTO #8842]

Signed-off-by Elliot Smith <[email protected]>

---
  bitbake/lib/bb/ui/buildinfohelper.py               | 72
++++++++++------------
  .../orm/migrations/0005_task_field_separation.py   | 48 +++++++++++++++
  bitbake/lib/toaster/orm/models.py                  | 18 +++++-
  .../toastergui/templates/basebuildpage.html        |  4 +-
  bitbake/lib/toaster/toastergui/templates/task.html | 23 ++++---
  .../lib/toaster/toastergui/templates/tasks.html    | 20 +++---
  bitbake/lib/toaster/toastergui/urls.py             |  2 +-
  bitbake/lib/toaster/toastergui/views.py            | 55
+++++++++++------
  8 files changed, 163 insertions(+), 79 deletions(-)

create mode 100644
bitbake/lib/toaster/orm/migrations/0005_task_field_separation.py

bitbake/lib/bb/ui/buildinfohelper.py +1 -0

index 81abede..7fedb76 100644
@@ -50,6 +50,7 @@ from bb.msg import BBLogFormatter as formatter
  from django.db import models
  from pprint import pformat
  import logging
+from datetime import datetime, timedelta
    from django.db import transaction, connection
  @@ -120,6 +121,12 @@ class ORMWrapper(object):
            return vars(self)[dictname][key]

+ def _timestamp_to_datetime(self, secs):

bitbake/lib/bb/ui/buildinfohelper.py class ORMWrapper(object): · line 128

Quoted 4 lines

bitbake/lib/bb/ui/buildinfohelper.py +4 -0

+        """
+        Convert timestamp in seconds to Python datetime
+        """
+        return datetime(1970, 1, 1) + timedelta(seconds=secs)

Would it be possible to just use the inbuilt method from timestamp?

e.g. datetime.fromtimestamp(sec)

+

Quoted 449 lines

# pylint: disable=no-self-use
# we disable detection of no self use in functions because the
methods actually work on the object

      # even if they don't touch self anywhere

bitbake/lib/bb/ui/buildinfohelper.py +0 -0

@@ -223,6 +230,28 @@ class ORMWrapper(object):
          target.license_manifest_path = license_manifest_path
          target.save()
  +    def update_task_object(self, build, task_name, recipe_name,

task_stats):

bitbake/lib/bb/ui/buildinfohelper.py +11 -0

+        """
+        Find the task for build which matches the recipe and task name
+        to be stored
+        """
+        task_to_update = Task.objects.get(
+            build = build,
+            task_name = task_name,
+            recipe__name = recipe_name
+        )
+
+        task_to_update.started =

self._timestamp_to_datetime(task_stats['started'])
+ task_to_update.ended =
self._timestamp_to_datetime(task_stats['ended'])
+ task_to_update.elapsed_time = (task_stats['ended'] -
task_stats['started'])

bitbake/lib/bb/ui/buildinfohelper.py +5 -0

+        task_to_update.cpu_time_user = task_stats['cpu_time_user']
+        task_to_update.cpu_time_system = task_stats['cpu_time_system']
+        task_to_update.disk_io_read = task_stats['disk_io_read']
+        task_to_update.disk_io_write = task_stats['disk_io_write']
+        task_to_update.disk_io = task_stats['disk_io_read'] +

task_stats['disk_io_write']

bitbake/lib/bb/ui/buildinfohelper.py +3 -0

+
+        task_to_update.save()
+
      def get_update_task_object(self, task_information, must_exist =

False):
assert 'build' in task_information
assert 'recipe' in task_information

bitbake/lib/bb/ui/buildinfohelper.py +0 -1

@@ -259,14 +288,6 @@ class ORMWrapper(object):
                  task_object.sstate_result = Task.SSTATE_FAILED
                  object_changed = True
  -        # mark down duration if we have a start time and a current time
-        if 'start_time' in task_information.keys() and 'end_time' in

task_information.keys():
- duration = task_information['end_time'] -
task_information['start_time']

bitbake/lib/bb/ui/buildinfohelper.py +0 -6

-            task_object.elapsed_time = duration
-            object_changed = True
-            del task_information['start_time']
-            del task_information['end_time']
-
          if object_changed:
              task_object.save()
          return task_object

@@ -1091,31 +1112,11 @@ class BuildInfoHelper(object):
          def store_tasks_stats(self, event):
-        for (taskfile, taskname, taskstats, recipename) in

BuildInfoHelper._get_data_from_event(event):

bitbake/lib/bb/ui/buildinfohelper.py +1 -2

-            localfilepath = taskfile.split(":")[-1]
-            assert localfilepath.startswith("/")
+        task_data = BuildInfoHelper._get_data_from_event(event)
  -            recipe_information =

self._get_recipe_information_from_taskfile(taskfile)

bitbake/lib/bb/ui/buildinfohelper.py +0 -2

-            try:
-                if

recipe_information['file_path'].startswith(recipe_information['layer_version'].local_path):
- recipe_information['file_path'] =
recipe_information['file_path'][len(recipe_information['layer_version'].local_path):].lstrip("/")

bitbake/lib/bb/ui/buildinfohelper.py +0 -2

-
-                recipe_object = Recipe.objects.get(layer_version =

recipe_information['layer_version'],
- file_path__endswith =
recipe_information['file_path'],

bitbake/lib/bb/ui/buildinfohelper.py +0 -3

-                            name = recipename)
-            except Recipe.DoesNotExist:
-                logger.error("Could not find recipe for

recipe_information %s name %s" , pformat(recipe_information), recipename)

bitbake/lib/bb/ui/buildinfohelper.py +0 -10

-                raise
-
-            task_information = {}
-            task_information['build'] = self.internal_state['build']
-            task_information['recipe'] = recipe_object
-            task_information['task_name'] = taskname
-            task_information['cpu_usage'] = taskstats['cpu_usage']
-            task_information['disk_io'] = taskstats['disk_io']
-            if 'elapsed_time' in taskstats:
-                task_information['elapsed_time'] =

taskstats['elapsed_time']

bitbake/lib/bb/ui/buildinfohelper.py +3 -1

-            self.orm_wrapper.get_update_task_object(task_information)
+        for (task_file, task_name, task_stats, recipe_name) in task_data:
+            build = self.internal_state['build']
+            self.orm_wrapper.update_task_object(build, task_name,

recipe_name, task_stats)
def update_and_store_task(self, event):
assert 'taskfile' in vars(event)

bitbake/lib/bb/ui/buildinfohelper.py +0 -1

@@ -1137,13 +1138,6 @@ class BuildInfoHelper(object):
          recipe =
 self.orm_wrapper.get_update_recipe_object(recipe_information, True)
          task_information = self._get_task_information(event,recipe)
  -        if 'time' in vars(event):
-            if not 'start_time' in

self.internal_state['taskdata'][identifier]:
-
self.internal_state['taskdata'][identifier]['start_time'] = event.time

bitbake/lib/bb/ui/buildinfohelper.py +0 -3

-            else:
-                task_information['end_time'] = event.time
-                task_information['start_time'] =

self.internal_state['taskdata'][identifier]['start_time'] -
task_information['outcome'] =
self.internal_state['taskdata'][identifier]['outcome']
if 'logfile' in vars(event):
diff --git a/bitbake/lib/toaster/orm/migrations/0005_task_field_separation.py b/bitbake/lib/toaster/orm/migrations/0005_task_field_separation.py new file mode 100644
index 0000000..fb1196b

bitbake/lib/toaster/orm/migrations/0005_task_field_separation.py new file +21 -0

@@ -0,0 +1,48 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('orm', '0004_provides'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='task',
+            name='cpu_usage',
+        ),
+        migrations.AddField(
+            model_name='task',
+            name='cpu_time_system',
+            field=models.DecimalField(null=True, max_digits=8,

decimal_places=2),

bitbake/lib/toaster/orm/migrations/0005_task_field_separation.py new file +5 -0

+        ),
+        migrations.AddField(
+            model_name='task',
+            name='cpu_time_user',
+            field=models.DecimalField(null=True, max_digits=8,

decimal_places=2),

bitbake/lib/toaster/orm/migrations/0005_task_field_separation.py new file +22 -0

+        ),
+        migrations.AddField(
+            model_name='task',
+            name='disk_io_read',
+            field=models.IntegerField(null=True),
+        ),
+        migrations.AddField(
+            model_name='task',
+            name='disk_io_write',
+            field=models.IntegerField(null=True),
+        ),
+        migrations.AddField(
+            model_name='task',
+            name='ended',
+            field=models.DateTimeField(null=True),
+        ),
+        migrations.AddField(
+            model_name='task',
+            name='started',
+            field=models.DateTimeField(null=True),
+        ),
+    ]

bitbake/lib/toaster/orm/models.py +14 -2

index 93b5df3..182d355 100644
@@ -720,9 +720,23 @@ class Task(models.Model):
      work_directory = models.FilePathField(max_length=255, blank=True)
      script_type = models.IntegerField(choices=TASK_CODING,
 default=CODING_NA)
      line_number = models.IntegerField(default=0)
-    disk_io = models.IntegerField(null=True)
-    cpu_usage = models.DecimalField(max_digits=8, decimal_places=2,
 null=True)
+
+    # start/end times
+    started = models.DateTimeField(null=True)
+    ended = models.DateTimeField(null=True)
+
+    # in seconds; this is stored to enable sorting
      elapsed_time = models.DecimalField(max_digits=8, decimal_places=2,
 null=True)
+
+    # in bytes; note that disk_io is stored to enable sorting
+    disk_io = models.IntegerField(null=True)
+    disk_io_read = models.IntegerField(null=True)
+    disk_io_write = models.IntegerField(null=True)
+
+    # in seconds
+    cpu_time_user = models.DecimalField(max_digits=8, decimal_places=2,

null=True)
+ cpu_time_system = models.DecimalField(max_digits=8,
decimal_places=2, null=True)
+
sstate_result = models.IntegerField(choices=SSTATE_RESULT,
default=SSTATE_NA)
message = models.CharField(max_length=240) logfile = models.FilePathField(max_length=255, blank=True)

bitbake/lib/toaster/toastergui/templates/basebuildpage.html +0 -0

index 0dc71f5..ff9433e 100644
@@ -67,8 +67,8 @@
                {% block nav-buildtime %}
                    <li><a href="{% url 'buildtime' build.pk

%}">Time</a></li>

                {% endblock %}

bitbake/lib/toaster/toastergui/templates/basebuildpage.html +0 -2

-              {% block nav-cpuusage %}
-                  <li><a href="{% url 'cpuusage' build.pk %}">CPU

usage</a></li>

bitbake/lib/toaster/toastergui/templates/basebuildpage.html +2 -0

+              {% block nav-cputime %}
+                  <li><a href="{% url 'cputime' build.pk %}">CPU

time</a></li>

                {% endblock %}
                {% block nav-diskio %}
                    <li><a href="{% url 'diskio' build.pk %}">Disk

I/O</a></li>

bitbake/lib/toaster/toastergui/templates/task.html +1 -1

index ef628d9..5768262 100644
@@ -238,7 +238,7 @@
  </dl>
    {# Performance section - shown only for executed tasks #}
-{%if task.elapsed_time or task.cpu_usage or task.disk_io %}
+{%if task.elapsed_time or task.cpu_time_user or task.cpu_time_system or
 task.disk_io %}
      <h2 class="details">Performance</h2>
  {% endif %}
      <dl class="dl-horizontal">

@@ -249,19 +249,26 @@
          </dt>
 

<dd>{{task.elapsed_time|format_none_and_zero|floatformat:2}}</dd>

          {% endif %}

bitbake/lib/toaster/toastergui/templates/task.html +1 -2

-        {% if task.cpu_usage > 0 %}
+        {% if task.cpu_time_user > 0 %}
          <dt>
-            <i class="icon-question-sign get-help" title="The percentage

of task CPU utilization"></i>

bitbake/lib/toaster/toastergui/templates/task.html +1 -1

-            CPU usage
+            <i class="icon-question-sign get-help" title="Total amount

of time spent executing in user mode, in seconds. Note that this time can be greater than the task time due to parallel execution."></i>

bitbake/lib/toaster/toastergui/templates/task.html +2 -1

+            User CPU time (secs)
          </dt>
-        <dd>{{task.cpu_usage|format_none_and_zero|floatformat:2}}%</dd>
+

<dd>{{task.cpu_time_user|format_none_and_zero|floatformat:2}}</dd>

bitbake/lib/toaster/toastergui/templates/task.html +4 -0

+        {% endif %}
+        {% if task.cpu_time_system > 0 %}
+        <dt>
+            <i class="icon-question-sign get-help" title="Total amount

of time spent executing in kernel mode, in seconds. Note that this time can be greater than the task time due to parallel execution."></i>

bitbake/lib/toaster/toastergui/templates/task.html +3 -0

+            System CPU time (secs)
+        </dt>
+

<dd>{{task.cpu_time_system|format_none_and_zero|floatformat:2}}</dd>

          {% endif %}
          {% if task.disk_io > 0 %}
          <dt>

- <i class="icon-question-sign get-help" title="Number of
miliseconds the task spent doing disk input and output"></i>

bitbake/lib/toaster/toastergui/templates/task.html +1 -1

-            Disk I/O (ms)
+            <i class="icon-question-sign get-help" title="Number of

bytes written to and read from the disk during the task"></i>

bitbake/lib/toaster/toastergui/templates/task.html +2 -1

+            Disk I/O (bytes)
          </dt>
-        <dd>{{task.disk_io|format_none_and_zero}}</dd>
+        <dd>{{task.disk_io|format_none_and_zero|intcomma}}</dd>
          {% endif %}
      </dl>
  diff --git a/bitbake/lib/toaster/toastergui/templates/tasks.html

b/bitbake/lib/toaster/toastergui/templates/tasks.html index 353410f..23eb957 100644

bitbake/lib/toaster/toastergui/templates/tasks.html +1 -0

@@ -1,4 +1,5 @@
  {% extends "basebuildpage.html" %}
+{% load humanize %}
  {% load projecttags %}
    {% block title %} {{mainheading}} -
 {{build.target_set.all|dictsort:"target"|join:", "}} {{build.machine}} - {{

build.project.name}} - Toaster{% endblock %}

bitbake/lib/toaster/toastergui/templates/tasks.html +0 -3

@@ -20,13 +21,15 @@
      <li><a href="{% url 'buildtime' build.pk %}">Time</a></li>
    {% endif %}
  {% endblock %}
-{% block nav-cpuusage %}
-  {% if 'CPU usage' == mainheading %}
-    <li class="active"><a href="{% url 'cpuusage' build.pk %}">CPU

usage</a></li>

bitbake/lib/toaster/toastergui/templates/tasks.html +4 -0

+
+{% block nav-cputime %}
+  {% if 'CPU time' == mainheading %}
+    <li class="active"><a href="{% url 'cputime' build.pk %}">CPU

time</a></li>

    {% else %}

bitbake/lib/toaster/toastergui/templates/tasks.html +2 -1

-    <li><a href="{% url 'cpuusage' build.pk %}">CPU usage</a></li>
+    <li><a href="{% url 'cputime' build.pk %}">CPU time</a></li>
    {% endif %}
  {% endblock %}
+
  {% block nav-diskio %}
    {% if 'Disk I/O' == mainheading %}
      <li class="active"><a href="{% url 'diskio' build.pk %}">Disk

I/O</a></li>

bitbake/lib/toaster/toastergui/templates/tasks.html +6 -3

@@ -107,11 +110,14 @@
              <td class="time_taken">
                  {{task.elapsed_time|format_none_and_zero|floatformat:2}}
              </td>
-            <td class="cpu_used">
-                {{task.cpu_usage|format_none_and_zero|floatformat:2}}{%
 if task.cpu_usage %}%{% endif %}
+            <td class="cpu_time_system">
+
 {{task.cpu_time_system|format_none_and_zero|floatformat:2}}
+            </td>
+            <td class="cpu_time_user">
+                {{task.cpu_time_user|format_none_and_zero|floatformat:2}}
              </td>
              <td class="disk_io">
-                {{task.disk_io|format_none_and_zero}}
+                {{task.disk_io|format_none_and_zero|intcomma}}
              </td>

            </tr>

bitbake/lib/toaster/toastergui/urls.py +0 -1

index 4aa6488..400580a 100644
@@ -64,7 +64,7 @@ urlpatterns = patterns('toastergui.views',
          url(r'^build/(?P<build_id>\d+)/configuration$',
 'configuration', name='configuration'),
          url(r'^build/(?P<build_id>\d+)/configvars$', 'configvars',
 name='configvars'),
          url(r'^build/(?P<build_id>\d+)/buildtime$', 'buildtime',
 name='buildtime'),
-        url(r'^build/(?P<build_id>\d+)/cpuusage$', 'cpuusage',

name='cpuusage'),
+ url(r'^build/(?P<build_id>\d+)/cputime$', 'cputime',
name='cputime'),

          url(r'^build/(?P<build_id>\d+)/diskio$', 'diskio',
name='diskio'),
            # image information dir

bitbake/lib/toaster/toastergui/views.py +0 -0

index bd11892..85ca9be 100755
@@ -1005,11 +1005,11 @@ def tasks_common(request, build_id, variant,

task_anchor):
object_search_display="disk I/O data" filter_search_display="tasks"
(pagesize, orderby) = _get_parameters_values(request, 25,
'disk_io:-')

bitbake/lib/toaster/toastergui/views.py +3 -4

-    elif 'cpuusage'  == variant:
-        title_variant='CPU usage'
-        object_search_display="CPU usage data"
+    elif 'cputime'  == variant:
+        title_variant='CPU time'
+        object_search_display="CPU time data"
          filter_search_display="tasks"
-        (pagesize, orderby) = _get_parameters_values(request, 25,

'cpu_usage:-')
+ (pagesize, orderby) = _get_parameters_values(request, 25,
'cpu_time_system:-')
else :
title_variant='Tasks'
object_search_display="tasks"

bitbake/lib/toaster/toastergui/views.py +0 -0

@@ -1161,23 +1161,38 @@ def tasks_common(request, build_id, variant,

task_anchor):

          del tc_time['clclass']
          tc_cache['hidden']='1'

- tc_cpu={

bitbake/lib/toaster/toastergui/views.py +3 -6

-        'name':'CPU usage',
-        'qhelp':'The percentage of task CPU utilization',
-        'orderfield': _get_toggle_order(request, "cpu_usage", True),
-        'ordericon':_get_toggle_order_icon(request, "cpu_usage"),
-        'orderkey' : 'cpu_usage',
-        'clclass': 'cpu_used', 'hidden' : 1,
+    tc_cpu_time_system={
+        'name':'System CPU time (secs)',
+        'qhelp':'Total amount of time spent executing in kernel mode, in

' +
+ 'seconds. Note that this time can be greater than the
task ' +

bitbake/lib/toaster/toastergui/views.py +2 -0

+                'time due to parallel execution.',
+        'orderfield': _get_toggle_order(request, "cpu_time_system",

True),

bitbake/lib/toaster/toastergui/views.py +6 -2

+        'ordericon':_get_toggle_order_icon(request, "cpu_time_system"),
+        'orderkey' : 'cpu_time_system',
+        'clclass': 'cpu_time_system', 'hidden' : 1,
      }
  -    if  'cpuusage' == variant:
-        tc_cpu['hidden']='0'
-        del tc_cpu['clclass']
+    tc_cpu_time_user={
+        'name':'User CPU time (secs)',
+        'qhelp':'Total amount of time spent executing in user mode, in

seconds. ' +
+ 'Note that this time can be greater than the task time
due to ' +

bitbake/lib/toaster/toastergui/views.py +12 -2

+                'parallel execution.',
+        'orderfield': _get_toggle_order(request, "cpu_time_user", True),
+        'ordericon':_get_toggle_order_icon(request, "cpu_time_user"),
+        'orderkey' : 'cpu_time_user',
+        'clclass': 'cpu_time_user', 'hidden' : 1,
+    }
+
+    if 'cputime' == variant:
+        tc_cpu_time_system['hidden']='0'
+        tc_cpu_time_user['hidden']='0'
+        del tc_cpu_time_system['clclass']
+        del tc_cpu_time_user['clclass']
          tc_cache['hidden']='1'
        tc_diskio={
-        'name':'Disk I/O (ms)',
-        'qhelp':'Number of miliseconds the task spent doing disk input

and output',

bitbake/lib/toaster/toastergui/views.py +2 -0

+        'name':'Disk I/O (bytes)',
+        'qhelp':'Number of bytes written to and read from the disk

during the task',

          'orderfield': _get_toggle_order(request, "disk_io", True),
          'ordericon':_get_toggle_order_icon(request, "disk_io"),
          'orderkey' : 'disk_io',

bitbake/lib/toaster/toastergui/views.py +0 -0

@@ -1208,7 +1223,8 @@ def tasks_common(request, build_id, variant,

task_anchor):
tc_outcome,
tc_cache,
tc_time,

bitbake/lib/toaster/toastergui/views.py +2 -1

-                    tc_cpu,
+                    tc_cpu_time_system,
+                    tc_cpu_time_user,
                      tc_diskio,
                  ]}

bitbake/lib/toaster/toastergui/views.py

Quoted 10 lines

bitbake/lib/toaster/toastergui/views.py +2 -2

  @@ -1229,9 +1245,8 @@ def buildtime(request, build_id):
  def diskio(request, build_id):
      return tasks_common(request, build_id, 'diskio', '')
  -def cpuusage(request, build_id):
-    return tasks_common(request, build_id, 'cpuusage', '')
-
+def cputime(request, build_id):
+    return tasks_common(request, build_id, 'cputime', '')
    def recipes(request, build_id):
      template = 'recipes.html'

---------------------------------------------------------------------
Intel Corporation (UK) Limited
Registered No. 1134945 (England)
Registered Office: Pipers Way, Swindon SN3 1RJ
VAT No: 860 2173 47

This e-mail and any attachments may contain confidential material for the sole use of the intended recipient(s). Any review or distribution by others is strictly prohibited. If you are not the intended recipient, please contact the sender and delete all copies.

--
_______________________________________________
toaster mailing list
[email protected]
https://lists.yoctoproject.org/listinfo/toaster

Will do. Didn't realise it existed.

Elliot

Quoted 540 lines · Michael Wood

Quoted 52 lines · Elliot Smith

The data available from buildstats is now more fine grained than previously, so take advantage of that to enrich the data we save against tasks:

* Store the CPU usage for user and system separately, and display them separately.
* Disk IO is now measured in bytes, not ms. Also store the read/write bytes separately.
* Store started and ended times, as well as elapsed_time. This will enable future features such as showing which tasks were running at a particular point in the build.

There was also a problem with how we were looking up the Task object, which meant that the buildstats were being added to new tasks which weren't correctly associated with the build. Fix how we look up the Task (only looking for tasks which match the build, and the task and recipe names in the build stats data) so the build stats are associated with the correct task.

[YOCTO #8842]

Signed-off-by Elliot Smith <[email protected]>

---
  bitbake/lib/bb/ui/buildinfohelper.py               | 72
++++++++++------------
  .../orm/migrations/0005_task_field_separation.py   | 48 +++++++++++++++
  bitbake/lib/toaster/orm/models.py                  | 18 +++++-
  .../toastergui/templates/basebuildpage.html        |  4 +-
  bitbake/lib/toaster/toastergui/templates/task.html | 23 ++++---
  .../lib/toaster/toastergui/templates/tasks.html    | 20 +++---
  bitbake/lib/toaster/toastergui/urls.py             |  2 +-
  bitbake/lib/toaster/toastergui/views.py            | 55
+++++++++++------
  8 files changed, 163 insertions(+), 79 deletions(-)

create mode 100644
bitbake/lib/toaster/orm/migrations/0005_task_field_separation.py

bitbake/lib/bb/ui/buildinfohelper.py +1 -0

index 81abede..7fedb76 100644
@@ -50,6 +50,7 @@ from bb.msg import BBLogFormatter as formatter
  from django.db import models
  from pprint import pformat
  import logging
+from datetime import datetime, timedelta
    from django.db import transaction, connection
  @@ -120,6 +121,12 @@ class ORMWrapper(object):
            return vars(self)[dictname][key]

+ def _timestamp_to_datetime(self, secs):

bitbake/lib/bb/ui/buildinfohelper.py class ORMWrapper(object): · line 128

Quoted 4 lines

bitbake/lib/bb/ui/buildinfohelper.py +4 -0

+        """
+        Convert timestamp in seconds to Python datetime
+        """
+        return datetime(1970, 1, 1) + timedelta(seconds=secs)

Would it be possible to just use the inbuilt method from timestamp?

e.g. datetime.fromtimestamp(sec)

+

Quoted 453 lines

# pylint: disable=no-self-use
# we disable detection of no self use in functions because the
methods actually work on the object

      # even if they don't touch self anywhere

bitbake/lib/bb/ui/buildinfohelper.py +0 -0

@@ -223,6 +230,28 @@ class ORMWrapper(object):
          target.license_manifest_path = license_manifest_path
          target.save()
  +    def update_task_object(self, build, task_name, recipe_name,

task_stats):

bitbake/lib/bb/ui/buildinfohelper.py +11 -0

+        """
+        Find the task for build which matches the recipe and task name
+        to be stored
+        """
+        task_to_update = Task.objects.get(
+            build = build,
+            task_name = task_name,
+            recipe__name = recipe_name
+        )
+
+        task_to_update.started =

self._timestamp_to_datetime(task_stats['started'])
+ task_to_update.ended =
self._timestamp_to_datetime(task_stats['ended'])
+ task_to_update.elapsed_time = (task_stats['ended'] -
task_stats['started'])

bitbake/lib/bb/ui/buildinfohelper.py +5 -0

+        task_to_update.cpu_time_user = task_stats['cpu_time_user']
+        task_to_update.cpu_time_system = task_stats['cpu_time_system']
+        task_to_update.disk_io_read = task_stats['disk_io_read']
+        task_to_update.disk_io_write = task_stats['disk_io_write']
+        task_to_update.disk_io = task_stats['disk_io_read'] +

task_stats['disk_io_write']

bitbake/lib/bb/ui/buildinfohelper.py +3 -0

+
+        task_to_update.save()
+
      def get_update_task_object(self, task_information, must_exist =

False):
assert 'build' in task_information
assert 'recipe' in task_information

bitbake/lib/bb/ui/buildinfohelper.py +0 -0

@@ -259,14 +288,6 @@ class ORMWrapper(object):
                  task_object.sstate_result = Task.SSTATE_FAILED
                  object_changed = True
  -        # mark down duration if we have a start time and a current

time
- if 'start_time' in task_information.keys() and 'end_time' in
task_information.keys():
- duration = task_information['end_time'] -
task_information['start_time']

bitbake/lib/bb/ui/buildinfohelper.py +0 -6

-            task_object.elapsed_time = duration
-            object_changed = True
-            del task_information['start_time']
-            del task_information['end_time']
-
          if object_changed:
              task_object.save()
          return task_object

@@ -1091,31 +1112,11 @@ class BuildInfoHelper(object):
          def store_tasks_stats(self, event):
-        for (taskfile, taskname, taskstats, recipename) in

BuildInfoHelper._get_data_from_event(event):

bitbake/lib/bb/ui/buildinfohelper.py +1 -2

-            localfilepath = taskfile.split(":")[-1]
-            assert localfilepath.startswith("/")
+        task_data = BuildInfoHelper._get_data_from_event(event)
  -            recipe_information =

self._get_recipe_information_from_taskfile(taskfile)

bitbake/lib/bb/ui/buildinfohelper.py +0 -2

-            try:
-                if

recipe_information['file_path'].startswith(recipe_information['layer_version'].local_path):
- recipe_information['file_path'] =
recipe_information['file_path'][len(recipe_information['layer_version'].local_path):].lstrip("/")

bitbake/lib/bb/ui/buildinfohelper.py +0 -2

-
-                recipe_object = Recipe.objects.get(layer_version =

recipe_information['layer_version'],
- file_path__endswith =
recipe_information['file_path'],

bitbake/lib/bb/ui/buildinfohelper.py +0 -3

-                            name = recipename)
-            except Recipe.DoesNotExist:
-                logger.error("Could not find recipe for

recipe_information %s name %s" , pformat(recipe_information), recipename)

bitbake/lib/bb/ui/buildinfohelper.py +0 -10

-                raise
-
-            task_information = {}
-            task_information['build'] = self.internal_state['build']
-            task_information['recipe'] = recipe_object
-            task_information['task_name'] = taskname
-            task_information['cpu_usage'] = taskstats['cpu_usage']
-            task_information['disk_io'] = taskstats['disk_io']
-            if 'elapsed_time' in taskstats:
-                task_information['elapsed_time'] =

taskstats['elapsed_time']

bitbake/lib/bb/ui/buildinfohelper.py +1 -1

-            self.orm_wrapper.get_update_task_object(task_information)
+        for (task_file, task_name, task_stats, recipe_name) in

task_data:

bitbake/lib/bb/ui/buildinfohelper.py +2 -0

+            build = self.internal_state['build']
+            self.orm_wrapper.update_task_object(build, task_name,

recipe_name, task_stats)
def update_and_store_task(self, event):
assert 'taskfile' in vars(event)

bitbake/lib/bb/ui/buildinfohelper.py +0 -1

@@ -1137,13 +1138,6 @@ class BuildInfoHelper(object):
          recipe =
 self.orm_wrapper.get_update_recipe_object(recipe_information, True)
          task_information = self._get_task_information(event,recipe)
  -        if 'time' in vars(event):
-            if not 'start_time' in

self.internal_state['taskdata'][identifier]:
-
self.internal_state['taskdata'][identifier]['start_time'] = event.time

bitbake/lib/bb/ui/buildinfohelper.py +0 -3

-            else:
-                task_information['end_time'] = event.time
-                task_information['start_time'] =

self.internal_state['taskdata'][identifier]['start_time'] -
task_information['outcome'] =
self.internal_state['taskdata'][identifier]['outcome']
if 'logfile' in vars(event):
diff --git a/bitbake/lib/toaster/orm/migrations/0005_task_field_separation.py b/bitbake/lib/toaster/orm/migrations/0005_task_field_separation.py new file mode 100644
index 0000000..fb1196b

bitbake/lib/toaster/orm/migrations/0005_task_field_separation.py new file +21 -0

@@ -0,0 +1,48 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('orm', '0004_provides'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='task',
+            name='cpu_usage',
+        ),
+        migrations.AddField(
+            model_name='task',
+            name='cpu_time_system',
+            field=models.DecimalField(null=True, max_digits=8,

decimal_places=2),

bitbake/lib/toaster/orm/migrations/0005_task_field_separation.py new file +5 -0

+        ),
+        migrations.AddField(
+            model_name='task',
+            name='cpu_time_user',
+            field=models.DecimalField(null=True, max_digits=8,

decimal_places=2),

bitbake/lib/toaster/orm/migrations/0005_task_field_separation.py new file +22 -0

+        ),
+        migrations.AddField(
+            model_name='task',
+            name='disk_io_read',
+            field=models.IntegerField(null=True),
+        ),
+        migrations.AddField(
+            model_name='task',
+            name='disk_io_write',
+            field=models.IntegerField(null=True),
+        ),
+        migrations.AddField(
+            model_name='task',
+            name='ended',
+            field=models.DateTimeField(null=True),
+        ),
+        migrations.AddField(
+            model_name='task',
+            name='started',
+            field=models.DateTimeField(null=True),
+        ),
+    ]

bitbake/lib/toaster/orm/models.py +14 -2

index 93b5df3..182d355 100644
@@ -720,9 +720,23 @@ class Task(models.Model):
      work_directory = models.FilePathField(max_length=255, blank=True)
      script_type = models.IntegerField(choices=TASK_CODING,
 default=CODING_NA)
      line_number = models.IntegerField(default=0)
-    disk_io = models.IntegerField(null=True)
-    cpu_usage = models.DecimalField(max_digits=8, decimal_places=2,
 null=True)
+
+    # start/end times
+    started = models.DateTimeField(null=True)
+    ended = models.DateTimeField(null=True)
+
+    # in seconds; this is stored to enable sorting
      elapsed_time = models.DecimalField(max_digits=8, decimal_places=2,
 null=True)
+
+    # in bytes; note that disk_io is stored to enable sorting
+    disk_io = models.IntegerField(null=True)
+    disk_io_read = models.IntegerField(null=True)
+    disk_io_write = models.IntegerField(null=True)
+
+    # in seconds
+    cpu_time_user = models.DecimalField(max_digits=8, decimal_places=2,

null=True)
+ cpu_time_system = models.DecimalField(max_digits=8,
decimal_places=2, null=True)
+
sstate_result = models.IntegerField(choices=SSTATE_RESULT,
default=SSTATE_NA)
message = models.CharField(max_length=240) logfile = models.FilePathField(max_length=255, blank=True)

bitbake/lib/toaster/toastergui/templates/basebuildpage.html +0 -0

index 0dc71f5..ff9433e 100644
@@ -67,8 +67,8 @@
                {% block nav-buildtime %}
                    <li><a href="{% url 'buildtime' build.pk

%}">Time</a></li>

                {% endblock %}

bitbake/lib/toaster/toastergui/templates/basebuildpage.html +0 -2

-              {% block nav-cpuusage %}
-                  <li><a href="{% url 'cpuusage' build.pk %}">CPU

usage</a></li>

bitbake/lib/toaster/toastergui/templates/basebuildpage.html +2 -0

+              {% block nav-cputime %}
+                  <li><a href="{% url 'cputime' build.pk %}">CPU

time</a></li>

                {% endblock %}
                {% block nav-diskio %}
                    <li><a href="{% url 'diskio' build.pk %}">Disk

I/O</a></li>

bitbake/lib/toaster/toastergui/templates/task.html +1 -1

index ef628d9..5768262 100644
@@ -238,7 +238,7 @@
  </dl>
    {# Performance section - shown only for executed tasks #}
-{%if task.elapsed_time or task.cpu_usage or task.disk_io %}
+{%if task.elapsed_time or task.cpu_time_user or task.cpu_time_system or
 task.disk_io %}
      <h2 class="details">Performance</h2>
  {% endif %}
      <dl class="dl-horizontal">

@@ -249,19 +249,26 @@
          </dt>
 

<dd>{{task.elapsed_time|format_none_and_zero|floatformat:2}}</dd>

          {% endif %}

bitbake/lib/toaster/toastergui/templates/task.html +1 -2

-        {% if task.cpu_usage > 0 %}
+        {% if task.cpu_time_user > 0 %}
          <dt>
-            <i class="icon-question-sign get-help" title="The

percentage of task CPU utilization"></i>

bitbake/lib/toaster/toastergui/templates/task.html +1 -1

-            CPU usage
+            <i class="icon-question-sign get-help" title="Total amount

of time spent executing in user mode, in seconds. Note that this time can be greater than the task time due to parallel execution."></i>

bitbake/lib/toaster/toastergui/templates/task.html +2 -1

+            User CPU time (secs)
          </dt>
-        <dd>{{task.cpu_usage|format_none_and_zero|floatformat:2}}%</dd>
+

<dd>{{task.cpu_time_user|format_none_and_zero|floatformat:2}}</dd>

bitbake/lib/toaster/toastergui/templates/task.html +4 -0

+        {% endif %}
+        {% if task.cpu_time_system > 0 %}
+        <dt>
+            <i class="icon-question-sign get-help" title="Total amount

of time spent executing in kernel mode, in seconds. Note that this time can be greater than the task time due to parallel execution."></i>

bitbake/lib/toaster/toastergui/templates/task.html +3 -0

+            System CPU time (secs)
+        </dt>
+

<dd>{{task.cpu_time_system|format_none_and_zero|floatformat:2}}</dd>

          {% endif %}
          {% if task.disk_io > 0 %}
          <dt>

- <i class="icon-question-sign get-help" title="Number of
miliseconds the task spent doing disk input and output"></i>

bitbake/lib/toaster/toastergui/templates/task.html +1 -1

-            Disk I/O (ms)
+            <i class="icon-question-sign get-help" title="Number of

bytes written to and read from the disk during the task"></i>

bitbake/lib/toaster/toastergui/templates/task.html +2 -1

+            Disk I/O (bytes)
          </dt>
-        <dd>{{task.disk_io|format_none_and_zero}}</dd>
+        <dd>{{task.disk_io|format_none_and_zero|intcomma}}</dd>
          {% endif %}
      </dl>
  diff --git a/bitbake/lib/toaster/toastergui/templates/tasks.html

b/bitbake/lib/toaster/toastergui/templates/tasks.html index 353410f..23eb957 100644

bitbake/lib/toaster/toastergui/templates/tasks.html +1 -0

@@ -1,4 +1,5 @@
  {% extends "basebuildpage.html" %}
+{% load humanize %}
  {% load projecttags %}
    {% block title %} {{mainheading}} -
 {{build.target_set.all|dictsort:"target"|join:", "}} {{build.machine}} - {{

build.project.name}} - Toaster{% endblock %}

bitbake/lib/toaster/toastergui/templates/tasks.html +0 -3

@@ -20,13 +21,15 @@
      <li><a href="{% url 'buildtime' build.pk %}">Time</a></li>
    {% endif %}
  {% endblock %}
-{% block nav-cpuusage %}
-  {% if 'CPU usage' == mainheading %}
-    <li class="active"><a href="{% url 'cpuusage' build.pk %}">CPU

usage</a></li>

bitbake/lib/toaster/toastergui/templates/tasks.html +4 -0

+
+{% block nav-cputime %}
+  {% if 'CPU time' == mainheading %}
+    <li class="active"><a href="{% url 'cputime' build.pk %}">CPU

time</a></li>

    {% else %}

bitbake/lib/toaster/toastergui/templates/tasks.html +2 -1

-    <li><a href="{% url 'cpuusage' build.pk %}">CPU usage</a></li>
+    <li><a href="{% url 'cputime' build.pk %}">CPU time</a></li>
    {% endif %}
  {% endblock %}
+
  {% block nav-diskio %}
    {% if 'Disk I/O' == mainheading %}
      <li class="active"><a href="{% url 'diskio' build.pk %}">Disk

I/O</a></li>

bitbake/lib/toaster/toastergui/templates/tasks.html +6 -3

@@ -107,11 +110,14 @@
              <td class="time_taken">
 
 {{task.elapsed_time|format_none_and_zero|floatformat:2}}
              </td>
-            <td class="cpu_used">
-                {{task.cpu_usage|format_none_and_zero|floatformat:2}}{%
 if task.cpu_usage %}%{% endif %}
+            <td class="cpu_time_system">
+
 {{task.cpu_time_system|format_none_and_zero|floatformat:2}}
+            </td>
+            <td class="cpu_time_user">
+
 {{task.cpu_time_user|format_none_and_zero|floatformat:2}}
              </td>
              <td class="disk_io">
-                {{task.disk_io|format_none_and_zero}}
+                {{task.disk_io|format_none_and_zero|intcomma}}
              </td>
            </tr>

bitbake/lib/toaster/toastergui/urls.py +0 -1

index 4aa6488..400580a 100644
@@ -64,7 +64,7 @@ urlpatterns = patterns('toastergui.views',
          url(r'^build/(?P<build_id>\d+)/configuration$',
 'configuration', name='configuration'),
          url(r'^build/(?P<build_id>\d+)/configvars$', 'configvars',
 name='configvars'),
          url(r'^build/(?P<build_id>\d+)/buildtime$', 'buildtime',
 name='buildtime'),
-        url(r'^build/(?P<build_id>\d+)/cpuusage$', 'cpuusage',

name='cpuusage'),
+ url(r'^build/(?P<build_id>\d+)/cputime$', 'cputime',
name='cputime'),

          url(r'^build/(?P<build_id>\d+)/diskio$', 'diskio',
name='diskio'),
            # image information dir

bitbake/lib/toaster/toastergui/views.py +0 -0

index bd11892..85ca9be 100755
@@ -1005,11 +1005,11 @@ def tasks_common(request, build_id, variant,

task_anchor):
object_search_display="disk I/O data" filter_search_display="tasks"
(pagesize, orderby) = _get_parameters_values(request, 25,
'disk_io:-')

bitbake/lib/toaster/toastergui/views.py +3 -4

-    elif 'cpuusage'  == variant:
-        title_variant='CPU usage'
-        object_search_display="CPU usage data"
+    elif 'cputime'  == variant:
+        title_variant='CPU time'
+        object_search_display="CPU time data"
          filter_search_display="tasks"
-        (pagesize, orderby) = _get_parameters_values(request, 25,

'cpu_usage:-')
+ (pagesize, orderby) = _get_parameters_values(request, 25,
'cpu_time_system:-')
else :
title_variant='Tasks'
object_search_display="tasks"

bitbake/lib/toaster/toastergui/views.py +0 -0

@@ -1161,23 +1161,38 @@ def tasks_common(request, build_id, variant,

task_anchor):

          del tc_time['clclass']
          tc_cache['hidden']='1'

- tc_cpu={

bitbake/lib/toaster/toastergui/views.py +3 -6

-        'name':'CPU usage',
-        'qhelp':'The percentage of task CPU utilization',
-        'orderfield': _get_toggle_order(request, "cpu_usage", True),
-        'ordericon':_get_toggle_order_icon(request, "cpu_usage"),
-        'orderkey' : 'cpu_usage',
-        'clclass': 'cpu_used', 'hidden' : 1,
+    tc_cpu_time_system={
+        'name':'System CPU time (secs)',
+        'qhelp':'Total amount of time spent executing in kernel mode,

in ' +
+ 'seconds. Note that this time can be greater than the
task ' +

bitbake/lib/toaster/toastergui/views.py +2 -0

+                'time due to parallel execution.',
+        'orderfield': _get_toggle_order(request, "cpu_time_system",

True),

bitbake/lib/toaster/toastergui/views.py +6 -2

+        'ordericon':_get_toggle_order_icon(request, "cpu_time_system"),
+        'orderkey' : 'cpu_time_system',
+        'clclass': 'cpu_time_system', 'hidden' : 1,
      }
  -    if  'cpuusage' == variant:
-        tc_cpu['hidden']='0'
-        del tc_cpu['clclass']
+    tc_cpu_time_user={
+        'name':'User CPU time (secs)',
+        'qhelp':'Total amount of time spent executing in user mode, in

seconds. ' +
+ 'Note that this time can be greater than the task time
due to ' +

bitbake/lib/toaster/toastergui/views.py +12 -2

+                'parallel execution.',
+        'orderfield': _get_toggle_order(request, "cpu_time_user", True),
+        'ordericon':_get_toggle_order_icon(request, "cpu_time_user"),
+        'orderkey' : 'cpu_time_user',
+        'clclass': 'cpu_time_user', 'hidden' : 1,
+    }
+
+    if 'cputime' == variant:
+        tc_cpu_time_system['hidden']='0'
+        tc_cpu_time_user['hidden']='0'
+        del tc_cpu_time_system['clclass']
+        del tc_cpu_time_user['clclass']
          tc_cache['hidden']='1'
        tc_diskio={
-        'name':'Disk I/O (ms)',
-        'qhelp':'Number of miliseconds the task spent doing disk input

and output',

bitbake/lib/toaster/toastergui/views.py +2 -0

+        'name':'Disk I/O (bytes)',
+        'qhelp':'Number of bytes written to and read from the disk

during the task',

          'orderfield': _get_toggle_order(request, "disk_io", True),
          'ordericon':_get_toggle_order_icon(request, "disk_io"),
          'orderkey' : 'disk_io',

bitbake/lib/toaster/toastergui/views.py +0 -0

@@ -1208,7 +1223,8 @@ def tasks_common(request, build_id, variant,

task_anchor):
tc_outcome,
tc_cache,
tc_time,

bitbake/lib/toaster/toastergui/views.py +2 -1

-                    tc_cpu,
+                    tc_cpu_time_system,
+                    tc_cpu_time_user,
                      tc_diskio,
                  ]}

bitbake/lib/toaster/toastergui/views.py

Quoted 10 lines

bitbake/lib/toaster/toastergui/views.py +2 -2

  @@ -1229,9 +1245,8 @@ def buildtime(request, build_id):
  def diskio(request, build_id):
      return tasks_common(request, build_id, 'diskio', '')
  -def cpuusage(request, build_id):
-    return tasks_common(request, build_id, 'cpuusage', '')
-
+def cputime(request, build_id):
+    return tasks_common(request, build_id, 'cputime', '')
    def recipes(request, build_id):
      template = 'recipes.html'

---------------------------------------------------------------------
Intel Corporation (UK) Limited
Registered No. 1134945 (England)
Registered Office: Pipers Way, Swindon SN3 1RJ
VAT No: 860 2173 47

This e-mail and any attachments may contain confidential material for the sole use of the intended recipient(s). Any review or distribution by others is strictly prohibited. If you are not the intended recipient, please contact the sender and delete all copies.

Quoted 6 lines

--
_______________________________________________
toaster mailing list
[email protected]
https://lists.yoctoproject.org/listinfo/toaster

--
Elliot Smith
Software Engineer
Intel Open Source Technology Centre

Add a basic test which checks that the CPU time subpage for the build dashboard shows both the User and System CPU time columns by default.

[YOCTO #8842]

Signed-off-by Elliot Smith <[email protected]>

---
  bitbake/lib/toaster/toastergui/tests.py | 45 ++++++++++++++++++++++++++++++++-
  1 file changed, 44 insertions(+), 1 deletion(-)

bitbake/lib/toaster/toastergui/tests.py +34 -1

index 6b05916..b5862b7 100644
@@ -29,7 +29,7 @@ from django.utils import timezone
  from orm.models import Project, Release, BitbakeVersion, Package, LogMessage
  from orm.models import ReleaseLayerSourcePriority, LayerSource, Layer, Build
  from orm.models import Layer_Version, Recipe, Machine, ProjectLayer, Target
-from orm.models import CustomImageRecipe, ProjectVariable
+from orm.models import CustomImageRecipe, ProjectVariable, Task
  from orm.models import Branch, CustomImagePackage
 
  import toastermain

@@ -1054,6 +1054,16 @@ class BuildDashboardTests(TestCase):
                                             started_on=now,
                                             completed_on=now)
 
+        # target for build, so breadcrumb can display
+        Target.objects.create(build=self.build1, target='bash', task='build')
+
+        # layer_version, recipe and task for build, so CPU time data can display
+        layer = Layer.objects.create()
+        layer_version = Layer_Version.objects.create(layer=layer)
+        recipe = Recipe.objects.create(name='zlib', layer_version=layer_version)
+        Task.objects.create(build=self.build1, recipe=recipe, order=1,
+                            outcome=Task.OUTCOME_SUCCESS)
+
          # exception
          msg1 = 'an exception was thrown'
          self.exception_message = LogMessage.objects.create(

@@ -1123,3 +1133,36 @@ class BuildDashboardTests(TestCase):
          section of the page
          """
          self._check_for_log_message(self.critical_message)
+
+    def test_cputime(self):
+        """
+        Check that the system and user CPU time columns are displayed
+        when the cputime subpage is shown
+        """
+        url = reverse('cputime', args=(self.build1.id,))
+        response = self.client.get(url, follow=True)
+        soup = BeautifulSoup(response.content)
+
+        # check nav item is highlighted
+        elements = soup.select('#nav li.active')
+        self.assertEquals(len(elements), 1,
+                          'should be one active nav element, ' +
+                          'but found %s' % len(elements))
+        link_text = elements[0].find('a').text.strip()
+        self.assertEquals(link_text, 'CPU time',
+                          'active nav element should have text "CPU time", ' +
+                          'but text was %s' % link_text)
+
+        # check page heading
+        heading = soup.select('.page-header h1')[0]
+        self.assertEquals(heading.text.strip(), 'CPU time')

Not really that keen on mixing HTML/content tests with the django functional unit tests. To me this would be better tested by the selenium/front end tests.

Quoted 85 lines · Elliot Smith

Add a basic test which checks that the CPU time subpage for the build dashboard shows both the User and System CPU time columns by default.

[YOCTO #8842]

Signed-off-by Elliot Smith <[email protected]>

---
  bitbake/lib/toaster/toastergui/tests.py | 45
++++++++++++++++++++++++++++++++-
  1 file changed, 44 insertions(+), 1 deletion(-)

bitbake/lib/toaster/toastergui/tests.py +0 -0

index 6b05916..b5862b7 100644
@@ -29,7 +29,7 @@ from django.utils import timezone
  from orm.models import Project, Release, BitbakeVersion, Package,

LogMessage
from orm.models import ReleaseLayerSourcePriority, LayerSource, Layer,
Build
from orm.models import Layer_Version, Recipe, Machine, ProjectLayer,
Target

bitbake/lib/toaster/toastergui/tests.py +4 -1

-from orm.models import CustomImageRecipe, ProjectVariable
+from orm.models import CustomImageRecipe, ProjectVariable, Task
  from orm.models import Branch, CustomImagePackage
    import toastermain

@@ -1054,6 +1054,16 @@ class BuildDashboardTests(TestCase):
                                             started_on=now,
                                             completed_on=now)
  +        # target for build, so breadcrumb can display
+        Target.objects.create(build=self.build1, target='bash',
 task='build')
+
+        # layer_version, recipe and task for build, so CPU time data can

display

bitbake/lib/toaster/toastergui/tests.py +3 -0

+        layer = Layer.objects.create()
+        layer_version = Layer_Version.objects.create(layer=layer)
+        recipe = Recipe.objects.create(name='zlib',

layer_version=layer_version)

bitbake/lib/toaster/toastergui/tests.py +21 -0

+        Task.objects.create(build=self.build1, recipe=recipe, order=1,
+                            outcome=Task.OUTCOME_SUCCESS)
+
          # exception
          msg1 = 'an exception was thrown'
          self.exception_message = LogMessage.objects.create(

@@ -1123,3 +1133,36 @@ class BuildDashboardTests(TestCase):
          section of the page
          """
          self._check_for_log_message(self.critical_message)
+
+    def test_cputime(self):
+        """
+        Check that the system and user CPU time columns are displayed
+        when the cputime subpage is shown
+        """
+        url = reverse('cputime', args=(self.build1.id,))
+        response = self.client.get(url, follow=True)
+        soup = BeautifulSoup(response.content)
+
+        # check nav item is highlighted
+        elements = soup.select('#nav li.active')
+        self.assertEquals(len(elements), 1,
+                          'should be one active nav element, ' +
+                          'but found %s' % len(elements))
+        link_text = elements[0].find('a').text.strip()
+        self.assertEquals(link_text, 'CPU time',
+                          'active nav element should have text "CPU

time", ' +

bitbake/lib/toaster/toastergui/tests.py +15 -0

+                          'but text was %s' % link_text)
+
+        # check page heading
+        heading = soup.select('.page-header h1')[0]
+        self.assertEquals(heading.text.strip(), 'CPU time')
+
+        # check CPU column headings are both present
+        cpu_time_headings = ['System CPU time', 'User CPU time']
+        table_heading_links = soup.select('#otable th a')
+        for link in table_heading_links:
+            if link.text.strip() in cpu_time_headings:
+                cpu_time_headings.remove(link.text)
+        self.assertEquals(len(cpu_time_headings), 0,
+                          'Both CPU time headings (user, system) ' +
+                          'must be present; missing %s' %

Quoted 2 lines

cpu_time_headings)

---------------------------------------------------------------------
Intel Corporation (UK) Limited
Registered No. 1134945 (England)
Registered Office: Pipers Way, Swindon SN3 1RJ
VAT No: 860 2173 47

This e-mail and any attachments may contain confidential material for the sole use of the intended recipient(s). Any review or distribution by others is strictly prohibited. If you are not the intended recipient, please contact the sender and delete all copies.

--
_______________________________________________
toaster mailing list
[email protected]
https://lists.yoctoproject.org/listinfo/toaster

So would you suggest removing these for now and adding a separate issue to create equivalent Selenium tests, once the UI test framework is cleaned up?

Elliot

Quoted 107 lines · Michael Wood

Not really that keen on mixing HTML/content tests with the django functional unit tests. To me this would be better tested by the selenium/front end tests.

Quoted 85 lines · Elliot Smith

Add a basic test which checks that the CPU time subpage for the build dashboard shows both the User and System CPU time columns by default.

[YOCTO #8842]

Signed-off-by Elliot Smith <[email protected]>

---
  bitbake/lib/toaster/toastergui/tests.py | 45
++++++++++++++++++++++++++++++++-
  1 file changed, 44 insertions(+), 1 deletion(-)

bitbake/lib/toaster/toastergui/tests.py +0 -0

index 6b05916..b5862b7 100644
@@ -29,7 +29,7 @@ from django.utils import timezone
  from orm.models import Project, Release, BitbakeVersion, Package,

LogMessage
from orm.models import ReleaseLayerSourcePriority, LayerSource, Layer,
Build
from orm.models import Layer_Version, Recipe, Machine, ProjectLayer,
Target

bitbake/lib/toaster/toastergui/tests.py +4 -1

-from orm.models import CustomImageRecipe, ProjectVariable
+from orm.models import CustomImageRecipe, ProjectVariable, Task
  from orm.models import Branch, CustomImagePackage
    import toastermain

@@ -1054,6 +1054,16 @@ class BuildDashboardTests(TestCase):
                                             started_on=now,
                                             completed_on=now)
  +        # target for build, so breadcrumb can display
+        Target.objects.create(build=self.build1, target='bash',
 task='build')
+
+        # layer_version, recipe and task for build, so CPU time data

can display

bitbake/lib/toaster/toastergui/tests.py +3 -0

+        layer = Layer.objects.create()
+        layer_version = Layer_Version.objects.create(layer=layer)
+        recipe = Recipe.objects.create(name='zlib',

layer_version=layer_version)

bitbake/lib/toaster/toastergui/tests.py +21 -0

+        Task.objects.create(build=self.build1, recipe=recipe, order=1,
+                            outcome=Task.OUTCOME_SUCCESS)
+
          # exception
          msg1 = 'an exception was thrown'
          self.exception_message = LogMessage.objects.create(

@@ -1123,3 +1133,36 @@ class BuildDashboardTests(TestCase):
          section of the page
          """
          self._check_for_log_message(self.critical_message)
+
+    def test_cputime(self):
+        """
+        Check that the system and user CPU time columns are displayed
+        when the cputime subpage is shown
+        """
+        url = reverse('cputime', args=(self.build1.id,))
+        response = self.client.get(url, follow=True)
+        soup = BeautifulSoup(response.content)
+
+        # check nav item is highlighted
+        elements = soup.select('#nav li.active')
+        self.assertEquals(len(elements), 1,
+                          'should be one active nav element, ' +
+                          'but found %s' % len(elements))
+        link_text = elements[0].find('a').text.strip()
+        self.assertEquals(link_text, 'CPU time',
+                          'active nav element should have text "CPU

time", ' +

bitbake/lib/toaster/toastergui/tests.py +15 -0

+                          'but text was %s' % link_text)
+
+        # check page heading
+        heading = soup.select('.page-header h1')[0]
+        self.assertEquals(heading.text.strip(), 'CPU time')
+
+        # check CPU column headings are both present
+        cpu_time_headings = ['System CPU time', 'User CPU time']
+        table_heading_links = soup.select('#otable th a')
+        for link in table_heading_links:
+            if link.text.strip() in cpu_time_headings:
+                cpu_time_headings.remove(link.text)
+        self.assertEquals(len(cpu_time_headings), 0,
+                          'Both CPU time headings (user, system) ' +
+                          'must be present; missing %s' %

Quoted 2 lines

cpu_time_headings)

---------------------------------------------------------------------
Intel Corporation (UK) Limited
Registered No. 1134945 (England)
Registered Office: Pipers Way, Swindon SN3 1RJ
VAT No: 860 2173 47

This e-mail and any attachments may contain confidential material for the sole use of the intended recipient(s). Any review or distribution by others is strictly prohibited. If you are not the intended recipient, please contact the sender and delete all copies.

Quoted 6 lines

--
_______________________________________________
toaster mailing list
[email protected]
https://lists.yoctoproject.org/listinfo/toaster

--
Elliot Smith
Software Engineer
Intel Open Source Technology Centre