toaster: Better SDK artifacts and fstypes V2

index 57be15a..ad14997 100644
@@ -1219,8 +1219,8 @@ class RunQueue:
                 pass
             self.state = runQueueComplete
             raise
-        except:
-            logger.error("An uncaught exception occured in runqueue, please see the failure below:")
+        except Exception as err:
+            logger.exception("An uncaught exception occured in runqueue")
             try:
                 self.teardown_workers()
             except:

index 04a57ef..dcda2a8 100644
@@ -1,8 +1,9 @@
 {% extends "basebuildpage.html" %}
 {% load humanize %}
 {% load projecttags %}
+{% load field_values_filter %}
 
-{% block title %} {{build.target_set.all|dictsort:"target"|join:", "}} {{build.machine}} - {{build.project.name}} - Toaster {% endblock %}
+{% block title %} {{build.get_sorted_target_list|field_values:"target"|join:", "}} {{build.machine}} - {{build.project.name}} - Toaster {% endblock %}
 {% block parentbreadcrumb %}
 {% if build.get_sorted_target_list.count > 0 %}
   {{build.get_sorted_target_list.0.target}}

@@ -15,7 +16,7 @@
 <!-- page title -->
 <div class="col-md-10">
  <div class="page-header build-data">
-     <h1>{{build.target_set.all|dictsort:"target"|join:", "}} {{build.machine}}</h1>
+     <h1>{{build.get_sorted_target_list|field_values:"target"|join:", "}} {{build.machine}}</h1>
  </div>
 
 <!-- build result bar -->

@@ -113,7 +114,7 @@
             </dt>
             <dd>
                 <ul class="list-unstyled">
-                    {% for i in target.imageFiles %}
+                    {% for i in target.imageFiles|dictsort:"suffix" %}
                         <li>
                             <a href="{% url 'build_artifact' build.pk 'imagefile' i.id %}">
                                 {{i.suffix}}

@@ -261,7 +262,7 @@
         if (location.href.search('#warnings') > -1) {
             $('#warning-info').addClass('in');
         }
-	
+
 	//show warnings section when requested from the build outcome
 	$(".show-warnings").click(function() {
 	    $('#warning-info').addClass('in');

new file mode 100644
index 0000000..5a73af7
@@ -0,0 +1,18 @@
+from django import template
+
+register = template.Library()
+
+def field_values(iterable, field):
+    """
+    Convert an iterable of models into a list of strings, one for each model,
+    where the string for each model is the value of the field "field".
+    """
+    objects = []
+
+    if field:
+        for item in iterable:
+            objects.append(getattr(item, field))
+
+    return objects
+
+register.filter('field_values', field_values)
\ No newline at end of file

index 447670c..8bdc9cc 100644
@@ -678,6 +678,16 @@ class ORMWrapper(object):
                             file_name = file_name,
                             file_size = file_size)
 
+    def save_artifact_information_no_dedupe(self, build_obj, file_name, file_size):
+        """
+        Save artifact information without checking for duplicate paths;
+        this is used when we are saving data about an artifact which was
+        generated by a previous build but which is also relevant to this build,
+        e.g. a bzImage file.
+        """
+        BuildArtifact.objects.create(build=build_obj, file_name=file_name,
+            file_size=file_size)
+
     def save_artifact_information(self, build_obj, file_name, file_size):
         # we skip the image files from other builds
         if Target_Image_File.objects.filter(file_name = file_name).count() > 0:

@@ -687,7 +697,8 @@ class ORMWrapper(object):
         if BuildArtifact.objects.filter(file_name = file_name).count() > 0:
             return
 
-        BuildArtifact.objects.create(build = build_obj, file_name = file_name, file_size = file_size)
+        self.save_artifact_information_no_dedupe(self, build_obj, file_name,
+            file_size)
 
     def create_logmessage(self, log_information):
         assert 'build' in log_information

@@ -1061,17 +1072,6 @@ class BuildInfoHelper(object):
 
         return self.brbe
 
-
-    def update_target_image_file(self, event):
-        evdata = BuildInfoHelper._get_data_from_event(event)
-
-        for t in self.internal_state['targets']:
-            if t.is_image == True:
-                output_files = list(evdata.keys())
-                for output in output_files:
-                    if t.target in output and 'rootfs' in output and not output.endswith(".manifest"):
-                        self.orm_wrapper.save_target_image_file_information(t, output, evdata[output])
-
     def update_artifact_image_file(self, event):
         evdata = BuildInfoHelper._get_data_from_event(event)
         for artifact_path in evdata.keys():

@@ -1081,16 +1081,6 @@ class BuildInfoHelper(object):
         if 'build' in self.internal_state:
             self.orm_wrapper.update_build_object(self.internal_state['build'], errors, warnings, taskfailures)
 
-
-    def store_license_manifest_path(self, event):
-        deploy_dir = BuildInfoHelper._get_data_from_event(event)['deploy_dir']
-        image_name = BuildInfoHelper._get_data_from_event(event)['image_name']
-        path = deploy_dir + "/licenses/" + image_name + "/license.manifest"
-        for target in self.internal_state['targets']:
-            if target.target in image_name:
-                self.orm_wrapper.update_target_set_license_manifest(target, path)
-
-
     def store_started_task(self, event):
         assert isinstance(event, (bb.runqueue.sceneQueueTaskStarted, bb.runqueue.runQueueTaskStarted, bb.runqueue.runQueueTaskSkipped))
         assert 'taskfile' in vars(event)

@@ -1506,6 +1496,173 @@ class BuildInfoHelper(object):
 
         self.orm_wrapper.create_logmessage(log_information)
 
+    def _get_files_from_image_license(self, image_license_manifest_path):
+        """
+        Find the FILES line in the image_license.manifest file,
+        which has the basenames of the bzImage and modules files
+        in this format:
+        FILES: bzImage--4.4.11+git0+3a5f494784_53e84104c5-r0-qemux86-20160603165040.bin modules--4.4.11+git0+3a5f494784_53e84104c5-r0-qemux86-20160603165040.tgz
+        """
+        files = []
+        with open(image_license_manifest_path) as image_license:
+            for line in image_license:
+                if line.startswith('FILES'):
+                    files_str = line.split(':')[1].strip()
+                    files_str = re.sub(r' {2,}', ' ', files_str)
+                    files = files_str.split(' ')
+        return files
+
+    def _endswith(self, str_to_test, endings):
+        """
+        Returns True if str ends with one of the strings in the list
+        endings, False otherwise
+        """
+        endswith = False
+        for ending in endings:
+            if str_to_test.endswith(ending):
+                endswith = True
+                break
+        return endswith
+
+    def _get_image_files(self, deploy_dir_image, image_name, image_file_extensions):
+        """
+        Find files in deploy_dir_image whose basename starts with the
+        string image_name and ends with one of the strings in
+        image_file_extensions.
+
+        Returns a list of file dictionaries like
+
+        [
+            {
+                'path': '/path/to/image/file',
+                'size': <file size in bytes>
+            }
+        ]
+        """
+        image_files = []
+
+        for dirpath, _, filenames in os.walk(deploy_dir_image):
+            for filename in filenames:
+                if filename.startswith(image_name) and \
+                self._endswith(filename, image_file_extensions):
+                    image_file_path = os.path.join(dirpath, filename)
+                    image_file_size = os.stat(image_file_path).st_size
+
+                    image_files.append({
+                        'path': image_file_path,
+                        'size': image_file_size
+                    })
+
+        return image_files
+
+    def scan_build_artifacts(self):
+        """
+        Scan for build artifacts in DEPLOY_DIR_IMAGE and associate them
+        with a Target object in self.internal_state['targets'].
+
+        We have two situations to handle:
+
+        1. This is the first time a target + machine has been built, so
+        add files from the DEPLOY_DIR_IMAGE to the target.
+
+        OR
+
+        2. There are no files for the target, so copy them from a
+        previous build with the same target + machine.
+        """
+        deploy_dir_image = \
+            self.server.runCommand(['getVariable', 'DEPLOY_DIR_IMAGE'])[0]
+
+        # if there's no DEPLOY_DIR_IMAGE, there aren't going to be
+        # any build artifacts, so we can return immediately
+        if not deploy_dir_image:
+            return
+
+        buildname = self.server.runCommand(['getVariable', 'BUILDNAME'])[0]
+        machine =  self.server.runCommand(['getVariable', 'MACHINE'])[0]
+        image_name = self.server.runCommand(['getVariable', 'IMAGE_NAME'])[0]
+
+        # location of the image_license.manifest files for this build;
+        # note that this file is only produced if an image is produced
+        license_directory = \
+            self.server.runCommand(['getVariable', 'LICENSE_DIRECTORY'])[0]
+
+        # file name extensions for image files
+        image_file_extensions_unique = {}
+        image_fstypes = self.server.runCommand(
+            ['getVariable', 'IMAGE_FSTYPES'])[0]
+        if image_fstypes != None:
+            image_types_str = image_fstypes.strip()
+            image_file_extensions = re.sub(r' {2,}', ' ', image_types_str)
+            image_file_extensions_unique = set(image_file_extensions.split(' '))
+
+        targets = self.internal_state['targets']
+        image_targets = [target for target in targets if target.is_image]
+        for target in image_targets:
+            # this is set to True if we find at least one file relating to
+            # this target; if this remains False after the scan, we copy the
+            # files from the most-recent Target with the same target + machine
+            # onto this Target instead
+            has_files = False
+
+            # we construct this because by the time we reach
+            # BuildCompleted, this has reset to
+            # 'defaultpkgname-<MACHINE>-<BUILDNAME>';
+            # we need to change it to
+            # <TARGET>-<MACHINE>-<BUILDNAME>
+            real_image_name = re.sub(r'^defaultpkgname', target.target,
+                image_name)
+
+            image_license_manifest_path = os.path.join(
+                license_directory,
+                real_image_name,
+                'image_license.manifest')
+
+            # if image_license.manifest exists, we can read the names of bzImage
+            # and modules files for this build from it, then look for them
+            # in the DEPLOY_DIR_IMAGE; note that this file is only produced
+            # if an image file was produced
+            if os.path.isfile(image_license_manifest_path):
+                has_files = True
+
+                basenames = self._get_files_from_image_license(
+                    image_license_manifest_path)
+
+                for basename in basenames:
+                    artifact_path = os.path.join(deploy_dir_image, basename)
+                    artifact_size = os.stat(artifact_path).st_size
+
+                    self.orm_wrapper.save_artifact_information_no_dedupe(
+                        self.internal_state['build'], artifact_path,
+                        artifact_size)
+
+                # store the license manifest path on the target
+                # (this file is also created any time an image file is created)
+                license_path = os.path.join(license_directory,
+                    real_image_name, 'license.manifest')
+
+                self.orm_wrapper.update_target_set_license_manifest(target,
+                    license_path)
+
+            # scan the directory for image files relating to this build
+            # (via real_image_name); note that we don't have to set
+            # has_files = True, as searching for the license manifest file
+            # will already have set it to true if at least one image file was
+            # produced
+            image_files = self._get_image_files(deploy_dir_image,
+                real_image_name, image_file_extensions_unique)
+
+            for image_file in image_files:
+                self.orm_wrapper.save_target_image_file_information(
+                    target, image_file['path'], image_file['size'])
+
+            if not has_files:
+                # TODO copy artifact and image files from the
+                # most-recently-built Target with the same target + machine
+                # as this Target; also copy the license manifest path,
+                # as that is treated differently
+                pass
+
     def close(self, errorcode):
         if self.brbe is not None:
             self._store_build_done(errorcode)

index 5382935..d8bccdb 100644
@@ -363,6 +363,8 @@ def main(server, eventHandler, params):
                     errors += 1
                     errorcode = 1
                     logger.error("Command execution failed: %s", event.error)
+                elif isinstance(event, bb.event.BuildCompleted):
+                    buildinfohelper.scan_build_artifacts()
 
                 # turn off logging to the current build log
                 _close_build_log(build_log)

@@ -410,12 +412,8 @@ def main(server, eventHandler, params):
                     buildinfohelper.store_target_package_data(event)
                 elif event.type == "MissedSstate":
                     buildinfohelper.store_missed_state_tasks(event)
-                elif event.type == "ImageFileSize":
-                    buildinfohelper.update_target_image_file(event)
                 elif event.type == "ArtifactFileSize":
                     buildinfohelper.update_artifact_image_file(event)
-                elif event.type == "LicenseManifestPath":
-                    buildinfohelper.store_license_manifest_path(event)
                 elif event.type == "SetBRBE":
                     buildinfohelper.brbe = buildinfohelper._get_data_from_event(event)
                 elif event.type == "OSErrorException":

index 61737c7..40cdb9e 100644
@@ -440,51 +440,21 @@ class Build(models.Model):
         """
         Get list of file name extensions for images produced by this build
         """
-        targets = Target.objects.filter(build_id = self.id)
         extensions = []
 
-        # pattern to match against file path for building extension string
-        pattern = re.compile('\.([^\.]+?)$')
-
+        targets = Target.objects.filter(build_id = self.id)
         for target in targets:
             if (not target.is_image):
                 continue
 
-            target_image_files = Target_Image_File.objects.filter(target_id = target.id)
+            target_image_files = Target_Image_File.objects.filter(
+                target_id=target.id)
 
             for target_image_file in target_image_files:
-                file_name = os.path.basename(target_image_file.file_name)
-                suffix = ''
-
-                continue_matching = True
-
-                # incrementally extract the suffix from the file path,
-                # checking it against the list of valid suffixes at each
-                # step; if the path is stripped of all potential suffix
-                # parts without matching a valid suffix, this returns all
-                # characters after the first '.' in the file name
-                while continue_matching:
-                    matches = pattern.search(file_name)
-
-                    if None == matches:
-                        continue_matching = False
-                        suffix = re.sub('^\.', '', suffix)
-                        continue
-                    else:
-                        suffix = matches.group(1) + suffix
-
-                    if suffix in Target_Image_File.SUFFIXES:
-                        continue_matching = False
-                        continue
-                    else:
-                        # reduce the file name and try to find the next
-                        # segment from the path which might be part
-                        # of the suffix
-                        file_name = re.sub('.' + matches.group(1), '', file_name)
-                        suffix = '.' + suffix
-
-                if not suffix in extensions:
-                    extensions.append(suffix)
+                extensions.append(target_image_file.suffix)
+
+        extensions = list(set(extensions))
+        extensions.sort()
 
         return ', '.join(extensions)
 

@@ -658,6 +628,10 @@ class Target_Image_File(models.Model):
 
     @property
     def suffix(self):
+        for suffix in Target_Image_File.SUFFIXES:
+            if self.file_name.endswith(suffix):
+                return suffix
+
         filename, suffix = os.path.splitext(self.file_name)
         suffix = suffix.lstrip('.')
         return suffix

index 8bdc9cc..a5b2237 100644
@@ -37,7 +37,7 @@ os.environ["DJANGO_SETTINGS_MODULE"] =\
 django.setup()
 
 from orm.models import Build, Task, Recipe, Layer_Version, Layer, Target, LogMessage, HelpText
-from orm.models import Target_Image_File, BuildArtifact
+from orm.models import Target_Image_File, BuildArtifact, TargetArtifactFile
 from orm.models import Variable, VariableHistory
 from orm.models import Package, Package_File, Target_Installed_Package, Target_File
 from orm.models import Task_Dependency, Package_Dependency

@@ -121,6 +121,13 @@ class ORMWrapper(object):
 
         return vars(self)[dictname][key]
 
+    def get_similar_target_with_image_files(self, target):
+        """
+        Get a Target object "similar" to target; i.e. with the same target
+        name ('core-image-minimal' etc.) and machine.
+        """
+        return target.get_similar_target_with_image_files()
+
     def _timestamp_to_datetime(self, secs):
         """
         Convert timestamp in seconds to Python datetime

@@ -678,27 +685,32 @@ class ORMWrapper(object):
                             file_name = file_name,
                             file_size = file_size)
 
-    def save_artifact_information_no_dedupe(self, build_obj, file_name, file_size):
+    def save_target_artifact_file(self, target_obj, file_name, file_size):
         """
-        Save artifact information without checking for duplicate paths;
-        this is used when we are saving data about an artifact which was
-        generated by a previous build but which is also relevant to this build,
-        e.g. a bzImage file.
+        Save artifact file information for a Target target_obj.
+
+        Note that this doesn't include SDK artifacts, only images and
+        related files (e.g. bzImage).
         """
-        BuildArtifact.objects.create(build=build_obj, file_name=file_name,
-            file_size=file_size)
+        TargetArtifactFile.objects.create(target=target_obj,
+            file_name=file_name, file_size=file_size)
 
     def save_artifact_information(self, build_obj, file_name, file_size):
-        # we skip the image files from other builds
-        if Target_Image_File.objects.filter(file_name = file_name).count() > 0:
-            return
-
+        """
+        TODO this is currently used to save SDK artifacts to the database,
+        but will be replaced in future once SDK artifacts are associated with
+        Target objects (as they eventually should be)
+        """
         # do not update artifacts found in other builds
         if BuildArtifact.objects.filter(file_name = file_name).count() > 0:
             return
 
-        self.save_artifact_information_no_dedupe(self, build_obj, file_name,
-            file_size)
+        # do not update artifact if already a target artifact file for that path
+        if TargetArtifactFile.objects.filter(file_name = file_name).count() > 0:
+            return
+
+        BuildArtifact.objects.create(build=build_obj, file_name=file_name,
+            file_size=file_size)
 
     def create_logmessage(self, log_information):
         assert 'build' in log_information

@@ -1496,7 +1508,7 @@ class BuildInfoHelper(object):
 
         self.orm_wrapper.create_logmessage(log_information)
 
-    def _get_files_from_image_license(self, image_license_manifest_path):
+    def _get_filenames_from_image_license(self, image_license_manifest_path):
         """
         Find the FILES line in the image_license.manifest file,
         which has the basenames of the bzImage and modules files

@@ -1567,19 +1579,20 @@ class BuildInfoHelper(object):
 
         OR
 
-        2. There are no files for the target, so copy them from a
-        previous build with the same target + machine.
+        2. There are no new files for the target (they were already produced by
+        a previous build), so copy them from the most recent previous build with
+        the same target, task and machine.
         """
         deploy_dir_image = \
             self.server.runCommand(['getVariable', 'DEPLOY_DIR_IMAGE'])[0]
 
         # if there's no DEPLOY_DIR_IMAGE, there aren't going to be
-        # any build artifacts, so we can return immediately
+        # any image artifacts, so we can return immediately
         if not deploy_dir_image:
             return
 
         buildname = self.server.runCommand(['getVariable', 'BUILDNAME'])[0]
-        machine =  self.server.runCommand(['getVariable', 'MACHINE'])[0]
+        machine = self.server.runCommand(['getVariable', 'MACHINE'])[0]
         image_name = self.server.runCommand(['getVariable', 'IMAGE_NAME'])[0]
 
         # location of the image_license.manifest files for this build;

@@ -1597,7 +1610,10 @@ class BuildInfoHelper(object):
             image_file_extensions_unique = set(image_file_extensions.split(' '))
 
         targets = self.internal_state['targets']
+
+        # filter out anything which isn't an image target
         image_targets = [target for target in targets if target.is_image]
+
         for target in image_targets:
             # this is set to True if we find at least one file relating to
             # this target; if this remains False after the scan, we copy the

@@ -1625,16 +1641,17 @@ class BuildInfoHelper(object):
             if os.path.isfile(image_license_manifest_path):
                 has_files = True
 
-                basenames = self._get_files_from_image_license(
+                basenames = self._get_filenames_from_image_license(
                     image_license_manifest_path)
 
                 for basename in basenames:
                     artifact_path = os.path.join(deploy_dir_image, basename)
                     artifact_size = os.stat(artifact_path).st_size
 
-                    self.orm_wrapper.save_artifact_information_no_dedupe(
-                        self.internal_state['build'], artifact_path,
-                        artifact_size)
+                    # note that the artifact will only be saved against this
+                    # build if it hasn't been already
+                    self.orm_wrapper.save_target_artifact_file(target,
+                        artifact_path, artifact_size)
 
                 # store the license manifest path on the target
                 # (this file is also created any time an image file is created)

@@ -1648,7 +1665,10 @@ class BuildInfoHelper(object):
             # (via real_image_name); note that we don't have to set
             # has_files = True, as searching for the license manifest file
             # will already have set it to true if at least one image file was
-            # produced
+            # produced; note that the real_image_name includes BUILDNAME, which
+            # in turn includes a timestamp; so if no files were produced for
+            # this timestamp (i.e. the build reused existing image files already
+            # in the directory), no files will be recorded against this target
             image_files = self._get_image_files(deploy_dir_image,
                 real_image_name, image_file_extensions_unique)
 

@@ -1657,11 +1677,18 @@ class BuildInfoHelper(object):
                     target, image_file['path'], image_file['size'])
 
             if not has_files:
-                # TODO copy artifact and image files from the
-                # most-recently-built Target with the same target + machine
-                # as this Target; also copy the license manifest path,
-                # as that is treated differently
-                pass
+                # copy image files and build artifacts from the
+                # most-recently-built Target with the
+                # same target + machine as this Target; also copy the license
+                # manifest path, as that is not treated as an artifact and needs
+                # to be set separately
+                most_recent = \
+                    self.orm_wrapper.get_similar_target_with_image_files(target)
+
+                if most_recent:
+                    logger.info('image artifacts for target %s cloned from ' \
+                        'target %s' % (target.pk, most_recent.pk))
+                    target.clone_artifacts_from(most_recent)
 
     def close(self, errorcode):
         if self.brbe is not None:

new file mode 100644
index 0000000..9f1d9bf
@@ -0,0 +1,23 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('orm', '0007_auto_20160523_1446'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='TargetArtifactFile',
+            fields=[
+                ('id', models.AutoField(primary_key=True, serialize=False, auto_created=True, verbose_name='ID')),
+                ('file_name', models.FilePathField()),
+                ('file_size', models.IntegerField()),
+                ('target', models.ForeignKey(to='orm.Target')),
+            ],
+        ),
+    ]

147 lines not on this page · raw message

index dcda2a8..f6d62b9 100644
@@ -74,7 +74,7 @@
     {% for target in targets %}
         {% if target.target.is_image %}
     <div class="well well-transparent dashboard-section">
-        <h3><a href="{% url 'target' build.pk target.target.pk %}">{{target.target}}</a></h3>
+        <h3><a href="{% url 'target' build.pk target.target.pk %}">{{target.target.target}}</a></h3>
         <dl class="dl-horizontal">
             <dt>Packages included</dt>
             <dd><a href="{% url 'target' build.pk target.target.pk %}">{{target.npkg}}</a></dd>

@@ -124,6 +124,19 @@
                     {% endfor %}
                 </ul>
             </dd>
+            <dt>
+                Kernel artifacts
+            </dt>
+            <dd>
+                <ul class="list-unstyled">
+                    {% for artifact in target.target_artifacts|dictsort:"basename" %}
+                        <li>
+                            <a href="{% url 'build_artifact' build.id 'targetartifactfile' artifact.id %}">{{artifact.basename}}</a>
+                            ({{artifact.file_size|filtered_filesizeformat}})
+                        </li>
+                    {% endfor %}
+                  </ul>
+            </dd>
         </dl>
         {% endif %}
     </div>

index ad85faf..0ec88d9 100755
@@ -31,6 +31,7 @@ from django.shortcuts import render, redirect, get_object_or_404
 from orm.models import Build, Target, Task, Layer, Layer_Version, Recipe, LogMessage, Variable
 from orm.models import Task_Dependency, Recipe_Dependency, Package, Package_File, Package_Dependency
 from orm.models import Target_Installed_Package, Target_File, Target_Image_File, BuildArtifact, CustomImagePackage
+from orm.models import TargetArtifactFile
 from orm.models import BitbakeVersion, CustomImageRecipe
 from bldcontrol import bbcontroller
 from django.views.decorators.cache import cache_control

@@ -509,6 +510,8 @@ def builddashboard( request, build_id ):
             targetHasNoImages = True
         elem[ 'imageFiles' ] = imageFiles
         elem[ 'targetHasNoImages' ] = targetHasNoImages
+        elem['target_artifacts'] = t.targetartifactfile_set.all()
+
         targets.append( elem )
 
     ##

@@ -2294,6 +2297,10 @@ if True:
         elif artifact_type == "buildartifact":
             file_name = BuildArtifact.objects.get(build = build, pk = artifact_id).file_name
 
+        elif artifact_type == "targetartifactfile":
+            target = TargetArtifactFile.objects.get(pk=artifact_id)
+            file_name = target.file_name
+
         elif artifact_type == "licensemanifest":
             file_name = Target.objects.get(build = build, pk = artifact_id).license_manifest_path
 

index a5b2237..e1b59c3 100644
@@ -201,6 +201,10 @@ class ORMWrapper(object):
 
     @staticmethod
     def get_or_create_targets(target_info):
+        """
+        NB get_or_create() is used here because for Toaster-triggered builds,
+        we already created the targets when the build was triggered.
+        """
         result = []
         for target in target_info['targets']:
             task = ''

@@ -210,13 +214,10 @@ class ORMWrapper(object):
                 task = task[3:]
             if task == 'build':
                 task = ''
-            obj, created = Target.objects.get_or_create(build=target_info['build'],
-                                                        target=target)
-            if created:
-                obj.is_image = False
-                if task:
-                    obj.task = task
-                obj.save()
+
+            obj, _ = Target.objects.get_or_create(build=target_info['build'],
+                                                  target=target,
+                                                  task=task)
             result.append(obj)
         return result
 

index e1b59c3..bf032ae 100644
@@ -37,7 +37,7 @@ os.environ["DJANGO_SETTINGS_MODULE"] =\
 django.setup()
 
 from orm.models import Build, Task, Recipe, Layer_Version, Layer, Target, LogMessage, HelpText
-from orm.models import Target_Image_File, BuildArtifact, TargetArtifactFile
+from orm.models import Target_Image_File, TargetKernelFile, TargetSDKFile
 from orm.models import Variable, VariableHistory
 from orm.models import Package, Package_File, Target_Installed_Package, Target_File
 from orm.models import Task_Dependency, Package_Dependency

@@ -128,6 +128,15 @@ class ORMWrapper(object):
         """
         return target.get_similar_target_with_image_files()
 
+    def get_similar_target_with_sdk_files(self, target):
+        return target.get_similar_target_with_sdk_files()
+
+    def clone_image_artifacts(self, target_from, target_to):
+        target_to.clone_image_artifacts_from(target_from)
+
+    def clone_sdk_artifacts(self, target_from, target_to):
+        target_to.clone_sdk_artifacts_from(target_from)
+
     def _timestamp_to_datetime(self, secs):
         """
         Convert timestamp in seconds to Python datetime

@@ -682,35 +691,22 @@ class ORMWrapper(object):
             logger.warning("buildinfohelper: target_package_info could not identify recipes: \n%s", errormsg)
 
     def save_target_image_file_information(self, target_obj, file_name, file_size):
-        Target_Image_File.objects.create( target = target_obj,
-                            file_name = file_name,
-                            file_size = file_size)
+        Target_Image_File.objects.create(target=target_obj,
+            file_name=file_name, file_size=file_size)
 
-    def save_target_artifact_file(self, target_obj, file_name, file_size):
+    def save_target_kernel_file(self, target_obj, file_name, file_size):
         """
-        Save artifact file information for a Target target_obj.
-
-        Note that this doesn't include SDK artifacts, only images and
-        related files (e.g. bzImage).
+        Save kernel file (bzImage, modules*) information for a Target target_obj.
         """
-        TargetArtifactFile.objects.create(target=target_obj,
+        TargetKernelFile.objects.create(target=target_obj,
             file_name=file_name, file_size=file_size)
 
-    def save_artifact_information(self, build_obj, file_name, file_size):
+    def save_target_sdk_file(self, target_obj, file_name, file_size):
         """
-        TODO this is currently used to save SDK artifacts to the database,
-        but will be replaced in future once SDK artifacts are associated with
-        Target objects (as they eventually should be)
+        Save SDK artifacts to the database, associating them with a
+        Target object.
         """
-        # do not update artifacts found in other builds
-        if BuildArtifact.objects.filter(file_name = file_name).count() > 0:
-            return
-
-        # do not update artifact if already a target artifact file for that path
-        if TargetArtifactFile.objects.filter(file_name = file_name).count() > 0:
-            return
-
-        BuildArtifact.objects.create(build=build_obj, file_name=file_name,
+        TargetSDKFile.objects.create(target=target_obj, file_name=file_name,
             file_size=file_size)
 
     def create_logmessage(self, log_information):

@@ -1085,11 +1081,6 @@ class BuildInfoHelper(object):
 
         return self.brbe
 
-    def update_artifact_image_file(self, event):
-        evdata = BuildInfoHelper._get_data_from_event(event)
-        for artifact_path in evdata.keys():
-            self.orm_wrapper.save_artifact_information(self.internal_state['build'], artifact_path, evdata[artifact_path])
-
     def update_build_information(self, event, errors, warnings, taskfailures):
         if 'build' in self.internal_state:
             self.orm_wrapper.update_build_object(self.internal_state['build'], errors, warnings, taskfailures)

@@ -1568,9 +1559,9 @@ class BuildInfoHelper(object):
 
         return image_files
 
-    def scan_build_artifacts(self):
+    def scan_image_artifacts(self):
         """
-        Scan for build artifacts in DEPLOY_DIR_IMAGE and associate them
+        Scan for built image artifacts in DEPLOY_DIR_IMAGE and associate them
         with a Target object in self.internal_state['targets'].
 
         We have two situations to handle:

@@ -1615,7 +1606,7 @@ class BuildInfoHelper(object):
         # filter out anything which isn't an image target
         image_targets = [target for target in targets if target.is_image]
 
-        for target in image_targets:
+        for image_target in image_targets:
             # this is set to True if we find at least one file relating to
             # this target; if this remains False after the scan, we copy the
             # files from the most-recent Target with the same target + machine

@@ -1627,7 +1618,7 @@ class BuildInfoHelper(object):
             # 'defaultpkgname-<MACHINE>-<BUILDNAME>';
             # we need to change it to
             # <TARGET>-<MACHINE>-<BUILDNAME>
-            real_image_name = re.sub(r'^defaultpkgname', target.target,
+            real_image_name = re.sub(r'^defaultpkgname', image_target.target,
                 image_name)
 
             image_license_manifest_path = os.path.join(

@@ -1651,7 +1642,7 @@ class BuildInfoHelper(object):
 
                     # note that the artifact will only be saved against this
                     # build if it hasn't been already
-                    self.orm_wrapper.save_target_artifact_file(target,
+                    self.orm_wrapper.save_target_kernel_file(image_target,
                         artifact_path, artifact_size)
 
                 # store the license manifest path on the target

@@ -1659,8 +1650,8 @@ class BuildInfoHelper(object):
                 license_path = os.path.join(license_directory,
                     real_image_name, 'license.manifest')
 
-                self.orm_wrapper.update_target_set_license_manifest(target,
-                    license_path)
+                self.orm_wrapper.update_target_set_license_manifest(
+                    image_target, license_path)
 
             # scan the directory for image files relating to this build
             # (via real_image_name); note that we don't have to set

@@ -1675,7 +1666,7 @@ class BuildInfoHelper(object):
 
             for image_file in image_files:
                 self.orm_wrapper.save_target_image_file_information(
-                    target, image_file['path'], image_file['size'])
+                    image_target, image_file['path'], image_file['size'])
 
             if not has_files:
                 # copy image files and build artifacts from the

@@ -1683,13 +1674,115 @@ class BuildInfoHelper(object):
                 # same target + machine as this Target; also copy the license
                 # manifest path, as that is not treated as an artifact and needs
                 # to be set separately
-                most_recent = \
-                    self.orm_wrapper.get_similar_target_with_image_files(target)
+                similar_target = \
+                    self.orm_wrapper.get_similar_target_with_image_files(
+                        image_target)
 
-                if most_recent:
+                if similar_target:
                     logger.info('image artifacts for target %s cloned from ' \
-                        'target %s' % (target.pk, most_recent.pk))
-                    target.clone_artifacts_from(most_recent)
+                        'target %s' % (image_target.pk, similar_target.pk))
+                    self.orm_wrapper.clone_image_artifacts(similar_target,
+                        image_target)
+
+    def _get_sdk_targets(self):
+        """
+        Return targets which could generate SDK artifacts, i.e.
+        "do_populate_sdk" and "do_populate_sdk_ext".
+        """
+        return [target for target in self.internal_state['targets'] \
+            if target.task in ['populate_sdk', 'populate_sdk_ext']]
+
+    def scan_sdk_artifacts(self, event):
+        """
+        Note that we have to intercept an SDKArtifactInfo event from
+        toaster.bbclass (via toasterui) to get hold of the SDK variables we
+        need to be able to scan for files accurately: this is because
+        variables like TOOLCHAIN_OUTPUTNAME have reset to None by the time
+        BuildCompleted is fired by bitbake, so we have to get those values
+        while the build is still in progress.
+
+        For populate_sdk_ext, this runs twice, with two different
+        TOOLCHAIN_OUTPUTNAME settings, each of which will capture some of the
+        files in the SDK output directory.
+        """
+        sdk_vars = BuildInfoHelper._get_data_from_event(event)
+        toolchain_outputname = sdk_vars['TOOLCHAIN_OUTPUTNAME']
+
+        # targets which might have created SDK artifacts
+        sdk_targets = self._get_sdk_targets()
+
+        # location of SDK artifacts
+        tmpdir = self.server.runCommand(['getVariable', 'TMPDIR'])[0]
+        sdk_dir = os.path.join(tmpdir, 'deploy', 'sdk')
+
+        # all files in the SDK directory
+        artifacts = []
+        for dir_path, _, filenames in os.walk(sdk_dir):
+            for filename in filenames:
+                full_path = os.path.join(dir_path, filename)
+                if not os.path.islink(full_path):
+                    artifacts.append(full_path)
+
+        for sdk_target in sdk_targets:
+            # find files in the SDK directory which haven't already been
+            # recorded against a Target and whose basename matches
+            # TOOLCHAIN_OUTPUTNAME
+            for artifact_path in artifacts:
+                basename = os.path.basename(artifact_path)
+
+                toolchain_match = basename.startswith(toolchain_outputname)
+
+                # files which match the name of the target which produced them;
+                # for example,
+                # poky-glibc-x86_64-core-image-sato-i586-toolchain-ext-2.1+snapshot.sh
+                target_match = re.search(sdk_target.target, basename)
+
+                # targets which produce "*-nativesdk-*" files
+                is_ext_sdk_target = sdk_target.task in \
+                    ['do_populate_sdk_ext', 'populate_sdk_ext']
+
+                # SDK files which don't match the target name, i.e.
+                # x86_64-nativesdk-libc.*
+                # poky-glibc-x86_64-buildtools-tarball-i586-buildtools-nativesdk-standalone-2.1+snapshot*
+                is_ext_sdk_file = re.search('-nativesdk-', basename)
+
+                file_from_target = (toolchain_match and target_match) or \
+                    (is_ext_sdk_target and is_ext_sdk_file)
+
+                if file_from_target:
+                    # don't record the file if it's already been added to this
+                    # target
+                    matching_files = TargetSDKFile.objects.filter(
+                        target=sdk_target, file_name=artifact_path)
+
+                    if matching_files.count() == 0:
+                        artifact_size = os.stat(artifact_path).st_size
+
+                        self.orm_wrapper.save_target_sdk_file(
+                            sdk_target, artifact_path, artifact_size)
+
+    def clone_required_sdk_artifacts(self):
+        """
+        If an SDK target doesn't have any SDK artifacts, this means that
+        the postfuncs of populate_sdk or populate_sdk_ext didn't fire, which
+        in turn means that the targets of this build didn't generate any new
+        artifacts.
+
+        In this case, clone SDK artifacts for targets in the current build
+        from existing targets for this build.
+        """
+        sdk_targets = self._get_sdk_targets()
+        for sdk_target in sdk_targets:
+            # only clone for SDK targets which have no TargetSDKFiles yet
+            if sdk_target.targetsdkfile_set.all().count() == 0:
+                similar_target = \
+                    self.orm_wrapper.get_similar_target_with_sdk_files(
+                        sdk_target)
+                if similar_target:
+                    logger.info('SDK artifacts for target %s cloned from ' \
+                        'target %s' % (sdk_target.pk, similar_target.pk))
+                    self.orm_wrapper.clone_sdk_artifacts(similar_target,
+                        sdk_target)
 
     def close(self, errorcode):
         if self.brbe is not None:

index d8bccdb..f399a7d 100644
@@ -364,7 +364,8 @@ def main(server, eventHandler, params):
                     errorcode = 1
                     logger.error("Command execution failed: %s", event.error)
                 elif isinstance(event, bb.event.BuildCompleted):
-                    buildinfohelper.scan_build_artifacts()
+                    buildinfohelper.scan_image_artifacts()
+                    buildinfohelper.clone_required_sdk_artifacts()
 
                 # turn off logging to the current build log
                 _close_build_log(build_log)

@@ -412,8 +413,8 @@ def main(server, eventHandler, params):
                     buildinfohelper.store_target_package_data(event)
                 elif event.type == "MissedSstate":
                     buildinfohelper.store_missed_state_tasks(event)
-                elif event.type == "ArtifactFileSize":
-                    buildinfohelper.update_artifact_image_file(event)
+                elif event.type == "SDKArtifactInfo":
+                    buildinfohelper.scan_sdk_artifacts(event)
                 elif event.type == "SetBRBE":
                     buildinfohelper.brbe = buildinfohelper._get_data_from_event(event)
                 elif event.type == "OSErrorException":

similarity index 40%
rename from lib/toaster/orm/migrations/0008_targetartifactfile.py
rename to lib/toaster/orm/migrations/0008_refactor_artifact_models.py
index 9f1d9bf..3367582 100644
@@ -12,12 +12,28 @@ class Migration(migrations.Migration):
 
     operations = [
         migrations.CreateModel(
-            name='TargetArtifactFile',
+            name='TargetKernelFile',
             fields=[
-                ('id', models.AutoField(primary_key=True, serialize=False, auto_created=True, verbose_name='ID')),
+                ('id', models.AutoField(auto_created=True, primary_key=True, verbose_name='ID', serialize=False)),
                 ('file_name', models.FilePathField()),
                 ('file_size', models.IntegerField()),
                 ('target', models.ForeignKey(to='orm.Target')),
             ],
         ),
+        migrations.CreateModel(
+            name='TargetSDKFile',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, verbose_name='ID', serialize=False)),
+                ('file_name', models.FilePathField()),
+                ('file_size', models.IntegerField()),
+                ('target', models.ForeignKey(to='orm.Target')),
+            ],
+        ),
+        migrations.RemoveField(
+            model_name='buildartifact',
+            name='build',
+        ),
+        migrations.DeleteModel(
+            name='BuildArtifact',
+        ),
     ]

169 lines not on this page · raw message

85 lines not on this page · raw message

index 0ec88d9..a82a261 100755
@@ -30,8 +30,8 @@ from django.db import IntegrityError, Error
 from django.shortcuts import render, redirect, get_object_or_404
 from orm.models import Build, Target, Task, Layer, Layer_Version, Recipe, LogMessage, Variable
 from orm.models import Task_Dependency, Recipe_Dependency, Package, Package_File, Package_Dependency
-from orm.models import Target_Installed_Package, Target_File, Target_Image_File, BuildArtifact, CustomImagePackage
-from orm.models import TargetArtifactFile
+from orm.models import Target_Installed_Package, Target_File, Target_Image_File, CustomImagePackage
+from orm.models import TargetKernelFile, TargetSDKFile
 from orm.models import BitbakeVersion, CustomImageRecipe
 from bldcontrol import bbcontroller
 from django.views.decorators.cache import cache_control

@@ -510,7 +510,11 @@ def builddashboard( request, build_id ):
             targetHasNoImages = True
         elem[ 'imageFiles' ] = imageFiles
         elem[ 'targetHasNoImages' ] = targetHasNoImages
-        elem['target_artifacts'] = t.targetartifactfile_set.all()
+        elem['target_kernel_artifacts'] = t.targetkernelfile_set.all()
+
+        target_sdk_files = t.targetsdkfile_set.all()
+        elem['target_sdk_artifacts_count'] = target_sdk_files.count()
+        elem['target_sdk_artifacts'] = target_sdk_files
 
         targets.append( elem )
 

@@ -2294,11 +2298,12 @@ if True:
         elif artifact_type == "imagefile":
             file_name = Target_Image_File.objects.get(target__build = build, pk = artifact_id).file_name
 
-        elif artifact_type == "buildartifact":
-            file_name = BuildArtifact.objects.get(build = build, pk = artifact_id).file_name
+        elif artifact_type == "targetkernelartifact":
+            target = TargetKernelFile.objects.get(pk=artifact_id)
+            file_name = target.file_name
 
-        elif artifact_type == "targetartifactfile":
-            target = TargetArtifactFile.objects.get(pk=artifact_id)
+        elif artifact_type == "targetsdkartifact":
+            target = TargetSDKFile.objects.get(pk=artifact_id)
             file_name = target.file_name
 
         elif artifact_type == "licensemanifest":

index bf032ae..52b5e12 100644
@@ -875,6 +875,11 @@ class BuildInfoHelper(object):
         Keeps in memory all data that needs matching before writing it to the database
     """
 
+    # tasks which produce image files; note we include '', as we set
+    # the task for a target to '' (i.e. 'build') if no target is
+    # explicitly defined
+    IMAGE_GENERATING_TASKS = ['', 'build', 'image', 'populate_sdk_ext']
+
     # pylint: disable=protected-access
     # the code will look into the protected variables of the event; no easy way around this
     # pylint: disable=bad-continuation

@@ -1235,11 +1240,16 @@ class BuildInfoHelper(object):
                     logger.warning("KeyError in save_target_package_information"
                                    "%s ", e)
 
-                try:
-                    self.orm_wrapper.save_target_file_information(self.internal_state['build'], target, filedata)
-                except KeyError as e:
-                    logger.warning("KeyError in save_target_file_information"
-                                   "%s ", e)
+                # only try to find files in the image if the task for this
+                # target is one which produces image files; otherwise, the old
+                # list of files in the files-in-image.txt file will be
+                # appended to the target even if it didn't produce any images
+                if target.task in BuildInfoHelper.IMAGE_GENERATING_TASKS:
+                    try:
+                        self.orm_wrapper.save_target_file_information(self.internal_state['build'], target, filedata)
+                    except KeyError as e:
+                        logger.warning("KeyError in save_target_file_information"
+                                       "%s ", e)
 
 
 

index 61f6a20..0443a45 100644
@@ -436,17 +436,32 @@ class Build(models.Model):
             eta += ((eta - self.started_on)*(100-completeper))/completeper
         return eta
 
+    def has_images(self):
+        """
+        Returns True if at least one of the targets for this build has an
+        image file associated with it, False otherwise
+        """
+        targets = Target.objects.filter(build_id=self.id)
+        has_images = False
+        for target in targets:
+            if target.has_images():
+                has_images = True
+                break
+        return has_images
+
     def get_image_file_extensions(self):
         """
-        Get list of file name extensions for images produced by this build;
+        Get string of file name extensions for images produced by this build;
         note that this is the actual list of extensions stored on Target objects
         for this build, and not the value of IMAGE_FSTYPES.
+
+        Returns comma-separated string, e.g. "vmdk, ext4"
         """
         extensions = []
 
         targets = Target.objects.filter(build_id = self.id)
         for target in targets:
-            if (not target.is_image):
+            if not target.is_image:
                 continue
 
             target_image_files = Target_Image_File.objects.filter(

@@ -739,6 +754,12 @@ class Target(models.Model):
             sdk_file.target = self
             sdk_file.save()
 
+    def has_images(self):
+        """
+        Returns True if this target has one or more image files attached to it.
+        """
+        return self.target_image_file_set.all().count() > 0
+
 # kernel artifacts for a target: bzImage and modules*
 class TargetKernelFile(models.Model):
     target = models.ForeignKey(Target)

@@ -775,6 +796,9 @@ class Target_Image_File(models.Model):
 
     @property
     def suffix(self):
+        """
+        Suffix for image file, minus leading "."
+        """
         for suffix in Target_Image_File.SUFFIXES:
             if self.file_name.endswith(suffix):
                 return suffix

index 856259a..e9927eb 100644
@@ -33,7 +33,7 @@
 
         $("#build-menu li a").each(function(){
           /* Set the page active state in the Build menu */
-          var currentUrl = window.location.href.split('?')[0]; 
+          var currentUrl = window.location.href.split('?')[0];
           if (currentUrl === $(this).prop("href")){
             $(this).parent().addClass("active");
           } else {

@@ -62,11 +62,13 @@
           {% endif %} >
           <a  href="{% url 'builddashboard' build.pk %}">Build summary</a>
         </li>
-        {% if build.target_set.all.0.is_image and build.outcome == 0 %}
+        {% if build.has_images and build.outcome == build.SUCCEEDED %}
           <li class="nav-header">Images</li>
           {% block nav-target %}
             {% for t in build.get_sorted_target_list %}
-            <li id="menu-{{t.target}}"><a href="{% url 'target' build.pk t.pk %}">{{t.target}}</a><li>
+              {% if t.has_images %}
+                <li id="menu-{{t.target}}"><a href="{% url 'target' build.pk t.pk %}">{{t.target}}</a><li>
+              {% endif %}
             {% endfor %}
           {% endblock %}
         {% endif %}

index a82a261..02caf54 100755
@@ -471,19 +471,14 @@ def builddashboard( request, build_id ):
     recipeCount = Recipe.objects.filter( layer_version__id__in = layerVersionId ).count( );
     tgts = Target.objects.filter( build_id = build_id ).order_by( 'target' );
 
-    ##
     # set up custom target list with computed package and image data
-    #
-
     targets = [ ]
     ntargets = 0
-    hasImages = False
+
     targetHasNoImages = False
     for t in tgts:
         elem = { }
         elem[ 'target' ] = t
-        if t.is_image:
-            hasImages = True
         npkg = 0
         pkgsz = 0
         package = None

@@ -533,7 +528,7 @@ def builddashboard( request, build_id ):
     context = {
             'build'           : build,
             'project'         : build.project,
-            'hasImages'       : hasImages,
+            'hasImages'       : build.has_images(),
             'ntargets'        : ntargets,
             'targets'         : targets,
             'recipecount'     : recipeCount,

index 0443a45..a111916 100644
@@ -449,6 +449,19 @@ class Build(models.Model):
                 break
         return has_images
 
+    def has_image_targets(self):
+        """
+        Returns True if a build has any targets which were built from
+        image recipes.
+        """
+        targets = Target.objects.filter(build_id=self.id)
+        has_image_targets = False
+        for target in targets:
+            if target.is_image:
+                has_image_targets = True
+                break
+        return has_image_targets
+
     def get_image_file_extensions(self):
         """
         Get string of file name extensions for images produced by this build;

52 lines not on this page · raw message

index e1bde21..febf8af 100644
@@ -103,12 +103,15 @@
         {% if not target.targetHasNoImages %}
           <dl class="dl-horizontal">
             <dt>
-                <span class="glyphicon glyphicon-question-sign get-help" title="The location in disk of the license manifest, a document listing all packages installed in your image and their licenses"></span>
-                License manifest
+              Manifests
             </dt>
+
             <dd>
-                <a href="{% url 'target' build.pk target.target.pk %}">View in Toaster</a> |
-                <a href="{% url 'build_artifact' build.pk 'licensemanifest' target.target.pk %}">Download</a></dd>
+              <a href="{% url 'build_artifact' build.pk 'licensemanifest' target.target.pk %}">License manifest</a>
+            </dd>
+          </dl>
+
+          <dl class="dl-horizontal">
             <dt>
                 <span class="glyphicon glyphicon-question-sign get-help" title="Image files are stored in <code>build/tmp/deploy/images/</code>"></span>
                 Image files

index febf8af..32212ea 100644
@@ -69,11 +69,11 @@
 
 {%if build.outcome == build.SUCCEEDED%}
 <!-- built images -->
-{% if hasImages %}
-    <h2>Images</h2>
+  {% if hasArtifacts %}
+    <h2>Build artifacts</h2>
     {% for target in targets %}
         {% if target.target.is_image %}
-    <div class="well well-transparent dashboard-section">
+    <div class="well well-transparent dashboard-section" data-artifacts-for-target="{{target.target.pk}}">
         <h3><a href="{% url 'target' build.pk target.target.pk %}">{{target.target.target}}</a></h3>
         <dl class="dl-horizontal">
             <dt>Packages included</dt>

@@ -81,26 +81,7 @@
             <dt>Total package size</dt>
             <dd>{{target.pkgsz|filtered_filesizeformat}}</dd>
         </dl>
-        {% if target.targetHasNoImages %}
-            <div class="row">
-              <div class="col-md-7">
-                <div class="alert alert-info">
-                  <p>
-                  <strong>This build did not create any image files</strong>
-                  </p>
-                  <p>
-                  This is probably because valid image and license manifest
-                  files from a previous build already exist in your
-                  <code>build/tmp/deploy</code>
-                  directory. You can
-                  also <a href="{% url 'target' build.pk target.target.pk %}">view the
-                    license manifest information</a> in Toaster.
-                  </p>
-                </div>
-              </div>
-            </div>
-        {% endif %}
-        {% if not target.targetHasNoImages %}
+        {% if target.targetHasImages %}
           <dl class="dl-horizontal">
             <dt>
               Manifests

@@ -163,7 +144,7 @@
     </div>
         {% endif %}
     {% endfor %}
-{% endif %}
+  {% endif %}
 
 {%else%}
 <!-- error dump -->

84 lines not on this page · raw message

index 52b5e12..91189f6 100644
@@ -256,6 +256,10 @@ class ORMWrapper(object):
         target.license_manifest_path = license_manifest_path
         target.save()
 
+    def update_target_set_package_manifest(self, target, package_manifest_path):
+        target.package_manifest_path = package_manifest_path
+        target.save()
+
     def update_task_object(self, build, task_name, recipe_name, task_stats):
         """
         Find the task for build which matches the recipe and task name

@@ -1597,7 +1601,7 @@ class BuildInfoHelper(object):
         machine = self.server.runCommand(['getVariable', 'MACHINE'])[0]
         image_name = self.server.runCommand(['getVariable', 'IMAGE_NAME'])[0]
 
-        # location of the image_license.manifest files for this build;
+        # location of the manifest files for this build;
         # note that this file is only produced if an image is produced
         license_directory = \
             self.server.runCommand(['getVariable', 'LICENSE_DIRECTORY'])[0]

@@ -1636,6 +1640,11 @@ class BuildInfoHelper(object):
                 real_image_name,
                 'image_license.manifest')
 
+            image_package_manifest_path = os.path.join(
+                license_directory,
+                real_image_name,
+                'image_license.manifest')
+
             # if image_license.manifest exists, we can read the names of bzImage
             # and modules files for this build from it, then look for them
             # in the DEPLOY_DIR_IMAGE; note that this file is only produced

@@ -1657,11 +1666,20 @@ class BuildInfoHelper(object):
 
                 # store the license manifest path on the target
                 # (this file is also created any time an image file is created)
-                license_path = os.path.join(license_directory,
+                license_manifest_path = os.path.join(license_directory,
                     real_image_name, 'license.manifest')
 
                 self.orm_wrapper.update_target_set_license_manifest(
-                    image_target, license_path)
+                    image_target, license_manifest_path)
+
+                # store the package manifest path on the target (this file
+                # is created any time an image file is created)
+                package_manifest_path = os.path.join(deploy_dir_image,
+                    real_image_name + '.rootfs.manifest')
+
+                if os.path.exists(package_manifest_path):
+                    self.orm_wrapper.update_target_set_package_manifest(
+                        image_target, package_manifest_path)
 
             # scan the directory for image files relating to this build
             # (via real_image_name); note that we don't have to set

new file mode 100644
index 0000000..c958f30
@@ -0,0 +1,19 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('orm', '0008_refactor_artifact_models'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='target',
+            name='package_manifest_path',
+            field=models.CharField(null=True, max_length=500),
+        ),
+    ]

index a111916..8e40f0a 100644
@@ -622,6 +622,7 @@ class Target(models.Model):
     is_image = models.BooleanField(default = False)
     image_size = models.IntegerField(default=0)
     license_manifest_path = models.CharField(max_length=500, null=True)
+    package_manifest_path = models.CharField(max_length=500, null=True)
 
     def package_count(self):
         return Target_Installed_Package.objects.filter(target_id__exact=self.id).count()

@@ -729,9 +730,9 @@ class Target(models.Model):
         Target_Image_File object for an ext4 image being associated with a
         target for a project which didn't produce an ext4 image (for example).
 
-        Also sets the license_manifest_path of this target to the same path
-        as that of target being cloned from, as the license manifest path is
-        also a build artifact but is treated differently.
+        Also sets the license_manifest_path and package_manifest_path
+        of this target to the same path as that of target being cloned from, as
+        the manifests are also build artifacts but are treated differently.
         """
 
         image_fstypes = self.build.get_image_fstypes()

@@ -754,6 +755,7 @@ class Target(models.Model):
             kernel_file.save()
 
         self.license_manifest_path = target.license_manifest_path
+        self.package_manifest_path = target.package_manifest_path
         self.save()
 
     def clone_sdk_artifacts_from(self, target):

index 32212ea..36c28b7 100644
@@ -90,6 +90,12 @@
             <dd>
               <a href="{% url 'build_artifact' build.pk 'licensemanifest' target.target.pk %}">License manifest</a>
             </dd>
+
+            {% if target.target.package_manifest_path %}
+              <dd>
+                <a href="{% url 'build_artifact' build.pk 'packagemanifest' target.target.pk %}">Package manifest</a>
+              </dd>
+            {% endif %}
           </dl>
 
           <dl class="dl-horizontal">

index baaa288..aab6536 100755
@@ -2315,6 +2315,9 @@ if True:
         elif artifact_type == "licensemanifest":
             file_name = Target.objects.get(build = build, pk = artifact_id).license_manifest_path
 
+        elif artifact_type == "packagemanifest":
+            file_name = Target.objects.get(build = build, pk = artifact_id).package_manifest_path
+
         elif artifact_type == "tasklogfile":
             file_name = Task.objects.get(build = build, pk = artifact_id).logfile
 

index 8e40f0a..048399f 100644
@@ -449,18 +449,13 @@ class Build(models.Model):
                 break
         return has_images
 
-    def has_image_targets(self):
+    def has_image_recipes(self):
         """
         Returns True if a build has any targets which were built from
         image recipes.
         """
-        targets = Target.objects.filter(build_id=self.id)
-        has_image_targets = False
-        for target in targets:
-            if target.is_image:
-                has_image_targets = True
-                break
-        return has_image_targets
+        image_recipes = self.get_image_recipes()
+        return len(image_recipes) > 0
 
     def get_image_file_extensions(self):
         """

index 8d7c562..eb709bb 100644
@@ -117,7 +117,7 @@
         {% endwith %}
 
           <!-- new custom image from image recipe in this build -->
-          {% if build.has_image_targets %}
+          {% if build.has_image_recipes %}
             <button class="btn btn-default btn-block navbar-btn" data-role="new-custom-image-trigger">New custom image</button>
             {% include 'newcustomimage_modal.html' %}
             <script>

index 000937f..ef07858 100644
@@ -163,6 +163,13 @@ class SeleniumTestCase(StaticLiveServerTestCase):
         """ Find all elements matching CSS selector """
         return self.driver.find_elements_by_css_selector(selector)
 
+    def element_exists(self, selector):
+        """
+        Return True if one element matching selector exists,
+        False otherwise
+        """
+        return len(self.find_all(selector)) == 1
+
     def focused_element(self):
         """ Return the element which currently has focus on the page """
         return self.driver.switch_to.active_element

177 lines not on this page · raw message

index eb709bb..44749bf 100644
@@ -63,7 +63,7 @@
           <a  href="{% url 'builddashboard' build.pk %}">Build summary</a>
         </li>
         {% if build.has_images and build.outcome == build.SUCCEEDED %}
-          <li class="nav-header">Images</li>
+          <li class="nav-header" data-menu-heading="images">Images</li>
           {% block nav-target %}
             {% for t in build.get_sorted_target_list %}
               {% if t.has_images %}

index 36c28b7..bc41e23 100644
@@ -70,7 +70,7 @@
 {%if build.outcome == build.SUCCEEDED%}
 <!-- built images -->
   {% if hasArtifacts %}
-    <h2>Build artifacts</h2>
+    <h2 data-heading="build-artifacts">Build artifacts</h2>
     {% for target in targets %}
         {% if target.target.is_image %}
     <div class="well well-transparent dashboard-section" data-artifacts-for-target="{{target.target.pk}}">

@@ -88,12 +88,12 @@
             </dt>
 
             <dd>
-              <a href="{% url 'build_artifact' build.pk 'licensemanifest' target.target.pk %}">License manifest</a>
+              <a data-link="license-manifest" href="{% url 'build_artifact' build.pk 'licensemanifest' target.target.pk %}">License manifest</a>
             </dd>
 
             {% if target.target.package_manifest_path %}
               <dd>
-                <a href="{% url 'build_artifact' build.pk 'packagemanifest' target.target.pk %}">Package manifest</a>
+                <a data-link="package-manifest" href="{% url 'build_artifact' build.pk 'packagemanifest' target.target.pk %}">Package manifest</a>
               </dd>
             {% endif %}
           </dl>

@@ -104,7 +104,7 @@
                 Image files
             </dt>
             <dd>
-                <ul class="list-unstyled">
+                <ul class="list-unstyled" data-links="image-artifacts">
                     {% for i in target.imageFiles|dictsort:"suffix" %}
                         <li>
                             <a href="{% url 'build_artifact' build.pk 'imagefile' i.id %}">

@@ -119,7 +119,7 @@
                 Kernel artifacts
             </dt>
             <dd>
-                <ul class="list-unstyled">
+                <ul class="list-unstyled" data-links="kernel-artifacts">
                     {% for artifact in target.target_kernel_artifacts|dictsort:"basename" %}
                         <li>
                             <a href="{% url 'build_artifact' build.id 'targetkernelartifact' artifact.id %}">{{artifact.basename}}</a>

@@ -136,7 +136,7 @@
               SDK artifacts
             </dt>
             <dd>
-              <ul class="list-unstyled">
+              <ul class="list-unstyled" data-links="sdk-artifacts">
                 {% for artifact in target.target_sdk_artifacts|dictsort:"basename" %}
                 <li>
                   <a href="{% url 'build_artifact' build.id 'targetsdkartifact' artifact.id %}">{{artifact.basename}}</a>

index cdb0616..14e77a3 100644
@@ -48,6 +48,10 @@ class TestBuildDashboardPage(SeleniumTestCase):
                                            started_on=now,
                                            completed_on=now)
 
+        self.build3 = Build.objects.create(project=project,
+                                           started_on=now,
+                                           completed_on=now)
+
         # exception
         msg1 = 'an exception was thrown'
         self.exception_message = LogMessage.objects.create(

@@ -71,6 +75,11 @@ class TestBuildDashboardPage(SeleniumTestCase):
             layer=layer, build=self.build1
         )
 
+        # non-image recipes related to a build, for testing the new custom
+        # image button
+        layer_version2 = Layer_Version.objects.create(layer=layer,
+            build=self.build3)
+
         # image recipes
         self.image_recipe1 = Recipe.objects.create(
             name='recipeA',

@@ -248,3 +257,14 @@ class TestBuildDashboardPage(SeleniumTestCase):
         ]
 
         self._check_labels_in_modal(modal, expected_recipes)
+
+    def test_new_custom_image_button_no_image(self):
+        """
+        Check that a build which builds non-image recipes doesn't show
+        the new custom image button on the dashboard.
+        """
+        self._get_build_dashboard(self.build3)
+        selector = '[data-role="new-custom-image-trigger"]'
+        self.assertFalse(self.element_exists(selector),
+            'new custom image button should not show for builds which ' \
+            'don\'t have any image recipes')

index 18e4475..39b0e20 100644
@@ -24,8 +24,9 @@ from django.utils import timezone
 
 from tests.browser.selenium_helpers import SeleniumTestCase
 
-from orm.models import Project, Release, BitbakeVersion, Build, Target
+from orm.models import Project, Release, BitbakeVersion, Build, Target, Package
 from orm.models import Target_Image_File, TargetSDKFile, TargetKernelFile
+from orm.models import Target_Installed_Package
 
 class TestBuildDashboardPageArtifacts(SeleniumTestCase):
     """ Tests for artifacts on the build dashboard /build/X """

@@ -86,6 +87,8 @@ class TestBuildDashboardPageArtifacts(SeleniumTestCase):
         """
         If a build produced SDK artifacts, they should be shown, but the section
         for image files and the images menu option should be hidden.
+
+        The packages count and size should also be hidden.
         """
         now = timezone.now()
         build = Build.objects.create(project=self.project,

@@ -120,11 +123,28 @@ class TestBuildDashboardPageArtifacts(SeleniumTestCase):
         self.assertEqual(len(sdk_artifact_links), 2,
             'should be links to 2 SDK artifacts')
 
+        # package count and size should not be visible, no link on
+        # target name
+        selector = '[data-value="target-package-count"]'
+        self.assertFalse(self.element_exists(selector),
+            'package count should not be shown for non-image builds')
+
+        selector = '[data-value="target-package-size"]'
+        self.assertFalse(self.element_exists(selector),
+            'package size should not be shown for non-image builds')
+
+        selector = '[data-link="target-packages"]'
+        self.assertFalse(self.element_exists(selector),
+            'link to target packages should not be on target heading')
+
     def test_image_artifacts(self):
         """
         If a build produced image files, kernel artifacts, and manifests,
         they should all be shown, as well as the image link in the left-hand
         menu.
+
+        The packages count and size should be shown, with a link to the
+        package display page.
         """
         now = timezone.now()
         build = Build.objects.create(project=self.project,

@@ -145,6 +165,11 @@ class TestBuildDashboardPageArtifacts(SeleniumTestCase):
         kernel_file2 = TargetKernelFile.objects.create(target=target,
             file_name='/home/foo/bzImage', file_size=2000)
 
+        package = Package.objects.create(build=build, name='foo', size=1024,
+            installed_name='foo1')
+        installed_package = Target_Installed_Package.objects.create(
+            target=target, package=package)
+
         self._get_build_dashboard(build)
 
         # check build artifacts heading

@@ -175,3 +200,18 @@ class TestBuildDashboardPageArtifacts(SeleniumTestCase):
         selector = 'a[data-link="package-manifest"]'
         self.assertTrue(self.element_exists(selector),
             'should be a link to the package manifest (selector %s)' % selector)
+
+        # check package count and size, link on target name
+        selector = '[data-value="target-package-count"]'
+        element = self.find(selector)
+        self.assertEquals(element.text, '1',
+            'package count should be shown for image builds')
+
+        selector = '[data-value="target-package-size"]'
+        element = self.find(selector)
+        self.assertEquals(element.text, '1.0 KB',
+            'package size should be shown for image builds')
+
+        selector = '[data-link="target-packages"]'
+        self.assertTrue(self.element_exists(selector),
+            'link to target packages should be on target heading')

index bc41e23..07fc26c 100644
@@ -74,13 +74,28 @@
     {% for target in targets %}
         {% if target.target.is_image %}
     <div class="well well-transparent dashboard-section" data-artifacts-for-target="{{target.target.pk}}">
-        <h3><a href="{% url 'target' build.pk target.target.pk %}">{{target.target.target}}</a></h3>
-        <dl class="dl-horizontal">
-            <dt>Packages included</dt>
-            <dd><a href="{% url 'target' build.pk target.target.pk %}">{{target.npkg}}</a></dd>
-            <dt>Total package size</dt>
-            <dd>{{target.pkgsz|filtered_filesizeformat}}</dd>
-        </dl>
+        {% if target.npkg > 0 %}
+            <h3>
+                <a href="{% url 'target' build.pk target.target.pk %}" data-link="target-packages">
+                  {{target.target.target}}
+                </a>
+            </h3>
+            <dl class="dl-horizontal">
+                <dt>Packages included</dt>
+                <dd>
+                    <a href="{% url 'target' build.pk target.target.pk %}">
+                        <span data-value="target-package-count">{{target.npkg}}</span>
+                    </a>
+                </dd>
+                <dt>Total package size</dt>
+                <dd>
+                    <span data-value="target-package-size">{{target.pkgsz|filtered_filesizeformat}}</span>
+                </dd>
+            </dl>
+        {% else %}
+            <h3>{{target.target.target}}</h3>
+        {% endif %}
+
         {% if target.targetHasImages %}
           <dl class="dl-horizontal">
             <dt>

index c9b6b84..48c01c0 100644
@@ -1175,8 +1175,8 @@ class RunQueue:
                 pass
             self.state = runQueueComplete
             raise
-        except:
-            logger.error("An uncaught exception occured in runqueue, please see the failure below:")
+        except Exception as err:
+            logger.exception("An uncaught exception occurred in runqueue")
             try:
                 self.teardown_workers()
             except:

index c9b6b84..48c01c0 100644
@@ -1175,8 +1175,8 @@ class RunQueue:
                 pass
             self.state = runQueueComplete
             raise
-        except:
-            logger.error("An uncaught exception occured in runqueue, please see the failure below:")
+        except Exception as err:
+            logger.exception("An uncaught exception occurred in runqueue")
             try:
                 self.teardown_workers()
             except: