Add initial version of Toaster web UI
cover Add initial version of Toaster web UI
This adds the initial implementation of Toaster, a web frontend for BitBake (formerly known as Web Hob).
Note: 2/3 is omitted because it contains files with lines longer than 998 characters that git send-email chokes on. You can still see it in the branch using the web link below.
The following changes (against poky, but apply cleanly to bitbake master with -p2) are available in the git repository at:
git://git.yoctoproject.org/poky-contrib paule/toaster
http://git.yoctoproject.org/cgit.cgi/poky-contrib/log/?h=paule/toaster
Alexandru DAMIAN (3):
toaster: add toaster code to bitbake
toaster: adding frameworks for the Simple UI
toaster: add Toaster UI interface
bitbake/LICENSE | 10 + bitbake/bin/toaster | 145 + bitbake/lib/bb/ui/buildinfohelper.py | 719 +++ bitbake/lib/bb/ui/toasterui.py | 273 ++ bitbake/lib/toaster/__init__.py | 0 bitbake/lib/toaster/bldviewer/__init__.py | 0 bitbake/lib/toaster/bldviewer/api.py | 37 + .../lib/toaster/bldviewer/static/css/bootstrap.css | 4797 ++++++++++++++++++++ .../lib/toaster/bldviewer/static/js/bootstrap.js | 1982 ++++++++ .../toaster/bldviewer/static/js/jquery-2.0.3.js | 6 + bitbake/lib/toaster/bldviewer/templates/base.html | 30 + .../toaster/bldviewer/templates/basebuildpage.html | 17 + .../lib/toaster/bldviewer/templates/basetable.html | 46 + bitbake/lib/toaster/bldviewer/templates/bfile.html | 24 + .../lib/toaster/bldviewer/templates/bpackage.html | 44 + bitbake/lib/toaster/bldviewer/templates/build.html | 43 + .../toaster/bldviewer/templates/configuration.html | 20 + bitbake/lib/toaster/bldviewer/templates/layer.html | 34 + .../lib/toaster/bldviewer/templates/package.html | 36 + .../lib/toaster/bldviewer/templates/recipe.html | 54 + bitbake/lib/toaster/bldviewer/templates/task.html | 63 + .../lib/toaster/bldviewer/templatetags/__init__.py | 0 .../toaster/bldviewer/templatetags/projecttags.py | 26 + bitbake/lib/toaster/bldviewer/urls.py | 32 + bitbake/lib/toaster/bldviewer/views.py | 260 ++ bitbake/lib/toaster/manage.py | 10 + bitbake/lib/toaster/orm/__init__.py | 0 bitbake/lib/toaster/orm/models.py | 258 ++ bitbake/lib/toaster/toastergui/__init__.py | 0 .../lib/toaster/toastergui/static/images/yocto.jpg | Bin 0 -> 6582 bytes .../lib/toaster/toastergui/templates/index.html | 13 + bitbake/lib/toaster/toastergui/urls.py | 27 + bitbake/lib/toaster/toastergui/views.py | 26 + bitbake/lib/toaster/toastermain/__init__.py | 0 bitbake/lib/toaster/toastermain/settings.py | 190 + bitbake/lib/toaster/toastermain/urls.py | 41 + bitbake/lib/toaster/toastermain/wsgi.py | 32 + 37 files changed, 9295 insertions(+) create mode 100644 bitbake/LICENSE create mode 100755 bitbake/bin/toaster create mode 100644 bitbake/lib/bb/ui/buildinfohelper.py create mode 100644 bitbake/lib/bb/ui/toasterui.py create mode 100644 bitbake/lib/toaster/__init__.py create mode 100644 bitbake/lib/toaster/bldviewer/__init__.py create mode 100644 bitbake/lib/toaster/bldviewer/api.py create mode 100644 bitbake/lib/toaster/bldviewer/static/css/bootstrap.css create mode 100644 bitbake/lib/toaster/bldviewer/static/js/bootstrap.js create mode 100644 bitbake/lib/toaster/bldviewer/static/js/jquery-2.0.3.js create mode 100644 bitbake/lib/toaster/bldviewer/templates/base.html create mode 100644 bitbake/lib/toaster/bldviewer/templates/basebuildpage.html create mode 100644 bitbake/lib/toaster/bldviewer/templates/basetable.html create mode 100644 bitbake/lib/toaster/bldviewer/templates/bfile.html create mode 100644 bitbake/lib/toaster/bldviewer/templates/bpackage.html create mode 100644 bitbake/lib/toaster/bldviewer/templates/build.html create mode 100644 bitbake/lib/toaster/bldviewer/templates/configuration.html create mode 100644 bitbake/lib/toaster/bldviewer/templates/layer.html create mode 100644 bitbake/lib/toaster/bldviewer/templates/package.html create mode 100644 bitbake/lib/toaster/bldviewer/templates/recipe.html create mode 100644 bitbake/lib/toaster/bldviewer/templates/task.html create mode 100644 bitbake/lib/toaster/bldviewer/templatetags/__init__.py create mode 100644 bitbake/lib/toaster/bldviewer/templatetags/projecttags.py create mode 100644 bitbake/lib/toaster/bldviewer/urls.py create mode 100644 bitbake/lib/toaster/bldviewer/views.py create mode 100755 bitbake/lib/toaster/manage.py create mode 100644 bitbake/lib/toaster/orm/__init__.py create mode 100644 bitbake/lib/toaster/orm/models.py create mode 100644 bitbake/lib/toaster/toastergui/__init__.py create mode 100644 bitbake/lib/toaster/toastergui/static/images/yocto.jpg create mode 100644 bitbake/lib/toaster/toastergui/templates/index.html create mode 100644 bitbake/lib/toaster/toastergui/urls.py create mode 100644 bitbake/lib/toaster/toastergui/views.py create mode 100644 bitbake/lib/toaster/toastermain/__init__.py create mode 100644 bitbake/lib/toaster/toastermain/settings.py create mode 100644 bitbake/lib/toaster/toastermain/urls.py create mode 100644 bitbake/lib/toaster/toastermain/wsgi.py
Signature · 1 line
1.8.1.2
PATCH 1/3 toaster: add toaster code to bitbake
From: Alexandru DAMIAN <[email protected]>
This patch adds the Toaster component to Bitbake.
Toaster is a module designed to record the progress of a Bitbake build, and data about the resultant artifacts.
It contains a web-based interface and a REST API allowing post-facto inspection of the build process and artifacts.
Features present in this build:
* toaster start script
* relational data model
* Django boilerplate code
* the REST API
* the Simple UI web interface
This patch has all the development history squashed together.
Code portions contributed by Calin Dragomir <[email protected]>.
--- bitbake/bin/toaster | 145 ++++++++++++ bitbake/lib/toaster/__init__.py | 0 bitbake/lib/toaster/bldviewer/__init__.py | 0 bitbake/lib/toaster/bldviewer/api.py | 37 +++ bitbake/lib/toaster/bldviewer/templates/base.html | 30 +++ .../toaster/bldviewer/templates/basebuildpage.html | 17 ++ .../lib/toaster/bldviewer/templates/basetable.html | 46 ++++ bitbake/lib/toaster/bldviewer/templates/bfile.html | 24 ++ .../lib/toaster/bldviewer/templates/bpackage.html | 44 ++++ bitbake/lib/toaster/bldviewer/templates/build.html | 43 ++++ .../toaster/bldviewer/templates/configuration.html | 20 ++ bitbake/lib/toaster/bldviewer/templates/layer.html | 34 +++ .../lib/toaster/bldviewer/templates/package.html | 36 +++ .../lib/toaster/bldviewer/templates/recipe.html | 54 +++++ bitbake/lib/toaster/bldviewer/templates/task.html | 63 +++++ .../lib/toaster/bldviewer/templatetags/__init__.py | 0 .../toaster/bldviewer/templatetags/projecttags.py | 26 +++ bitbake/lib/toaster/bldviewer/urls.py | 32 +++ bitbake/lib/toaster/bldviewer/views.py | 260 +++++++++++++++++++++ bitbake/lib/toaster/manage.py | 10 + bitbake/lib/toaster/orm/__init__.py | 0 bitbake/lib/toaster/orm/models.py | 258 ++++++++++++++++++++ bitbake/lib/toaster/toastergui/__init__.py | 0 .../lib/toaster/toastergui/static/images/yocto.jpg | Bin 0 -> 6582 bytes .../lib/toaster/toastergui/templates/index.html | 13 ++ bitbake/lib/toaster/toastergui/urls.py | 27 +++ bitbake/lib/toaster/toastergui/views.py | 26 +++ bitbake/lib/toaster/toastermain/__init__.py | 0 bitbake/lib/toaster/toastermain/settings.py | 190 +++++++++++++++ bitbake/lib/toaster/toastermain/urls.py | 41 ++++ bitbake/lib/toaster/toastermain/wsgi.py | 32 +++ 31 files changed, 1508 insertions(+) create mode 100755 bitbake/bin/toaster create mode 100644 bitbake/lib/toaster/__init__.py create mode 100644 bitbake/lib/toaster/bldviewer/__init__.py create mode 100644 bitbake/lib/toaster/bldviewer/api.py create mode 100644 bitbake/lib/toaster/bldviewer/templates/base.html create mode 100644 bitbake/lib/toaster/bldviewer/templates/basebuildpage.html create mode 100644 bitbake/lib/toaster/bldviewer/templates/basetable.html create mode 100644 bitbake/lib/toaster/bldviewer/templates/bfile.html create mode 100644 bitbake/lib/toaster/bldviewer/templates/bpackage.html create mode 100644 bitbake/lib/toaster/bldviewer/templates/build.html create mode 100644 bitbake/lib/toaster/bldviewer/templates/configuration.html create mode 100644 bitbake/lib/toaster/bldviewer/templates/layer.html create mode 100644 bitbake/lib/toaster/bldviewer/templates/package.html create mode 100644 bitbake/lib/toaster/bldviewer/templates/recipe.html create mode 100644 bitbake/lib/toaster/bldviewer/templates/task.html create mode 100644 bitbake/lib/toaster/bldviewer/templatetags/__init__.py create mode 100644 bitbake/lib/toaster/bldviewer/templatetags/projecttags.py create mode 100644 bitbake/lib/toaster/bldviewer/urls.py create mode 100644 bitbake/lib/toaster/bldviewer/views.py create mode 100755 bitbake/lib/toaster/manage.py create mode 100644 bitbake/lib/toaster/orm/__init__.py create mode 100644 bitbake/lib/toaster/orm/models.py create mode 100644 bitbake/lib/toaster/toastergui/__init__.py create mode 100644 bitbake/lib/toaster/toastergui/static/images/yocto.jpg create mode 100644 bitbake/lib/toaster/toastergui/templates/index.html create mode 100644 bitbake/lib/toaster/toastergui/urls.py create mode 100644 bitbake/lib/toaster/toastergui/views.py create mode 100644 bitbake/lib/toaster/toastermain/__init__.py create mode 100644 bitbake/lib/toaster/toastermain/settings.py create mode 100644 bitbake/lib/toaster/toastermain/urls.py create mode 100644 bitbake/lib/toaster/toastermain/wsgi.py
bitbake/bin/toaster new file +145 -0
@@ -0,0 +1,145 @@ +#!/bin/bash +# (c) 2013 Intel Corp. + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + + +# This script enables toaster event logging and +# starts bitbake resident server +# use as: source toaster [start|stop] + +# Helper function to kill a background toaster development server + +function webserverKillAll() +{ + local pidfile + for pidfile in ${BUILDDIR}/.toastermain.pid; do + if [ -f ${pidfile} ]; then + while kill -0 $(< ${pidfile}) 2>/dev/null; do + kill -SIGTERM -$(< ${pidfile}) 2>/dev/null + sleep 1; + done; + rm ${pidfile} + fi + done +} + + +function webserverStartAll() +{ + retval=0 + python $BBBASEDIR/lib/toaster/manage.py syncdb || retval=1 + if [ $retval -eq 1 ]; then + echo "Failed db sync, stopping system start" 1>&2 + else + python $BBBASEDIR/lib/toaster/manage.py runserver 0.0.0.0:8000 </dev/null >${BUILDDIR}/toaster_web.log 2>&1 & echo $! >${BUILDDIR}/.toastermain.pid + fi + return $retval +} + + +# We make sure we're running in the current shell and in a good environment + +if [ -z "$ZSH_NAME" ] && [ `basename \"$0\"` = `basename \"$BASH_SOURCE\"` ]; then + echo "Error: This script needs to be sourced. Please run as 'source toaster [start|stop]'" 1>&2; + exit 1 +fi + +if [ -z "$BUILDDIR" ] || [ -z `which bitbake` ]; then + echo "Error: Build environment is not setup or bitbake is not in path." 1>&2; + return 2 +fi + +BBBASEDIR=`dirname ${BASH_SOURCE}`/.. + + +# Verify prerequisites + +if ! echo "import django; print (1,4,5) == django.VERSION[0:3]" | python 2>/dev/null | grep True >/dev/null; then + echo -e "This program needs Django 1.4.5. Please install with\n\nsudo pip install django==1.4.5" + return 2 +fi + + + +# Determine the action. If specified by arguments, fine, if not, toggle it +if [ "x$1" == "xstart" ] || [ "x$1" == "xstop" ]; then + CMD="$1" +else + if [ -z "$BBSERVER" ]; then + CMD="start" + else + CMD="stop" + fi; +fi + +NOTOASTERUI=0 +if [ "x$2" == "xnoui" ]; then + NOTOASTERUI=1 +fi + +echo "The system will $CMD." + +# Make sure it's safe to run by checking bitbake lock + +lock=1 +if [ -e $BUILDDIR/bitbake.lock ]; then + (flock -n 200 ) 200<$BUILDDIR/bitbake.lock || lock=0 +fi + +if [ ${CMD} == "start" ] && ( [ $lock -eq 0 ] || [ -e $BUILDDIR/.toastermain.pid ] ); then + echo "Error: bitbake lock state error. System is already on." 2>&1 + return 3 +elif [ ${CMD} == "stop" ] && ( [ $lock -eq 1 ] || ! [ -e $BUILDDIR/.toastermain.pid ] ) ; then + echo "Error: bitbake lock state error. Trying to stop a stopped system ? +If you think the system is hanged up, you can try to manually stop system with the commands + +# BBSERVER=localhost:8200 bitbake -m + +and + +# webserverKillAll +" 2>&1 + return 3 +fi + + +# Execute the commands + +case $CMD in + start ) + webserverStartAll || return 4 + unset BBSERVER + bitbake --server-only -t xmlrpc -B localhost:8200 + export BBSERVER=localhost:8200 + if [ $NOTOASTERUI == 0 ]; then # we start the TOASTERUI only if not inhibited + bitbake --observe-only -u toasterui >${BUILDDIR}/toaster_ui.log 2>&1 & echo $! >${BUILDDIR}/.toasterui.pid + fi + ;; + stop ) + if [ -f ${BUILDDIR}/.toasterui.pid ]; then + kill $(< ${BUILDDIR}/.toasterui.pid ) + rm ${BUILDDIR}/.toasterui.pid + fi + bitbake -m + unset BBSERVER + webserverKillAll + # force stop any misbehaving bitbake server + lsof bitbake.lock | awk '{print $2}' | grep "[0-9]\+" | xargs -n1 -r kill + ;; +esac + +echo "Successful ${CMD}." +
bitbake/lib/toaster/__init__.py new file +0 -0
bitbake/lib/toaster/bldviewer/__init__.py new file +0 -0
bitbake/lib/toaster/bldviewer/api.py new file +37 -0
@@ -0,0 +1,37 @@ +# +# BitBake Toaster Implementation +# +# Copyright (C) 2013 Intel Corporation +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +from django.conf.urls import patterns, include, url + + +urlpatterns = patterns('bldviewer.views', + url(r'^builds$', 'model_explorer', {'model_name':'build'}, name='builds'), + url(r'^targets$', 'model_explorer', {'model_name':'target'}, name='targets'), + url(r'^tasks$', 'model_explorer', {'model_name':'task'}, name='task'), + url(r'^task_dependencies$', 'model_explorer', {'model_name':'task_dependency'}, name='task_dependencies'), + url(r'^packages$', 'model_explorer', {'model_name':'build_package'}, name='build_packages'), + url(r'^package_dependencies$', 'model_explorer', {'model_name':'build_package_dependency'}, name='build_package_dependencies'), + url(r'^target_packages$', 'model_explorer', {'model_name':'target_package'}, name='target_packages'), + url(r'^package_files$', 'model_explorer', {'model_name':'build_file'}, name='build_files'), + url(r'^layers$', 'model_explorer', {'model_name':'layer'}, name='layer'), + url(r'^layerversions$', 'model_explorer', {'model_name':'layerversion'}, name='layerversion'), + url(r'^recipes$', 'model_explorer', {'model_name':'recipe'}, name='recipe'), + url(r'^recipe_dependencies$', 'model_explorer', {'model_name':'recipe_dependency'}, name='recipe_dependencies'), + url(r'^variables$', 'model_explorer', {'model_name':'variable'}, name='variables'), + url(r'^logmessages$', 'model_explorer', {'model_name':'logmessage'}, name='logmessages'), +)
bitbake/lib/toaster/bldviewer/templates/base.html new file +30 -0
@@ -0,0 +1,30 @@ +<!DOCTYPE html> +{% load static %} +<html> + <head> + <title>Toaster Simple Explorer</title> +<script src="{% static 'js/jquery-2.0.3.js' %}"> +</script> +<script src="{% static 'js/bootstrap.js' %}"> +</script> +<link href="{% static 'css/bootstrap.css' %}" rel="stylesheet" type="text/css"> + </head> + +<body style="height: 100%"> +<div style="width:100%; height: 100%; position:absolute"> +<div style="width: 100%; height: 3em" class="nav"> + <ul class="nav nav-tabs"> + <li><a href="{% url all-builds %}">All Builds</a></li> + <li><a href="{% url all-layers %}">All Layers</a></li> + </ul> +</div> + +<div style="overflow-y:scroll; width: 100%; position: absolute; top: 3em; bottom:70px "> +{% block pagecontent %} +{% endblock %} +</div> +<div class="navbar" style="position: absolute; bottom: 0; width:100%"><br/>About Toaster | Yocto Project </div> +</div> +</body> +</html> +
bitbake/lib/toaster/bldviewer/templates/basebuildpage.html new file +17 -0
@@ -0,0 +1,17 @@ +{% extends "basetable.html" %} + +{% block pagename %} +<ul class="nav nav-tabs" style="display: inline-block"> + <li><a>Build {{build.target_set.all|join:" "}} at {{build.started_on}} : </a></li> + <li><a href="{% url task build.id %}"> Tasks </a></li> + <li><a href="{% url bpackage build.id %}"> Build Packages </a></li> + {% for t in build.target_set.all %} + {% if t.is_image %} + <li><a href="{% url tpackage build.id t.pk %}"> Packages for {{t.target}} </a> </li> + {% endif %} + {% endfor %} + <li><a href="{% url configuration build.id %}"> Configuration </a> </li> +</ul> + <h1>Toaster - Build {% block pagetitle %} {% endblock %}</h1> +{% endblock %} +
bitbake/lib/toaster/bldviewer/templates/basetable.html new file +46 -0
46 lines not on this page · raw message
bitbake/lib/toaster/bldviewer/templates/bfile.html new file +24 -0
@@ -0,0 +1,24 @@ +{% extends "basebuildpage.html" %} + +{% block pagetitle %}Files for package {{files.0.bpackage.name}} {% endblock %} +{% block pagetable %} + {% if not files %} + <p>No files were recorded for this package!</p> + {% else %} + + <tr> + <th>Name</th> + <th>Size (Bytes)</th> + </tr> + + {% for file in files %} + + <tr class="data"> + <td>{{file.path}}</td> + <td>{{file.size}}</td> + + {% endfor %} + + {% endif %} + +{% endblock %}
bitbake/lib/toaster/bldviewer/templates/bpackage.html new file +44 -0
44 lines not on this page · raw message
bitbake/lib/toaster/bldviewer/templates/build.html new file +43 -0
43 lines not on this page · raw message
bitbake/lib/toaster/bldviewer/templates/configuration.html new file +20 -0
@@ -0,0 +1,20 @@ +{% extends "basebuildpage.html" %} + +{% block pagetitle %}Configuration{% endblock %} +{% block pagetable %} + + <tr> + <th>Name</th> + <th>Value</th> + <th>Description</th> + </tr> + + {% for variable in configuration %} + + <tr class="data"> + <td>{{variable.variable_name}}</td> + <td>{{variable.variable_value}}</td> + <td>{% if variable.description %}{{variable.description}}{% endif %}</td> + {% endfor %} + +{% endblock %}
bitbake/lib/toaster/bldviewer/templates/layer.html new file +34 -0
@@ -0,0 +1,34 @@ +{% extends "basetable.html" %} + +{% block pagename %} + <h1>Toaster - Layers</h1> +{% endblock %} + +{% block pagetable %} + {% load projecttags %} + + <tr> + <th>Name</th> + <th>Local Path</th> + <th>Layer Index URL</th> + <th>Known Versions</th> + </tr> + + {% for layer in layers %} + + <tr class="data"> + <td>{{layer.name}}</td> + <td>{{layer.local_path}}</td> + <td><a href='{{layer.layer_index_url}}'>{{layer.layer_index_url}}</a></td> + <td><table> + {% for lv in layer.versions %} + <tr><td> + <a href="{% url layer_versions_recipes lv.id %}">({{lv.priority}}){{lv.branch}}:{{lv.commit}} ({{lv.count}} recipes)</a> + </td></tr> + {% endfor %} + </table></td> + </tr> + + {% endfor %} + +{% endblock %}
bitbake/lib/toaster/bldviewer/templates/package.html new file +36 -0
@@ -0,0 +1,36 @@ +{% extends "basebuildpage.html" %} + +{% block pagetable %} + {% if not packages %} + <p>No packages were recorded for this target!</p> + {% else %} + + <tr> + <th>Name</th> + <th>Version</th> + <th>Size (Bytes)</th> + <th>Recipe</th> + <th>Depends on</th> + </tr> + + {% for package in packages %} + + <tr class="data"> + <td><a name="#{{package.name}}">{{package.name}}</a></td> + <td>{{package.version}}</td> + <td>{{package.size}}</td> + <td><a name="{{package.recipe.name}}.{{package.package_name}}"> + <a href="{% url layer_versions_recipes package.recipe.layer_version_id %}#{{package.recipe.name}}">{{package.recipe.name}}</a>{{package.package_name}}</a></td> + <td> + <div style="height: 3em; overflow:auto"> + {% for d in package.depends_on %} + <a href="#{{d.name}}">{{d.name}}</a><br/> + {% endfor %} + </div> + </td> + + {% endfor %} + + {% endif %} + +{% endblock %}
bitbake/lib/toaster/bldviewer/templates/recipe.html new file +54 -0
54 lines not on this page · raw message
bitbake/lib/toaster/bldviewer/templates/task.html new file +63 -0
63 lines not on this page · raw message
bitbake/lib/toaster/bldviewer/templatetags/__init__.py new file +0 -0
bitbake/lib/toaster/bldviewer/templatetags/projecttags.py new file +26 -0
@@ -0,0 +1,26 @@ +# +# BitBake Toaster Implementation +# +# Copyright (C) 2013 Intel Corporation +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +from datetime import datetime +from django import template + +register = template.Library() + [email protected]_tag +def time_difference(start_time, end_time): + return end_time - start_time
bitbake/lib/toaster/bldviewer/urls.py new file +32 -0
@@ -0,0 +1,32 @@ +# +# BitBake Toaster Implementation +# +# Copyright (C) 2013 Intel Corporation +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +from django.conf.urls import patterns, include, url +from django.views.generic.simple import redirect_to + +urlpatterns = patterns('bldviewer.views', + url(r'^builds/$', 'build', name='all-builds'), + url(r'^build/(?P<build_id>\d+)/task/$', 'task', name='task'), + url(r'^build/(?P<build_id>\d+)/packages/$', 'bpackage', name='bpackage'), + url(r'^build/(?P<build_id>\d+)/package/(?P<package_id>\d+)/files/$', 'bfile', name='bfile'), + url(r'^build/(?P<build_id>\d+)/target/(?P<target_id>\d+)/packages/$', 'tpackage', name='tpackage'), + url(r'^build/(?P<build_id>\d+)/configuration/$', 'configuration', name='configuration'), + url(r'^layers/$', 'layer', name='all-layers'), + url(r'^layerversions/(?P<layerversion_id>\d+)/recipes/.*$', 'layer_versions_recipes', name='layer_versions_recipes'), + url(r'^$', redirect_to, {'url': 'builds/'}), +)
bitbake/lib/toaster/bldviewer/views.py new file +260 -0
260 lines not on this page · raw message
bitbake/lib/toaster/manage.py new file +10 -0
@@ -0,0 +1,10 @@ +#!/usr/bin/env python +import os +import sys + +if __name__ == "__main__": + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "toastermain.settings") + + from django.core.management import execute_from_command_line + + execute_from_command_line(sys.argv)
bitbake/lib/toaster/orm/__init__.py new file +0 -0
bitbake/lib/toaster/orm/models.py new file +258 -0
258 lines not on this page · raw message
bitbake/lib/toaster/toastergui/__init__.py new file +0 -0
bitbake/lib/toaster/toastergui/static/images/yocto.jpg new file binary +0 -0
bitbake/lib/toaster/toastergui/templates/index.html new file +13 -0
@@ -0,0 +1,13 @@ +<html> +<head> + <title>GUI Page</title> +</head> +<body> +{% load staticfiles %} + +<img src="{% static "/static/images/yocto.jpg" %}" alt="Yocto"/> + +This is your basic index page! +</body> + +</html>
bitbake/lib/toaster/toastergui/urls.py new file +27 -0
@@ -0,0 +1,27 @@ +# +# BitBake Toaster Implementation +# +# Copyright (C) 2013 Intel Corporation +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import os + +from django.conf import settings +from django.conf.urls import patterns, include, url + + +urlpatterns = patterns('toastergui.views', + url(r'^$', 'guihome', name='guihome'), +)
bitbake/lib/toaster/toastergui/views.py new file +26 -0
@@ -0,0 +1,26 @@ +# +# BitBake Toaster Implementation +# +# Copyright (C) 2013 Intel Corporation +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +from django.shortcuts import render +from orm.models import Build, Task + + +def guihome(request): + template = 'index.html' + + return render(request, template)
bitbake/lib/toaster/toastermain/__init__.py new file +0 -0
bitbake/lib/toaster/toastermain/settings.py new file +190 -0
190 lines not on this page · raw message
bitbake/lib/toaster/toastermain/urls.py new file +41 -0
41 lines not on this page · raw message
bitbake/lib/toaster/toastermain/wsgi.py new file +32 -0
@@ -0,0 +1,32 @@ +""" +WSGI config for Toaster project. + +This module contains the WSGI application used by Django's development server +and any production WSGI deployments. It should expose a module-level variable +named ``application``. Django's ``runserver`` and ``runfcgi`` commands discover +this application via the ``WSGI_APPLICATION`` setting. + +Usually you will have the standard Django WSGI application here, but it also +might make sense to replace the whole Django WSGI application with a custom one +that later delegates to the Django one. For example, you could introduce WSGI +middleware here, or combine a Django application with an application of another +framework. + +""" +import os + +# We defer to a DJANGO_SETTINGS_MODULE already in the environment. This breaks +# if running multiple sites in the same mod_wsgi process. To fix this, use +# mod_wsgi daemon mode with each site in its own daemon process, or use +# os.environ["DJANGO_SETTINGS_MODULE"] = "Toaster.settings" +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "toastermain.settings") + +# This application object is used by any WSGI server configured to use this +# file. This includes Django's development server, if the WSGI_APPLICATION +# setting points here. +from django.core.wsgi import get_wsgi_application +application = get_wsgi_application() + +# Apply WSGI middleware here. +# from helloworld.wsgi import HelloWorldApplication +# application = HelloWorldApplication(application)
Signature · 1 line
1.8.1.2
PATCH 3/3 toaster: add Toaster UI interface
In reply to Paul EggletonFrom: Alexandru DAMIAN <[email protected]>
Adding a new bitbake UI interface named 'toasterui'.
'toasterui' listens for events and data coming from a bitbake server during a run, and records it in a data store using the Toaster object model.
Adds a helper class named BuildInfoHelper that
reconstructs the state of the bitbake server and
saves relevant data to the data store.
Code portions contributed by Calin Dragomir <[email protected]>.
--- bitbake/lib/bb/ui/buildinfohelper.py | 719 +++++++++++++++++++++++++++++++++++ bitbake/lib/bb/ui/toasterui.py | 273 +++++++++++++ 2 files changed, 992 insertions(+) create mode 100644 bitbake/lib/bb/ui/buildinfohelper.py create mode 100644 bitbake/lib/bb/ui/toasterui.py
bitbake/lib/bb/ui/buildinfohelper.py new file +719 -0
@@ -0,0 +1,719 @@ +# +# BitBake ToasterUI Implementation +# +# Copyright (C) 2013 Intel Corporation +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import datetime +import sys +import bb +import re +import subprocess + + +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "toaster.toastermain.settings") + +import toaster.toastermain.settings as toaster_django_settings +from toaster.orm.models import Build, Task, Recipe, Layer_Version, Layer, Target, LogMessage +from toaster.orm.models import Target_Package, Build_Package, Variable, Build_File +from toaster.orm.models import Task_Dependency, Build_Package_Dependency, Target_Package_Dependency, Recipe_Dependency +from bb.msg import BBLogFormatter as format + +class ORMWrapper(object): + """ This class creates the dictionaries needed to store information in the database + following the format defined by the Django models. It is also used to save this + information in the database. + """ + + def __init__(self): + pass + + + def create_build_object(self, build_info): + + build = Build.objects.create( + machine=build_info['machine'], + image_fstypes=build_info['image_fstypes'], + distro=build_info['distro'], + distro_version=build_info['distro_version'], + started_on=build_info['started_on'], + completed_on=build_info['completed_on'], + cooker_log_path=build_info['cooker_log_path'], + build_name=build_info['build_name'], + bitbake_version=build_info['bitbake_version']) + + return build + + def create_target_objects(self, target_info): + targets = [] + for tgt_name in target_info['targets']: + tgt_object = Target.objects.create( build = target_info['build'], + target = tgt_name, + is_image = False, + file_name = "", + file_size = 0); + targets.append(tgt_object) + return targets + + def update_build_object(self, build, errors, warnings, taskfailures): + + outcome = Build.SUCCEEDED + if errors or taskfailures: + outcome = Build.FAILED + + build.completed_on = datetime.datetime.now() + build.errors_no = errors + build.warnings_no = warnings + build.outcome = outcome + build.save() + + + def get_update_task_object(self, task_information): + task_object, created = Task.objects.get_or_create( + build=task_information['build'], + recipe=task_information['recipe'], + task_name=task_information['task_name'], + ) + + for v in vars(task_object): + if v in task_information.keys(): + vars(task_object)[v] = task_information[v] + # if we got covered by a setscene task, we're SSTATE + if task_object.outcome == Task.OUTCOME_COVERED and 1 == Task.objects.filter(task_executed=True, build = task_object.build, recipe = task_object.recipe, task_name=task_object.task_name+"_setscene").count(): + task_object.outcome = Task.OUTCOME_SSTATE + + # mark down duration if we have a start time + if 'start_time' in task_information.keys(): + duration = datetime.datetime.now() - task_information['start_time'] + task_object.elapsed_time = duration.total_seconds() + + task_object.save() + return task_object + + + def get_update_recipe_object(self, recipe_information): + + recipe_object, created = Recipe.objects.get_or_create( + layer_version=recipe_information['layer_version'], + file_path=recipe_information['file_path']) + + for v in vars(recipe_object): + if v in recipe_information.keys(): + vars(recipe_object)[v] = recipe_information[v] + + recipe_object.save() + + return recipe_object + + def get_layer_version_object(self, layer_version_information): + + layer_version_object = Layer_Version.objects.get_or_create( + layer = layer_version_information['layer'], + branch = layer_version_information['branch'], + commit = layer_version_information['commit'], + priority = layer_version_information['priority'] + ) + + layer_version_object[0].save() + + return layer_version_object[0] + + def get_update_layer_object(self, layer_information): + + layer_object = Layer.objects.get_or_create( + name=layer_information['name'], + local_path=layer_information['local_path'], + layer_index_url=layer_information['layer_index_url']) + layer_object[0].save() + + return layer_object[0] + + + def save_target_package_information(self, target_obj, packagedict, bldpkgs, recipes): + for p in packagedict: + packagedict[p]['object'] = Target_Package.objects.create( target = target_obj, + name = p, + size = packagedict[p]['size']) + if p in bldpkgs: + packagedict[p]['object'].version = bldpkgs[p]['version'] + packagedict[p]['object'].recipe = recipes[bldpkgs[p]['pn']] + packagedict[p]['object'].save() + + for p in packagedict: + for (px,deptype) in packagedict[p]['depends']: + Target_Package_Dependency.objects.create( package = packagedict[p]['object'], + depends_on = packagedict[px]['object'], + dep_type = deptype); + + + def create_logmessage(self, log_information): + log_object = LogMessage.objects.create( + build = log_information['build'], + level = log_information['level'], + message = log_information['message']) + + for v in vars(log_object): + if v in log_information.keys(): + vars(log_object)[v] = log_information[v] + + return log_object.save() + + + def save_build_package_information(self, build_obj, package_info, recipes, files): + # create and save the object + bp_object = Build_Package.objects.create( build = build_obj, + recipe = recipes[package_info['PN']], + name = package_info['PKG'], + version = package_info['PKGV'], + revision = package_info['PKGR'], + summary = package_info['SUMMARY'], + description = package_info['DESCRIPTION'], + size = package_info['PKGSIZE'], + section = package_info['SECTION'], + license = package_info['LICENSE'], + ) + # save any attached file information + if bp_object.name in files.keys(): + for path, size in files[bp_object.name]: + fo = Build_File.objects.create( bpackage = bp_object, + path = path, + size = size ) + del files[bp_object.name] + + # save soft dependency information + if package_info['RDEPENDS']: + for p in bb.utils.explode_deps(package_info['RDEPENDS']): + Build_Package_Dependency.objects.get_or_create( package = bp_object, + depends_on = p, dep_type = Build_Package_Dependency.TYPE_RDEPENDS) + if package_info['RPROVIDES']: + for p in bb.utils.explode_deps(package_info['RPROVIDES']): + Build_Package_Dependency.objects.get_or_create( package = bp_object, + depends_on = p, dep_type = Build_Package_Dependency.TYPE_RPROVIDES) + if package_info['RRECOMMENDS']: + for p in bb.utils.explode_deps(package_info['RRECOMMENDS']): + Build_Package_Dependency.objects.get_or_create( package = bp_object, + depends_on = p, dep_type = Build_Package_Dependency.TYPE_RRECOMMENDS) + if package_info['RSUGGESTS']: + for p in bb.utils.explode_deps(package_info['RSUGGESTS']): + Build_Package_Dependency.objects.get_or_create( package = bp_object, + depends_on = p, dep_type = Build_Package_Dependency.TYPE_RSUGGESTS) + if package_info['RREPLACES']: + for p in bb.utils.explode_deps(package_info['RREPLACES']): + Build_Package_Dependency.objects.get_or_create( package = bp_object, + depends_on = p, dep_type = Build_Package_Dependency.TYPE_RREPLACES) + if package_info['RCONFLICTS']: + for p in bb.utils.explode_deps(package_info['RCONFLICTS']): + Build_Package_Dependency.objects.get_or_create( package = bp_object, + depends_on = p, dep_type = Build_Package_Dependency.TYPE_RCONFLICTS) + + return bp_object + + def save_build_variables(self, build_obj, vardump): + for k in vardump: + if not bool(vardump[k]['func']): + Variable.objects.create( build = build_obj, + variable_name = k, + variable_value = vardump[k]['v'], + description = vardump[k]['doc']) + + +class BuildInfoHelper(object): + """ This class gathers the build information from the server and sends it + towards the ORM wrapper for storing in the database + It is instantiated once per build + Keeps in memory all data that needs matching before writing it to the database + """ + + def __init__(self, server, has_build_history = False): + self._configure_django() + self.internal_state = {} + self.task_order = 0 + self.server = server + self.orm_wrapper = ORMWrapper() + self.has_build_history = has_build_history + self.tmp_dir = self.server.runCommand(["getVariable", "TMPDIR"])[0] + + def _configure_django(self): + # Add toaster to sys path for importing modules + sys.path.append(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), 'toaster')) + + ################### + ## methods to convert event/external info into objects that the ORM layer uses + + def _get_layer_dict(self, layer_path): + + layer_info = {} + layer_name = layer_path.split('/')[-1] + layer_url = 'http://layers.openembedded.org/layerindex/layer/{layer}/' + layer_url_name = self._get_url_map_name(layer_name) + + layer_info['name'] = layer_name + layer_info['local_path'] = layer_path + layer_info['layer_index_url'] = layer_url.format(layer=layer_url_name) + + return layer_info + + def _get_url_map_name(self, layer_name): + """ Some layers have a different name on openembedded.org site, + this method returns the correct name to use in the URL + """ + + url_name = layer_name + url_mapping = {'meta': 'openembedded-core'} + + for key in url_mapping.keys(): + if key == layer_name: + url_name = url_mapping[key] + + return url_name + + def _get_layer_information(self): + + layer_info = {} + + return layer_info + + def _get_layer_version_information(self, layer_object): + + layer_version_info = {} + layer_version_info['build'] = self.internal_state['build'] + layer_version_info['layer'] = layer_object + layer_version_info['branch'] = self._get_git_branch(layer_object.local_path) + layer_version_info['commit'] = self._get_git_revision(layer_object.local_path) + layer_version_info['priority'] = 0 + + return layer_version_info + + + def _get_git_branch(self, layer_path): + branch = subprocess.Popen("git symbolic-ref HEAD 2>/dev/null ", cwd=layer_path, shell=True, stdout=subprocess.PIPE).communicate()[0] + branch = branch.replace('refs/heads/', '').rstrip() + return branch + + def _get_git_revision(self, layer_path): + revision = subprocess.Popen("git rev-parse HEAD 2>/dev/null ", cwd=layer_path, shell=True, stdout=subprocess.PIPE).communicate()[0].rstrip() + return revision + + + def _get_build_information(self): + build_info = {} + # Generate an identifier for each new build + + build_info['machine'] = self.server.runCommand(["getVariable", "MACHINE"])[0] + build_info['distro'] = self.server.runCommand(["getVariable", "DISTRO"])[0] + build_info['distro_version'] = self.server.runCommand(["getVariable", "DISTRO_VERSION"])[0] + build_info['started_on'] = datetime.datetime.now() + build_info['completed_on'] = datetime.datetime.now() + build_info['image_fstypes'] = self._remove_redundant(self.server.runCommand(["getVariable", "IMAGE_FSTYPES"])[0] or "") + build_info['cooker_log_path'] = self.server.runCommand(["getVariable", "BB_CONSOLELOG"])[0] + build_info['build_name'] = self.server.runCommand(["getVariable", "BUILDNAME"])[0] + build_info['bitbake_version'] = self.server.runCommand(["getVariable", "BB_VERSION"])[0] + + return build_info + + def _get_task_information(self, event, recipe): + + + task_information = {} + task_information['build'] = self.internal_state['build'] + task_information['outcome'] = Task.OUTCOME_NA + task_information['recipe'] = recipe + task_information['task_name'] = event.taskname + try: + # some tasks don't come with a hash. and that's ok + task_information['sstate_checksum'] = event.taskhash + except AttributeError: + pass + return task_information + + def _get_layer_version_for_path(self, path): + def _slkey(layer_version): + return len(layer_version.layer.local_path) + + # Heuristics: we always match recipe to the deepest layer path that + # we can match to the recipe file path + for bl in sorted(self.internal_state['layer_versions'], reverse=True, key=_slkey): + if (path.startswith(bl.layer.local_path)): + return bl + + #TODO: if we get here, we didn't read layers correctly + assert False + return None + + def _get_recipe_information_from_build_event(self, event): + + layer_version_obj = self._get_layer_version_for_path(re.split(':', event.taskfile)[-1]) + + recipe_info = {} + recipe_info['layer_version'] = layer_version_obj + recipe_info['file_path'] = re.split(':', event.taskfile)[-1] + + return recipe_info + + def _get_task_build_stats(self, task_object): + bs_path = self._get_path_information(task_object) + for bp in bs_path: # TODO: split for each target + task_build_stats = self._get_build_stats_from_file(bp, task_object.task_name) + + return task_build_stats + + def _get_path_information(self, task_object): + build_stats_format = "{tmpdir}/buildstats/{target}-{machine}/{buildname}/{package}/" + build_stats_path = [] + + for t in self.internal_state['targets']: + target = t.target + machine = self.internal_state['build'].machine + buildname = self.internal_state['build'].build_name + package = task_object.recipe.name + "-" + task_object.recipe.version.strip(":") + + build_stats_path.append(build_stats_format.format(tmpdir=self.tmp_dir, target=target, + machine=machine, buildname=buildname, + package=package)) + + return build_stats_path + + def _get_build_stats_from_file(self, bs_path, task_name): + + task_bs_filename = str(bs_path) + str(task_name) + task_bs = open(task_bs_filename, 'r') + + cpu_usage = 0 + disk_io = 0 + startio = '' + endio = '' + + for line in task_bs.readlines(): + if line.startswith('CPU usage: '): + cpu_usage = line[11:] + elif line.startswith('EndTimeIO: '): + endio = line[11:] + elif line.startswith('StartTimeIO: '): + startio = line[13:] + + task_bs.close() + + if startio and endio: + disk_io = int(endio.strip('\n ')) - int(startio.strip('\n ')) + + if cpu_usage: + cpu_usage = float(cpu_usage.strip('% \n')) + + task_build_stats = {'cpu_usage': cpu_usage, 'disk_io': disk_io} + + return task_build_stats + + def _remove_redundant(self, string): + ret = [] + for i in string.split(): + if i not in ret: + ret.append(i) + return " ".join(ret) + + + ################################ + ## external available methods to store information + + def store_layer_info(self): + layers = self.server.runCommand(["getVariable", "BBLAYERS"])[0].strip().split(" ") + self.internal_state['layers'] = [] + for layer_path in { l for l in layers if len(l) }: + layer_information = self._get_layer_dict(layer_path) + self.internal_state['layers'].append(self.orm_wrapper.get_update_layer_object(layer_information)) + + def store_started_build(self, event): + + build_information = self._get_build_information() + + build_obj = self.orm_wrapper.create_build_object(build_information) + self.internal_state['build'] = build_obj + + # create target information + target_information = {} + target_information['targets'] = event.getPkgs() + target_information['build'] = build_obj + + self.internal_state['targets'] = self.orm_wrapper.create_target_objects(target_information) + + # Load layer information for the build + self.internal_state['layer_versions'] = [] + for layer_object in self.internal_state['layers']: + layer_version_information = self._get_layer_version_information(layer_object) + self.internal_state['layer_versions'].append(self.orm_wrapper.get_layer_version_object(layer_version_information)) + + del self.internal_state['layers'] + # Save build configuration + self.orm_wrapper.save_build_variables(build_obj, self.server.runCommand(["getAllKeysWithFlags", ["doc", "func"]])[0]) + + + def update_build_information(self, event, errors, warnings, taskfailures): + if 'build' in self.internal_state: + self.orm_wrapper.update_build_object(self.internal_state['build'], errors, warnings, taskfailures) + + def store_started_task(self, event): + identifier = event.taskfile + event.taskname + + recipe_information = self._get_recipe_information_from_build_event(event) + recipe = self.orm_wrapper.get_update_recipe_object(recipe_information) + + task_information = self._get_task_information(event, recipe) + task_information['outcome'] = Task.OUTCOME_NA + + if isinstance(event, bb.runqueue.runQueueTaskSkipped): + task_information['task_executed'] = False + if event.reason == "covered": + task_information['outcome'] = Task.OUTCOME_COVERED + if event.reason == "existing": + task_information['outcome'] = Task.OUTCOME_EXISTING + else: + task_information['task_executed'] = True + + self.task_order += 1 + task_information['order'] = self.task_order + task_obj = self.orm_wrapper.get_update_task_object(task_information) + + self.internal_state[identifier] = {'start_time': datetime.datetime.now()} + + def update_and_store_task(self, event): + identifier = event.taskfile + event.taskname + recipe_information = self._get_recipe_information_from_build_event(event) + recipe = self.orm_wrapper.get_update_recipe_object(recipe_information) + task_information = self._get_task_information(event,recipe) + try: + task_information['start_time'] = self.internal_state[identifier]['start_time'] + except: + pass + + if 'logfile' in vars(event): + task_information['logfile'] = event.logfile + + if '_message' in vars(event): + task_information['message'] = event._message + + if 'ispython' in vars(event): + if event.ispython: + task_information['script_type'] = Task.CODING_PYTHON + else: + task_information['script_type'] = Task.CODING_SHELL + + if isinstance(event, (bb.runqueue.runQueueTaskCompleted, bb.runqueue.sceneQueueTaskCompleted)): + task_information['outcome'] = Task.OUTCOME_SUCCESS + task_build_stats = self._get_task_build_stats(self.orm_wrapper.get_update_task_object(task_information)) + task_information['cpu_usage'] = task_build_stats['cpu_usage'] + task_information['disk_io'] = task_build_stats['disk_io'] + del self.internal_state[identifier] + + if isinstance(event, bb.runqueue.runQueueTaskFailed): + task_information['outcome'] = Task.OUTCOME_FAILED + del self.internal_state[identifier] + + self.orm_wrapper.get_update_task_object(task_information) + + + def read_target_package_dep_data(self, event): + # for all targets + for target in self.internal_state['targets']: + # verify that we have something to read + if not target.is_image or not self.has_build_history: + print "not collecting package info ", target.is_image, self.has_build_history + break + + # TODO this is a temporary replication of the code in buildhistory.bbclass + # This MUST be changed to query the actual BUILD_DIR_IMAGE in the target context when + # the capability will be implemented in Bitbake + + MACHINE_ARCH, error = self.server.runCommand(['getVariable', 'MACHINE_ARCH']) + TCLIBC, error = self.server.runCommand(['getVariable', 'TCLIBC']) + BUILDHISTORY_DIR = self.server.runCommand(['getVariable', 'BUILDHISTORY_DIR']) + BUILDHISTORY_DIR_IMAGE = "%s/images/%s/%s/%s" % (BUILDHISTORY_DIR, MACHINE_ARCH, TCLIBC, target.target) + + self.internal_state['packages'] = {} + + with open("%s/installed-package-sizes.txt" % BUILDHISTORY_DIR_IMAGE, "r") as fin: + for line in fin: + line = line.rstrip(";") + psize, px = line.split("\t") + punit, pname = px.split(" ") + self.internal_state['packages'][pname.strip()] = {'size':int(psize)*1024, 'depends' : []} + + with open("%s/depends.dot" % BUILDHISTORY_DIR_IMAGE, "r") as fin: + p = re.compile(r' -> ') + dot = re.compile(r'.*style=dotted') + for line in fin: + line = line.rstrip(';') + linesplit = p.split(line) + if len(linesplit) == 2: + pname = linesplit[0].rstrip('"').strip('"') + dependsname = linesplit[1].split(" ")[0].strip().strip(";").strip('"').rstrip('"') + deptype = Target_Package_Dependency.TYPE_DEPENDS + if dot.match(line): + deptype = Target_Package_Dependency.TYPE_RECOMMENDS + if not pname in self.internal_state['packages']: + self.internal_state['packages'][pname] = {'size': 0, 'depends' : []} + if not dependsname in self.internal_state['packages']: + self.internal_state['packages'][dependsname] = {'size': 0, 'depends' : []} + self.internal_state['packages'][pname]['depends'].append((dependsname, deptype)) + + self.orm_wrapper.save_target_package_information(target, + self.internal_state['packages'], + self.internal_state['bldpkgs'], self.internal_state['recipes']) + + + def store_dependency_information(self, event): + # save layer version priorities + if 'layer-priorities' in event._depgraph.keys(): + for lv in event._depgraph['layer-priorities']: + (name, path, regexp, priority) = lv + layer_version_obj = self._get_layer_version_for_path(path[1:]) # paths start with a ^ + assert layer_version_obj is not None + layer_version_obj.priority = priority + layer_version_obj.save() + + # save build time package information + self.internal_state['bldpkgs'] = {} + for pkg in event._depgraph['packages']: + self.internal_state['bldpkgs'][pkg] = event._depgraph['packages'][pkg] + + # save recipe information + self.internal_state['recipes'] = {} + for pn in event._depgraph['pn']: + + file_name = re.split(':', event._depgraph['pn'][pn]['filename'])[-1] + layer_version_obj = self._get_layer_version_for_path(re.split(':', file_name)[-1]) + + assert layer_version_obj is not None + + recipe_info = {} + recipe_info['name'] = pn + recipe_info['version'] = event._depgraph['pn'][pn]['version'] + recipe_info['layer_version'] = layer_version_obj + recipe_info['summary'] = event._depgraph['pn'][pn]['summary'] + recipe_info['license'] = event._depgraph['pn'][pn]['license'] + recipe_info['description'] = event._depgraph['pn'][pn]['description'] + recipe_info['section'] = event._depgraph['pn'][pn]['section'] + recipe_info['licensing_info'] = 'Not Available' + recipe_info['homepage'] = event._depgraph['pn'][pn]['homepage'] + recipe_info['bugtracker'] = event._depgraph['pn'][pn]['bugtracker'] + recipe_info['author'] = 'Not Available' + recipe_info['file_path'] = file_name + recipe = self.orm_wrapper.get_update_recipe_object(recipe_info) + if 'inherits' in event._depgraph['pn'][pn].keys(): + recipe.is_image = True in map(lambda x: x.endswith('image.bbclass'), event._depgraph['pn'][pn]['inherits']) + else: + recipe.is_image = False + if recipe.is_image: + for t in self.internal_state['targets']: + if pn == t.target: + t.is_image = True + t.save() + self.internal_state['recipes'][pn] = recipe + + # save recipe dependency + # buildtime + for recipe in event._depgraph['depends']: + try: + target = self.internal_state['recipes'][recipe] + for dep in event._depgraph['depends'][recipe]: + dependency = self.internal_state['recipes'][dep] + Recipe_Dependency.objects.get_or_create( recipe = target, + depends_on = dependency, dep_type = Recipe_Dependency.TYPE_DEPENDS) + except KeyError: # we'll not get recipes for key w/ values listed in ASSUME_PROVIDED + pass + + # runtime + for recipe in event._depgraph['rdepends-pn']: + try: + target = self.internal_state['recipes'][recipe] + for dep in event._depgraph['rdepends-pn'][recipe]: + dependency = self.internal_state['recipes'][dep] + Recipe_Dependency.objects.get_or_create( recipe = target, + depends_on = dependency, dep_type = Recipe_Dependency.TYPE_RDEPENDS) + + except KeyError: # we'll not get recipes for key w/ values listed in ASSUME_PROVIDED + pass + + # save all task information + def _save_a_task(taskdesc): + spec = re.split(r'\.', taskdesc); + pn = ".".join(spec[0:-1]) + taskname = spec[-1] + e = event + e.taskname = pn + recipe = self.internal_state['recipes'][pn] + task_info = self._get_task_information(e, recipe) + task_info['task_name'] = taskname + task_obj = self.orm_wrapper.get_update_task_object(task_info) + return task_obj + + for taskdesc in event._depgraph['tdepends']: + target = _save_a_task(taskdesc) + for taskdesc1 in event._depgraph['tdepends'][taskdesc]: + dep = _save_a_task(taskdesc1) + Task_Dependency.objects.get_or_create( task = target, depends_on = dep ) + + def store_build_package_information(self, event): + package_info = event.data + self.orm_wrapper.save_build_package_information(self.internal_state['build'], + package_info, + self.internal_state['recipes'], + self.internal_state['package_files']) + + + def store_package_file_information(self, event): + if not 'package_files' in self.internal_state.keys(): + self.internal_state['package_files'] = {} + + data = event.data + self.internal_state['package_files'][data['PKG']] = data['FILES'] + + def _store_log_information(self, level, text): + log_information = {} + log_information['build'] = self.internal_state['build'] + log_information['level'] = level + log_information['message'] = text + self.orm_wrapper.create_logmessage(log_information) + + def store_log_info(self, text): + self._store_log_information(LogMessage.INFO, text) + + def store_log_warn(self, text): + self._store_log_information(LogMessage.WARNING, text) + + def store_log_error(self, text): + self._store_log_information(LogMessage.ERROR, text) + + def store_log_event(self, event): + # look up license files info from insane.bbclass + m = re.match("([^:]*): md5 checksum matched for ([^;]*)", event.msg) + if m: + (pn, fn) = m.groups() + self.internal_state['recipes'][pn].licensing_info = fn + self.internal_state['recipes'][pn].save() + + if event.levelno < format.WARNING: + return + if not 'build' in self.internal_state: + return + log_information = {} + log_information['build'] = self.internal_state['build'] + if event.levelno >= format.ERROR: + log_information['level'] = LogMessage.ERROR + elif event.levelno == format.WARNING: + log_information['level'] = LogMessage.WARNING + log_information['message'] = event.msg + log_information['pathname'] = event.pathname + log_information['lineno'] = event.lineno + self.orm_wrapper.create_logmessage(log_information) +
bitbake/lib/bb/ui/toasterui.py new file +273 -0
273 lines not on this page · raw message
Signature · 1 line
1.8.1.2