Toaster-related improvements
cover Toaster-related improvements
The following changes since commit f3ac2d3678f48c68a250a0a20c08cf8687322d38:
monitordisk: lower inode check warning to note (2013-10-18 16:02:10 +0100)
are available in the git repository at:
git://git.yoctoproject.org/poky-contrib paule/toaster2-bb
http://git.yoctoproject.org/cgit.cgi/poky-contrib/log/?h=paule/toaster2-bb
Alexandru DAMIAN (7):
toaster: fixes for null values from events
cooker: add data to the dependency tree dump
cooker: do not recreate recipecache in buildfile mode build, toaster: record proper task type
toaster: fix timezone settings
toaster: server shutdown on terminal exit
toaster: enable required classes in the toaster startup script
Cristiana Voicu (1):
toaster: add variable description for prefixed/suffixed variables
bin/toaster | 41 +++++++++++++++++++++-------- lib/bb/build.py | 7 ++++- lib/bb/cooker.py | 3 ++- lib/bb/ui/buildinfohelper.py | 52 ++++++++++++++++++++----------------- lib/bb/ui/toasterui.py | 3 +-- lib/toaster/orm/models.py | 8 +++--- lib/toaster/toastermain/settings.py | 5 ++-- 7 files changed, 75 insertions(+), 44 deletions(-)
Signature · 1 line
1.8.1.2
PATCH 1/8 toaster: fixes for null values from events
From: Alexandru DAMIAN <[email protected]>
Some of the data values may come of as None through the event system, and the UI would encounter a problem saving the Configuration. It would be trying to save these values as NULL in the database, which is not allowed.
This patch adds more verification for data coming through the event system.
Other minor updates:
* update for the event model from toaster.bbclass
* minor code flow fix in the event system
--- lib/bb/ui/buildinfohelper.py | 40 ++++++++++++++++++---------------------- lib/bb/ui/toasterui.py | 3 +-- 2 files changed, 19 insertions(+), 24 deletions(-)
lib/bb/ui/buildinfohelper.py +18 -22
@@ -171,7 +171,7 @@ class ORMWrapper(object): return log_object.save() - def save_build_package_information(self, build_obj, package_info, recipes, files): + def save_build_package_information(self, build_obj, package_info, recipes): # create and save the object bp_object = Build_Package.objects.create( build = build_obj, recipe = recipes[package_info['PN']],
@@ -185,35 +185,33 @@ class ORMWrapper(object): license = package_info['LICENSE'], ) # save any attached file information - if bp_object.name in files.keys(): - for path, size in files[bp_object.name]: + for path in package_info['FILES_INFO']: fo = Build_File.objects.create( bpackage = bp_object, path = path, - size = size ) - del files[bp_object.name] + size = package_info['FILES_INFO'][path] ) # save soft dependency information - if package_info['RDEPENDS']: + if 'RDEPENDS' in package_info and package_info['RDEPENDS']: for p in bb.utils.explode_deps(package_info['RDEPENDS']): Build_Package_Dependency.objects.get_or_create( package = bp_object, depends_on = p, dep_type = Build_Package_Dependency.TYPE_RDEPENDS) - if package_info['RPROVIDES']: + if 'RPROVIDES' in package_info and package_info['RPROVIDES']: for p in bb.utils.explode_deps(package_info['RPROVIDES']): Build_Package_Dependency.objects.get_or_create( package = bp_object, depends_on = p, dep_type = Build_Package_Dependency.TYPE_RPROVIDES) - if package_info['RRECOMMENDS']: + if 'RRECOMMENDS' in package_info and package_info['RRECOMMENDS']: for p in bb.utils.explode_deps(package_info['RRECOMMENDS']): Build_Package_Dependency.objects.get_or_create( package = bp_object, depends_on = p, dep_type = Build_Package_Dependency.TYPE_RRECOMMENDS) - if package_info['RSUGGESTS']: + if 'RSUGGESTS' in package_info and package_info['RSUGGESTS']: for p in bb.utils.explode_deps(package_info['RSUGGESTS']): Build_Package_Dependency.objects.get_or_create( package = bp_object, depends_on = p, dep_type = Build_Package_Dependency.TYPE_RSUGGESTS) - if package_info['RREPLACES']: + if 'RREPLACES' in package_info and package_info['RREPLACES']: for p in bb.utils.explode_deps(package_info['RREPLACES']): Build_Package_Dependency.objects.get_or_create( package = bp_object, depends_on = p, dep_type = Build_Package_Dependency.TYPE_RREPLACES) - if package_info['RCONFLICTS']: + if 'RCONFLICTS' in package_info and package_info['RCONFLICTS']: for p in bb.utils.explode_deps(package_info['RCONFLICTS']): Build_Package_Dependency.objects.get_or_create( package = bp_object, depends_on = p, dep_type = Build_Package_Dependency.TYPE_RCONFLICTS)
@@ -223,10 +221,16 @@ class ORMWrapper(object): def save_build_variables(self, build_obj, vardump): for k in vardump: if not bool(vardump[k]['func']): + value = vardump[k]['v']; + if value is None: + value = '' + desc = vardump[k]['doc']; + if desc is None: + desc = '' Variable.objects.create( build = build_obj, variable_name = k, - variable_value = vardump[k]['v'], - description = vardump[k]['doc']) + variable_value = value, + description = desc) class BuildInfoHelper(object):
@@ -668,15 +672,7 @@ class BuildInfoHelper(object): self.orm_wrapper.save_build_package_information(self.internal_state['build'], package_info, self.internal_state['recipes'], - self.internal_state['package_files']) - - - def store_package_file_information(self, event): - if not 'package_files' in self.internal_state.keys(): - self.internal_state['package_files'] = {} - - data = event.data - self.internal_state['package_files'][data['PKG']] = data['FILES'] + ) def _store_log_information(self, level, text): log_information = {}
lib/bb/ui/toasterui.py +1 -2
@@ -140,6 +140,7 @@ def main(server, eventHandler, params ): logfile = event.logfile if logfile and os.path.exists(logfile): bb.error("Logfile of failure stored in: %s" % logfile) + continue # these events are unprocessed now, but may be used in the future to log # timing and error informations from the parsing phase in Toaster
@@ -230,8 +231,6 @@ def main(server, eventHandler, params ): if isinstance(event, bb.event.MetadataEvent): if event.type == "SinglePackageInfo": buildinfohelper.store_build_package_information(event) - elif event.type == "PackageFileSize": - buildinfohelper.store_package_file_information(event) continue # ignore
Signature · 1 line
1.8.1.2
PATCH 2/8 cooker: add data to the dependency tree dump
In reply to Paul EggletonFrom: Alexandru DAMIAN <[email protected]>
Toaster needes to record extra data that needs to
be moved at the time of the dependency tree dump.
This data includes:
* layer priorities for recording in the layer section
* the inherit list for each PN which allows to determine the type of the PN (regular package, image, etc).
This patch adds this data to the dependency tree dump.
--- lib/bb/cooker.py | 2 ++ 1 file changed, 2 insertions(+)
lib/bb/cooker.py +2 -0
@@ -511,6 +511,7 @@ class BBCooker: depend_tree["packages"] = {} depend_tree["rdepends-pkg"] = {} depend_tree["rrecs-pkg"] = {} + depend_tree["layer-priorities"] = self.recipecache.bbfile_config_priorities for task in xrange(len(rq.rqdata.runq_fnid)): taskname = rq.rqdata.runq_task[task]
@@ -522,6 +523,7 @@ class BBCooker: depend_tree["pn"][pn] = {} depend_tree["pn"][pn]["filename"] = fn depend_tree["pn"][pn]["version"] = version + depend_tree["pn"][pn]["inherits"] = self.recipecache.inherits.get(fn, None) # if we have extra caches, list all attributes they bring in extra_info = []
Signature · 1 line
1.8.1.2
PATCH 3/8 cooker: do not recreate recipecache in buildfile mode
In reply to Paul EggletonFrom: Alexandru DAMIAN <[email protected]>
When building a single file, the cooker will recreate the recipecache from scratch.
I suspect this is a remnant of past code, since:
* the current recipecache works fine
* the new recipecache will not have all the fields as requested by HOB_EXTRA_CACHES setting
This patch disables recreating the recipecache, leading to shorter times when building single build files (-b option) and better compatibility with Toaster.
--- lib/bb/cooker.py | 1 - 1 file changed, 1 deletion(-)
lib/bb/cooker.py +0 -1
@@ -1085,7 +1085,6 @@ class BBCooker: self.buildSetVars() - self.recipecache = bb.cache.CacheData(self.caches_array) infos = bb.cache.Cache.parse(fn, self.collection.get_file_appends(fn), \ self.data, self.caches_array)
Signature · 1 line
1.8.1.2
PATCH 4/8 build, toaster: record proper task type
In reply to Paul EggletonFrom: Alexandru DAMIAN <[email protected]>
Bitbake tasks may be of type 'python' or 'shell', or they may not be executed at all, which is record as task type 'noexec'.
In order to record proper task type, this patch:
* creates no exec task type as the default value in the toaster model definition
* adds full task flags to the bb.build.TaskStarted event in build.py
* if the task actually starts, the toaster ui will record the type of the task as either 'python' or 'shell' based on the task flags.
[YOCTO #5073]
[YOCTO #5075]
[YOCTO #5327]
--- lib/bb/build.py | 7 ++++++- lib/bb/ui/buildinfohelper.py | 7 +++++-- lib/toaster/orm/models.py | 8 +++++--- 3 files changed, 16 insertions(+), 6 deletions(-)
lib/bb/build.py +6 -1
@@ -91,6 +91,9 @@ class TaskBase(event.Event): class TaskStarted(TaskBase): """Task execution started""" + def __init__(self, t, logfile, taskflags, d): + super(TaskStarted, self).__init__(t, logfile, d) + self.taskflags = taskflags class TaskSucceeded(TaskBase): """Task execution completed"""
@@ -422,7 +425,9 @@ def _exec_task(fn, task, d, quieterr): localdata.setVar('BB_LOGFILE', logfn) localdata.setVar('BB_RUNTASK', task) - event.fire(TaskStarted(task, logfn, localdata), localdata) + flags = localdata.getVarFlags(task) + + event.fire(TaskStarted(task, logfn, flags, localdata), localdata) try: for func in (prefuncs or '').split(): exec_func(func, localdata)
lib/bb/ui/buildinfohelper.py +5 -2
@@ -483,6 +483,8 @@ class BuildInfoHelper(object): task_information['outcome'] = Task.OUTCOME_EXISTING else: task_information['task_executed'] = True + if 'noexec' in vars(event) and event.noexec == True: + task_information['script_type'] = Task.CODING_NOEXEC self.task_order += 1 task_information['order'] = self.task_order
@@ -506,8 +508,9 @@ class BuildInfoHelper(object): if '_message' in vars(event): task_information['message'] = event._message - if 'ispython' in vars(event): - if event.ispython: + if 'taskflags' in vars(event): + # with TaskStarted, we get even more information + if 'python' in event.taskflags.keys() and event.taskflags['python'] == '1': task_information['script_type'] = Task.CODING_PYTHON else: task_information['script_type'] = Task.CODING_SHELL
lib/toaster/orm/models.py +5 -3
@@ -74,10 +74,12 @@ class Task(models.Model): (SSTATE_RESTORED, 'Restored'), # succesfully restored ) - CODING_PYTHON = 0 - CODING_SHELL = 1 + CODING_NOEXEC = 0 + CODING_PYTHON = 1 + CODING_SHELL = 2 TASK_CODING = ( + (CODING_NOEXEC, 'NoExec'), (CODING_PYTHON, 'Python'), (CODING_SHELL, 'Shell'), )
@@ -108,7 +110,7 @@ class Task(models.Model): task_name = models.CharField(max_length=100) source_url = models.FilePathField(max_length=255, blank=True) work_directory = models.FilePathField(max_length=255, blank=True) - script_type = models.IntegerField(choices=TASK_CODING, default=CODING_PYTHON) + script_type = models.IntegerField(choices=TASK_CODING, default=CODING_NOEXEC) line_number = models.IntegerField(default=0) disk_io = models.IntegerField(null=True) cpu_usage = models.DecimalField(max_digits=6, decimal_places=2, null=True)
Signature · 1 line
1.8.1.2
PATCH 5/8 toaster: fix timezone settings
In reply to Paul EggletonFrom: Alexandru DAMIAN <[email protected]>
This patch fixes an issue where, if not defined,
the timezone defaults to 'America/Chicago'.
The solution is to set the timezone to current computer's timezone.
[YOCTO #5186]
--- lib/toaster/toastermain/settings.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-)
lib/toaster/toastermain/settings.py +3 -2
@@ -48,7 +48,8 @@ ALLOWED_HOSTS = [] # In a Windows environment this must be set to your system time zone. # Always use local computer's time zone -#TIME_ZONE = '' +import time +TIME_ZONE = time.tzname[0] # Language code for this installation. All choices can be found here: # http://www.i18nguy.com/unicode/language-identifiers.html
@@ -65,7 +66,7 @@ USE_I18N = True USE_L10N = True # If you set this to False, Django will not use timezone-aware datetimes. -USE_TZ = False +USE_TZ = True # Absolute filesystem path to the directory that will hold user-uploaded files. # Example: "/var/www/example.com/media/"
Signature · 1 line
1.8.1.2
PATCH 6/8 toaster: server shutdown on terminal exit
In reply to Paul EggletonFrom: Alexandru DAMIAN <[email protected]>
If the terminal where the server was started is closed, the bitbake server should shutdown. Currently the system is left in hanging state.
This patch uses "trap" command to make sure the servers are closed on terminal exit.
[YOCTO #5376]
--- bin/toaster | 29 ++++++++++++++++++++--------- 1 file changed, 20 insertions(+), 9 deletions(-)
bin/toaster +20 -9
@@ -49,6 +49,19 @@ function webserverStartAll() return $retval } +# define the stop command +function stop_system() +{ + if [ -f ${BUILDDIR}/.toasterui.pid ]; then + kill $(< ${BUILDDIR}/.toasterui.pid ) + rm ${BUILDDIR}/.toasterui.pid + fi + BBSERVER=localhost:8200 bitbake -m + unset BBSERVER + webserverKillAll + # force stop any misbehaving bitbake server + lsof bitbake.lock | awk '{print $2}' | grep "[0-9]\+" | xargs -n1 -r kill +} # We make sure we're running in the current shell and in a good environment
@@ -116,6 +129,9 @@ and fi + + + # Execute the commands case $CMD in
@@ -127,17 +143,12 @@ case $CMD in if [ $NOTOASTERUI == 0 ]; then # we start the TOASTERUI only if not inhibited bitbake --observe-only -u toasterui >${BUILDDIR}/toaster_ui.log 2>&1 & echo $! >${BUILDDIR}/.toasterui.pid fi + # stop system on terminal exit + trap stop_system SIGHUP ;; stop ) - if [ -f ${BUILDDIR}/.toasterui.pid ]; then - kill $(< ${BUILDDIR}/.toasterui.pid ) - rm ${BUILDDIR}/.toasterui.pid - fi - bitbake -m - unset BBSERVER - webserverKillAll - # force stop any misbehaving bitbake server - lsof bitbake.lock | awk '{print $2}' | grep "[0-9]\+" | xargs -n1 -r kill + stop_system + trap '' SIGHUP ;; esac
Signature · 1 line
1.8.1.2
PATCH 7/8 toaster: add variable description for prefixed/suffixed variables
In reply to Paul EggletonFrom: Cristiana Voicu <[email protected]>
In the Configuration table, we need to link prefixed / suffixed variables to the corresponding variable descriptions in documentation.conf.
[YOCTO #5198]
--- lib/bb/ui/buildinfohelper.py | 5 +++++ 1 file changed, 5 insertions(+)
lib/bb/ui/buildinfohelper.py +5 -0
@@ -226,6 +226,11 @@ class ORMWrapper(object): value = '' desc = vardump[k]['doc']; if desc is None: + var_words = [word for word in k.split('_')] + root_var = "_".join([word for word in var_words if word.isupper()]) + if root_var and root_var != k and root_var in vardump: + desc = vardump[root_var]['doc'] + if desc is None: desc = '' Variable.objects.create( build = build_obj, variable_name = k,
Signature · 1 line
1.8.1.2
PATCH 8/8 toaster: enable required classes in the toaster startup script
In reply to Paul EggletonFrom: Alexandru DAMIAN <[email protected]>
In order to use toaster, now you have to set INHERIT+="toaster buildhistory" To keep it simple, I've done some changes in order to automate it. When toaster is started, this line is added to a new file called toaster.conf.
This file is passed to the bitbake server with the --postread parameter.
Based on a patch by Cristiana Voicu <[email protected]>
--- bin/toaster | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-)
bin/toaster +10 -2
@@ -36,7 +36,6 @@ function webserverKillAll() done } - function webserverStartAll() { retval=0
@@ -49,6 +48,14 @@ function webserverStartAll() return $retval } +# Helper functions to add a special configuration file + +function addtoConfiguration() +{ + echo "#Created by toaster start script" > ${BUILDDIR}/conf/$2 + echo $1 >> ${BUILDDIR}/conf/$2 +} + # define the stop command function stop_system() {
@@ -136,9 +143,10 @@ fi case $CMD in start ) + addtoConfiguration "INHERIT+=\"toaster buildhistory\"" toaster.conf webserverStartAll || return 4 unset BBSERVER - bitbake --server-only -t xmlrpc -B localhost:8200 + bitbake --postread conf/toaster.conf --server-only -t xmlrpc -B localhost:8200 export BBSERVER=localhost:8200 if [ $NOTOASTERUI == 0 ]; then # we start the TOASTERUI only if not inhibited bitbake --observe-only -u toasterui >${BUILDDIR}/toaster_ui.log 2>&1 & echo $! >${BUILDDIR}/.toasterui.pid
Signature · 1 line
1.8.1.2