webhob related changes in bitbake
cover webhob related changes in bitbake
From: Alexandru DAMIAN <[email protected]>
I'm sending in a set of patches that bring in changes that prepare bitbake for supporting WebHob-related tasks.
The changes focus on improving and standardizing the information sent through the event system, and cleaning up code in bitbake.
The following changes since commit 0ca5d1fb38157564a2b9452ade32391d18a41b09:
yocto-bsp: remove apm as a default MACHINE_FEATURE (2013-09-09 16:28:47 +0100)
are available in the git repository at:
git://git.yoctoproject.org/poky-contrib adamian/bitbake-core-changes-0909
http://git.yoctoproject.org/cgit.cgi/poky-contrib/log/?h=adamian/bitbake-core-changes-0909
Alexandru DAMIAN (13):
bitbake: cooker: clean up code and avoid duplication
bitbake: runqueue: Add runQueueTaskSkipped event
bitbake: runqueue: added prebuild detailed information
bitbake: runqueue, build: event data change
bitbake: runqueue: add sceneQueueTaskCompleted event
bitbake: build: add link to logfile for Task events
bitbake: cooker, command: add a command to return global data
bitbake: cooker: use prepareTreeData when computing environment
bitbake: event: do not abort on UI handlers error
bitbake: add SinglePackageInfo event
bitbake: event: adding an event to pass build file sizes
bitbake: xmlrpc: delete unused code BitBakeUIEventServer
bitbake: cooker: add extra recipe information
bitbake/lib/bb/build.py | 4 +++ bitbake/lib/bb/command.py | 7 ++++ bitbake/lib/bb/cooker.py | 74 +++++++++++++++++++++++++---------------- bitbake/lib/bb/event.py | 48 ++++++++++++++++++-------- bitbake/lib/bb/runqueue.py | 33 ++++++++++++++++-- bitbake/lib/bb/server/xmlrpc.py | 45 ------------------------- bitbake/lib/bb/ui/knotty.py | 2 ++ 7 files changed, 123 insertions(+), 90 deletions(-)
Signature · 1 line
1.8.1.2
PATCH 1/13 bitbake: cooker: clean up code and avoid duplication
From: Alexandru DAMIAN <[email protected]>
Clean-up to avoid duplication and promote code reuse. We're using the prepareTreeData in all instances where this needs to be done.
When we're only testing target image contents (HOB), we do not abort with an error if the target is unbuildable.
--- bitbake/lib/bb/cooker.py | 30 +++++++++--------------------- 1 file changed, 9 insertions(+), 21 deletions(-)
bitbake/lib/bb/cooker.py +9 -21
@@ -401,9 +401,7 @@ class BBCooker: localdata = data.createCopy(self.data) bb.data.update_data(localdata) bb.data.expandKeys(localdata) - # We set abort to False here to prevent unbuildable targets raising - # an exception when we're just generating data - taskdata = bb.taskdata.TaskData(False, skiplist=self.skiplist) + taskdata = bb.taskdata.TaskData(self.configuration.abort, skiplist=self.skiplist) runlist = [] current = 0
@@ -495,7 +493,12 @@ class BBCooker: """ Create a dependency tree of pkgs_to_build, returning the data. """ + + # We set abort to False here to prevent unbuildable targets raising + # an exception when we're just generating data + self.configuration.abort = False _, taskdata = self.prepareTreeData(pkgs_to_build, task) + tasks_fnid = [] if len(taskdata.tasks_name) != 0: for task in xrange(len(taskdata.tasks_name)):
@@ -1076,13 +1079,6 @@ class BBCooker: Attempt to build the targets specified """ - # If we are told to do the NULL task then query the default task - if (task == None): - task = self.configuration.cmd - - universe = ('universe' in targets) - targets = self.checkPackages(targets) - def buildTargetsIdle(server, rq, abort): if abort or self.state == state.stop: rq.finish_runqueue(True)
@@ -1109,21 +1105,13 @@ class BBCooker: self.buildSetVars() buildname = self.data.getVar("BUILDNAME") - bb.event.fire(bb.event.BuildStarted(buildname, targets), self.data) - - localdata = data.createCopy(self.data) - bb.data.update_data(localdata) - bb.data.expandKeys(localdata) - taskdata = bb.taskdata.TaskData(self.configuration.abort, skiplist=self.skiplist) + bb.event.fire(bb.event.BuildStarted(buildname, targets), self.data) - runlist = [] - for k in targets: - taskdata.add_provider(localdata, self.recipecache, k) - runlist.append([k, "do_%s" % task]) - taskdata.add_unresolved(localdata, self.recipecache) + runlist, taskdata = self.prepareTreeData(targets, task) rq = bb.runqueue.RunQueue(self, self.data, self.recipecache, taskdata, runlist) + universe = ('universe' in targets) if universe: rq.rqdata.warn_multi_bb = True
Signature · 1 line
1.8.1.2
PATCH 2/13 bitbake: runqueue: Add runQueueTaskSkipped event
From: Alexandru DAMIAN <[email protected]>
Adding a runQueueTaskSkipped to notify all listeners about the tasks that are not run either because they are set-scened or they don't need an update (timestamps are ok)
--- bitbake/lib/bb/runqueue.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+)
bitbake/lib/bb/runqueue.py +16 -0
@@ -217,6 +217,12 @@ class RunQueueData: ret.extend([nam]) return ret + def get_task_name(self, task): + return self.runq_task[task] + + def get_task_package(self, task): + return self.taskData.fn_index[self.runq_fnid[task]] + def get_user_idstring(self, task, task_name_suffix = ""): fn = self.taskData.fn_index[self.runq_fnid[task]] taskname = self.runq_task[task] + task_name_suffix
@@ -1328,6 +1334,7 @@ class RunQueueExecuteTasks(RunQueueExecute): def task_skip(self, task): self.runq_running[task] = 1 self.runq_buildable[task] = 1 + bb.event.fire(runQueueTaskSkipped(task, self.stats, self.rq), self.cfgData) self.task_completeoutright(task) self.stats.taskCompleted() self.stats.taskSkipped()
@@ -1828,6 +1835,15 @@ class runQueueTaskCompleted(runQueueEvent): Event notifing a task completed """ +class runQueueTaskSkipped(runQueueEvent): + """ + Event notifing a task was skipped + """ + def __init__(self, task, stats, rq): + runQueueEvent.__init__(self, task, stats, rq) + self._package = rq.rqdata.get_task_package(task) + self._task = rq.rqdata.get_task_name(task) + class runQueuePipe(): """ Abstraction for a pipe between a worker thread and the server
Signature · 1 line
1.8.1.2
PATCH 3/13 bitbake: runqueue: added prebuild detailed information
In reply to Alex DAMIANFrom: Alexandru DAMIAN <[email protected]>
Adds why the prebuild tasks were skipped,
as to distinguish about covered/existing tasks.
Adds task name and task package to all runQueue
and sceneQueue events.
--- bitbake/lib/bb/runqueue.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-)
bitbake/lib/bb/runqueue.py +10 -5
@@ -1331,10 +1331,10 @@ class RunQueueExecuteTasks(RunQueueExecute): if self.rqdata.taskData.abort: self.rq.state = runQueueCleanUp - def task_skip(self, task): + def task_skip(self, task, skip_reason = ""): self.runq_running[task] = 1 self.runq_buildable[task] = 1 - bb.event.fire(runQueueTaskSkipped(task, self.stats, self.rq), self.cfgData) + bb.event.fire(runQueueTaskSkipped(task, self.stats, self.rq, skip_reason), self.cfgData) self.task_completeoutright(task) self.stats.taskCompleted() self.stats.taskSkipped()
@@ -1359,13 +1359,13 @@ class RunQueueExecuteTasks(RunQueueExecute): if task in self.rq.scenequeue_covered: logger.debug(2, "Setscene covered task %s (%s)", task, self.rqdata.get_user_idstring(task)) - self.task_skip(task) + self.task_skip(task, "covered") return True if self.rq.check_stamp_task(task, taskname, cache=self.stampcache): logger.debug(2, "Stamp current task %s (%s)", task, self.rqdata.get_user_idstring(task)) - self.task_skip(task) + self.task_skip(task, "existing") return True taskdep = self.rqdata.dataCache.task_deps[fn]
@@ -1786,6 +1786,8 @@ class runQueueEvent(bb.event.Event): def __init__(self, task, stats, rq): self.taskid = task self.taskstring = rq.rqdata.get_user_idstring(task) + self.taskname = rq.rqdata.get_task_name(task) + self.taskpackage = rq.rqdata.get_task_package(task) self.stats = stats.copy() bb.event.Event.__init__(self)
@@ -1797,6 +1799,8 @@ class sceneQueueEvent(runQueueEvent): runQueueEvent.__init__(self, task, stats, rq) realtask = rq.rqdata.runq_setscene[task] self.taskstring = rq.rqdata.get_user_idstring(realtask, "_setscene") + self.taskname = rq.rqdata.get_task_name(realtask) + "_setscene" + self.taskpackage = rq.rqdata.get_task_package(realtask) class runQueueTaskStarted(runQueueEvent): """
@@ -1839,10 +1843,11 @@ class runQueueTaskSkipped(runQueueEvent): """ Event notifing a task was skipped """ - def __init__(self, task, stats, rq): + def __init__(self, task, stats, rq, skip_reason): runQueueEvent.__init__(self, task, stats, rq) self._package = rq.rqdata.get_task_package(task) self._task = rq.rqdata.get_task_name(task) + self._skip = skip_reason class runQueuePipe(): """
Signature · 1 line
1.8.1.2
PATCH 4/13 bitbake: runqueue, build: event data change
In reply to Alex DAMIANFrom: Alexandru DAMIAN <[email protected]>
We modify event data to make sure we can match build events with runqueue events. This is important because the build events supply datastore-evaluated data while runqueue events supply bitbake-related data. We get the recipe name and version from the datastore, while the file paths and timing information are collected from the RunQueue.
We modify filling-in the information side to take advantage of the new info supplied.
--- bitbake/lib/bb/build.py | 3 +++ bitbake/lib/bb/runqueue.py | 8 ++++---- 2 files changed, 7 insertions(+), 4 deletions(-)
bitbake/lib/bb/build.py +3 -0
@@ -72,6 +72,9 @@ class TaskBase(event.Event): def __init__(self, t, d ): self._task = t self._package = d.getVar("PF", True) + self._file = d.getVar("FILE", True) + self.taskfile = self._file + self.taskname = self._task event.Event.__init__(self) self._message = "recipe %s: task %s: %s" % (d.getVar("PF", True), t, self.getDisplayName())
bitbake/lib/bb/runqueue.py +4 -4
@@ -220,7 +220,7 @@ class RunQueueData: def get_task_name(self, task): return self.runq_task[task] - def get_task_package(self, task): + def get_task_file(self, task): return self.taskData.fn_index[self.runq_fnid[task]] def get_user_idstring(self, task, task_name_suffix = ""):
@@ -1787,7 +1787,7 @@ class runQueueEvent(bb.event.Event): self.taskid = task self.taskstring = rq.rqdata.get_user_idstring(task) self.taskname = rq.rqdata.get_task_name(task) - self.taskpackage = rq.rqdata.get_task_package(task) + self.taskfile = rq.rqdata.get_task_file(task) self.stats = stats.copy() bb.event.Event.__init__(self)
@@ -1800,7 +1800,7 @@ class sceneQueueEvent(runQueueEvent): realtask = rq.rqdata.runq_setscene[task] self.taskstring = rq.rqdata.get_user_idstring(realtask, "_setscene") self.taskname = rq.rqdata.get_task_name(realtask) + "_setscene" - self.taskpackage = rq.rqdata.get_task_package(realtask) + self.taskfile = rq.rqdata.get_task_file(realtask) class runQueueTaskStarted(runQueueEvent): """
@@ -1845,7 +1845,7 @@ class runQueueTaskSkipped(runQueueEvent): """ def __init__(self, task, stats, rq, skip_reason): runQueueEvent.__init__(self, task, stats, rq) - self._package = rq.rqdata.get_task_package(task) + self._file = rq.rqdata.get_task_file(task) self._task = rq.rqdata.get_task_name(task) self._skip = skip_reason
Signature · 1 line
1.8.1.2
Re: [PATCH 04/13] bitbake: runqueue, build: event data change
Quoted 64 lines · Alex DAMIAN
From: Alexandru DAMIAN <[email protected]>
We modify event data to make sure we can match build events with runqueue events. This is important because the build events supply datastore-evaluated data while runqueue events supply bitbake-related data. We get the recipe name and version from the datastore, while the file paths and timing information are collected from the RunQueue.
We modify filling-in the information side to take advantage of the new info supplied.
Signed-off-by Alexandru DAMIAN <[email protected]>--- bitbake/lib/bb/build.py | 3 +++ bitbake/lib/bb/runqueue.py | 8 ++++---- 2 files changed, 7 insertions(+), 4 deletions(-)bitbake/lib/bb/build.py +3
-0@@ -72,6 +72,9 @@ class TaskBase(event.Event): def __init__(self, t, d ): self._task = t self._package = d.getVar("PF", True) + self._file = d.getVar("FILE", True) + self.taskfile = self._file + self.taskname = self._task event.Event.__init__(self) self._message = "recipe %s: task %s: %s" % (d.getVar("PF", True), t, self.getDisplayName())bitbake/lib/bb/runqueue.py +3
-3@@ -220,7 +220,7 @@ class RunQueueData: def get_task_name(self, task): return self.runq_task[task] - def get_task_package(self, task): + def get_task_file(self, task): return self.taskData.fn_index[self.runq_fnid[task]] def get_user_idstring(self, task, task_name_suffix = ""):@@ -1787,7 +1787,7 @@ class runQueueEvent(bb.event.Event): self.taskid = task self.taskstring = rq.rqdata.get_user_idstring(task) self.taskname = rq.rqdata.get_task_name(task) - self.taskpackage = rq.rqdata.get_task_package(task) + self.taskfile = rq.rqdata.get_task_file(task) self.stats = stats.copy() bb.event.Event.__init__(self)@@ -1800,7 +1800,7 @@ class sceneQueueEvent(runQueueEvent): realtask = rq.rqdata.runq_setscene[task] self.taskstring = rq.rqdata.get_user_idstring(realtask, "_setscene") self.taskname = rq.rqdata.get_task_name(realtask) + "_setscene" - self.taskpackage = rq.rqdata.get_task_package(realtask) + self.taskfile = rq.rqdata.get_task_file(realtask) class runQueueTaskStarted(runQueueEvent): """
bitbake/lib/bb/runqueue.py class runQueueTaskSkipped(runQueueEvent): · line 1851
Quoted 8 lines
bitbake/lib/bb/runqueue.py +1
-1@@ -1845,7 +1845,7 @@ class runQueueTaskSkipped(runQueueEvent): """ def __init__(self, task, stats, rq, skip_reason): runQueueEvent.__init__(self, task, stats, rq) - self._package = rq.rqdata.get_task_package(task) + self._file = rq.rqdata.get_task_file(task) self._task = rq.rqdata.get_task_name(task) self._skip = skip_reason
You added get_task_package() in 2/13, added calls to in 3/13, now we're getting rid of it. Is there not some way to clean up these patches so we just add the right things? Is get_task_package() still needed? Reviewing patches where things get added then removed again is rather confusing...
Cheers,
Richard
PATCH 5/13 bitbake: runqueue: add sceneQueueTaskCompleted event
In reply to Alex DAMIANFrom: Alexandru DAMIAN <[email protected]>
Adding an event to be fired when a scene task is completed. It is analogous to the run task completed event, and has been missing for some reason.
--- bitbake/lib/bb/runqueue.py | 6 ++++++ 1 file changed, 6 insertions(+)
bitbake/lib/bb/runqueue.py +6 -0
@@ -1652,6 +1652,7 @@ class RunQueueExecuteScenequeue(RunQueueExecute): def task_complete(self, task): self.stats.taskCompleted() + bb.event.fire(sceneQueueTaskCompleted(task, self.stats, self.rq), self.cfgData) self.task_completeoutright(task) def task_fail(self, task, result):
@@ -1839,6 +1840,11 @@ class runQueueTaskCompleted(runQueueEvent): Event notifing a task completed """ +class sceneQueueTaskCompleted(sceneQueueEvent): + """ + Event notifing a setscene task completed + """ + class runQueueTaskSkipped(runQueueEvent): """ Event notifing a task was skipped
Signature · 1 line
1.8.1.2
PATCH 6/13 bitbake: build: add link to logfile for Task events
In reply to Alex DAMIANFrom: Alexandru DAMIAN <[email protected]>
We add a pointer to the logfile for Task events, so that we can trace that in the Bitbake database
--- bitbake/lib/bb/build.py | 1 + 1 file changed, 1 insertion(+)
bitbake/lib/bb/build.py +1 -0
@@ -73,6 +73,7 @@ class TaskBase(event.Event): self._task = t self._package = d.getVar("PF", True) self._file = d.getVar("FILE", True) + self.logfile = d.getVar("BB_LOGFILE", True) self.taskfile = self._file self.taskname = self._task event.Event.__init__(self)
Signature · 1 line
1.8.1.2
Re: [PATCH 06/13] bitbake: build: add link to logfile for Task events
Quoted 15 lines · Alex DAMIAN
From: Alexandru DAMIAN <[email protected]>
We add a pointer to the logfile for Task events, so that we can trace that in the Bitbake database
Signed-off-by Alexandru DAMIAN <[email protected]>--- bitbake/lib/bb/build.py | 1 + 1 file changed, 1 insertion(+)bitbake/lib/bb/build.py +0
-0
bitbake/lib/bb/build.py class TaskBase(event.Event): · line 79
Quoted 7 lines
bitbake/lib/bb/build.py +1
-0@@ -73,6 +73,7 @@ class TaskBase(event.Event): self._task = t self._package = d.getVar("PF", True) self._file = d.getVar("FILE", True) + self.logfile = d.getVar("BB_LOGFILE", True) self.taskfile = self._file self.taskname = self._task event.Event.__init__(self)
This leaves the TaskFailed() events still manually passing in logfile as a separate parameter. I think we might as well adjust the code to always pass the logfile to the events instead of this, mainly as it makes it explictly clear where and when the logfile is uses. BB_LOGFILE is really meant more for external usage than for bitbake itself.
Cheers,
Richard
PATCH 7/13 bitbake: cooker, command: add a command to return global data
In reply to Alex DAMIANFrom: Alexandru DAMIAN <[email protected]>
Adding the 'getDataDump' read-only command that will return a dump of the global data state.
This will be used by the DSI to store the build configuration.
--- bitbake/lib/bb/command.py | 7 +++++++ bitbake/lib/bb/cooker.py | 13 +++++++++++++ 2 files changed, 20 insertions(+)
bitbake/lib/bb/command.py +7 -0
@@ -145,6 +145,13 @@ class CommandsSync: """ command.cooker.stop() + def getDataDump(self, command, params): + """ + Returns a dump of the global state. + """ + return command.cooker.getGlobalExpandedData() + getDataDump.readonly = True + def getVariable(self, command, params): """ Read the value of a variable from data
bitbake/lib/bb/cooker.py +13 -0
@@ -1117,6 +1117,19 @@ class BBCooker: self.configuration.server_register_idlecallback(buildTargetsIdle, rq) + + def getGlobalExpandedData(self): + dump = {} + for k in self.data.keys(): + try: + v = self.data.getVar(k, True) + if not k.startswith("__") and not bool(self.data.getVarFlag(k, 'func')) and not isinstance(v, bb.data_smart.DataSmart): + dump[k] = { 'v' : v, 'd' : self.data.getVarFlag(k, 'doc') } + except: + pass + return dump + + def generateNewImage(self, image, base_image, package_queue, timestamp, description): ''' Create a new image with a "require"/"inherit" base_image statement
Signature · 1 line
1.8.1.2
Re: [PATCH 07/13] bitbake: cooker, command: add a command to return global data
Quoted 40 lines · Alex DAMIAN
From: Alexandru DAMIAN <[email protected]>
Adding the 'getDataDump' read-only command that will return a dump of the global data state.
This will be used by the DSI to store the build configuration.
Signed-off-by Alexandru DAMIAN <[email protected]>--- bitbake/lib/bb/command.py | 7 +++++++ bitbake/lib/bb/cooker.py | 13 +++++++++++++ 2 files changed, 20 insertions(+)bitbake/lib/bb/command.py +7
-0@@ -145,6 +145,13 @@ class CommandsSync: """ command.cooker.stop() + def getDataDump(self, command, params): + """ + Returns a dump of the global state. + """ + return command.cooker.getGlobalExpandedData() + getDataDump.readonly = True + def getVariable(self, command, params): """ Read the value of a variable from databitbake/lib/bb/cooker.py +1
-0@@ -1117,6 +1117,19 @@ class BBCooker: self.configuration.server_register_idlecallback(buildTargetsIdle, rq) +
bitbake/lib/bb/cooker.py class BBCooker: · line 1130
Quoted 10 lines
bitbake/lib/bb/cooker.py +10
-0+ def getGlobalExpandedData(self): + dump = {} + for k in self.data.keys(): + try: + v = self.data.getVar(k, True) + if not k.startswith("__") and not bool(self.data.getVarFlag(k, 'func')) and not isinstance(v, bb.data_smart.DataSmart): + dump[k] = { 'v' : v, 'd' : self.data.getVarFlag(k, 'doc') } + except: + pass + return dump
There is a 'doc' hardcoded into there, perhaps this should be a better named command and take a parameter?
Cheers,
Richard
PATCH 8/13 bitbake: cooker: use prepareTreeData when computing environment
In reply to Alex DAMIANFrom: Alexandru DAMIAN <[email protected]>
Eliminate code duplication when building the task data and run list environment, using the prepareTreeData function that is used elsewhere.
--- bitbake/lib/bb/cooker.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-)
bitbake/lib/bb/cooker.py +1 -7
@@ -348,13 +348,7 @@ class BBCooker: if pkgs_to_build[0] in set(ignore.split()): bb.fatal("%s is in ASSUME_PROVIDED" % pkgs_to_build[0]) - localdata = data.createCopy(self.data) - bb.data.update_data(localdata) - bb.data.expandKeys(localdata) - - taskdata = bb.taskdata.TaskData(self.configuration.abort) - taskdata.add_provider(localdata, self.recipecache, pkgs_to_build[0]) - taskdata.add_unresolved(localdata, self.recipecache) + runlist, taskdata = self.prepareTreeData(pkgs_to_build, None) targetid = taskdata.getbuild_id(pkgs_to_build[0]) fnid = taskdata.build_targets[targetid][0]
Signature · 1 line
1.8.1.2
PATCH 9/13 bitbake: event: do not abort on UI handlers error
In reply to Alex DAMIANFrom: Alexandru DAMIAN <[email protected]>
Ignore runtime errors if UI handlers do something
funny, as exiting while the loop is running.
This allows the builder to continue even if something happens to the UI handlers, useful for remote bitbake server.
--- bitbake/lib/bb/event.py | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-)
bitbake/lib/bb/event.py +18 -14
@@ -133,20 +133,24 @@ def fire_ui_handlers(event, d): return errors = [] - for h in _ui_handlers: - #print "Sending event %s" % event - try: - if not _ui_logfilters[h].filter(event): - continue - # We use pickle here since it better handles object instances - # which xmlrpc's marshaller does not. Events *must* be serializable - # by pickle. - if hasattr(_ui_handlers[h].event, "sendpickle"): - _ui_handlers[h].event.sendpickle((pickle.dumps(event))) - else: - _ui_handlers[h].event.send(event) - except: - errors.append(h) + try: + for h in _ui_handlers: + #print "Sending event %s" % event + try: + if not _ui_logfilters[h].filter(event): + continue + # We use pickle here since it better handles object instances + # which xmlrpc's marshaller does not. Events *must* be serializable + # by pickle. + if hasattr(_ui_handlers[h].event, "sendpickle"): + _ui_handlers[h].event.sendpickle((pickle.dumps(event))) + else: + _ui_handlers[h].event.send(event) + except: + errors.append(h) + except RuntimeError: # may happen that the ui handler simple disappears :D + errors.append(h) + for h in errors: del _ui_handlers[h]
Signature · 1 line
1.8.1.2
Re: [PATCH 09/13] bitbake: event: do not abort on UI handlers error
Quoted 4 lines · Alex DAMIAN
From: Alexandru DAMIAN <[email protected]>
Ignore runtime errors if UI handlers do something
funny, as exiting while the loop is running.
This does look rather strange to me. How can a handler exit within the loop? The handler is either another process or something we're using an xmlrpc call too. Fine, the other end can disappear but our local end (and entry in _ui_handlers) should exist until we remove it ourselves?
Only the server itself can do the removal and the server is single threaded?
Cheers,
Richard
Quoted 52 lines
This allows the builder to continue even if something happens to the UI handlers, useful for remote bitbake server.
Signed-off-by Alexandru DAMIAN <[email protected]>--- bitbake/lib/bb/event.py | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-)bitbake/lib/bb/event.py +18
-14@@ -133,20 +133,24 @@ def fire_ui_handlers(event, d): return errors = [] - for h in _ui_handlers: - #print "Sending event %s" % event - try: - if not _ui_logfilters[h].filter(event): - continue - # We use pickle here since it better handles object instances - # which xmlrpc's marshaller does not. Events *must* be serializable - # by pickle. - if hasattr(_ui_handlers[h].event, "sendpickle"): - _ui_handlers[h].event.sendpickle((pickle.dumps(event))) - else: - _ui_handlers[h].event.send(event) - except: - errors.append(h) + try: + for h in _ui_handlers: + #print "Sending event %s" % event + try: + if not _ui_logfilters[h].filter(event): + continue + # We use pickle here since it better handles object instances + # which xmlrpc's marshaller does not. Events *must* be serializable + # by pickle. + if hasattr(_ui_handlers[h].event, "sendpickle"): + _ui_handlers[h].event.sendpickle((pickle.dumps(event))) + else: + _ui_handlers[h].event.send(event) + except: + errors.append(h) + except RuntimeError: # may happen that the ui handler simple disappears :D + errors.append(h) + for h in errors: del _ui_handlers[h]
PATCH 10/13 bitbake: add SinglePackageInfo event
In reply to Alex DAMIANFrom: Alexandru DAMIAN <[email protected]>
We add a new event that will be triggered by the package-building class, containing the information about the currently built package.
--- bitbake/lib/bb/event.py | 8 ++++++++ bitbake/lib/bb/ui/knotty.py | 1 + 2 files changed, 9 insertions(+)
bitbake/lib/bb/event.py +8 -0
@@ -593,6 +593,14 @@ class PackageInfo(Event): Event.__init__(self) self._pkginfolist = pkginfolist +class SinglePackageInfo(Event): + """ + Single Package Information sent on emit_pkgdata + """ + def __init__(self, data): + Event.__init__(self) + self._data = data + class SanityCheck(Event): """ Event to issue sanity check
bitbake/lib/bb/ui/knotty.py +1 -0
@@ -473,6 +473,7 @@ def main(server, eventHandler, params, tf = TerminalFilter): # ignore if isinstance(event, (bb.event.BuildBase, + bb.event.SinglePackageInfo, bb.event.StampUpdate, bb.event.ConfigParsed, bb.event.RecipeParsed,
Signature · 1 line
1.8.1.2
PATCH 11/13 bitbake: event: adding an event to pass build file sizes
In reply to Alex DAMIANFrom: Alexandru DAMIAN <[email protected]>
Adding an event, ignored by default, that will pass information about the content of each package file that gets built.
--- bitbake/lib/bb/event.py | 8 ++++++++ bitbake/lib/bb/ui/knotty.py | 1 + 2 files changed, 9 insertions(+)
bitbake/lib/bb/event.py +8 -0
@@ -601,6 +601,14 @@ class SinglePackageInfo(Event): Event.__init__(self) self._data = data +class PackageFileSizes(Event): + """ + Event that contains information about the file sizes contained in a package. + """ + def __init__(self, data): + Event.__init__(self) + self._data = data + class SanityCheck(Event): """ Event to issue sanity check
bitbake/lib/bb/ui/knotty.py +1 -0
@@ -473,6 +473,7 @@ def main(server, eventHandler, params, tf = TerminalFilter): # ignore if isinstance(event, (bb.event.BuildBase, + bb.event.PackageFileSizes, bb.event.SinglePackageInfo, bb.event.StampUpdate, bb.event.ConfigParsed,
Signature · 1 line
1.8.1.2
PATCH 12/13 bitbake: xmlrpc: delete unused code BitBakeUIEventServer
In reply to Alex DAMIANFrom: Alexandru DAMIAN <[email protected]>
BitBakeUIEventServer is an unused class that pushes UI events over a separate thread.
The current version of XMLRPC server works just fine with the classic UI event handlers, so this class is not needed.
--- bitbake/lib/bb/server/xmlrpc.py | 45 ----------------------------------------- 1 file changed, 45 deletions(-)
bitbake/lib/bb/server/xmlrpc.py +0 -45
@@ -169,51 +169,6 @@ class BitBakeXMLRPCRequestHandler(SimpleXMLRPCRequestHandler): self.end_headers() self.wfile.write(response) -class BitBakeUIEventServer(threading.Thread): - class EventAdapter(): - """ - Adapter to wrap our event queue since the caller (bb.event) expects to - call a send() method, but our actual queue only has put() - """ - def __init__(self, notify): - self.queue = [] - self.notify = notify - self.qlock = threading.Lock() - - def send(self, event): - self.qlock.acquire() - self.queue.append(event) - self.qlock.release() - self.notify.set() - - def get(self): - self.qlock.acquire() - if len(self.queue) == 0: - self.qlock.release() - return None - e = self.queue.pop(0) - if len(self.queue) == 0: - self.notify.clear() - self.qlock.release() - return e - - def __init__(self, connection): - self.connection = connection - self.notify = threading.Event() - self.event = BitBakeUIEventServer.EventAdapter(self.notify) - self.quit = False - threading.Thread.__init__(self) - - def terminateServer(self): - self.quit = True - - def run(self): - while not self.quit: - self.notify.wait(0.1) - evt = self.event.get() - if evt: - self.connection.event.sendpickle(pickle.dumps(evt)) - class XMLRPCProxyServer(BaseImplServer): """ not a real working server, but a stub for a proxy server connection
Signature · 1 line
1.8.1.2
PATCH 13/13 bitbake: cooker: add extra recipe information
In reply to Alex DAMIANFrom: Alexandru DAMIAN <[email protected]>
Adding in the extra recipe information when creating the dependency tree information.
This works in server-mode because the HOB extra_cache is already enabled. In normal mode, it will do nothing with no performance impact.
--- bitbake/lib/bb/cooker.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+)
bitbake/lib/bb/cooker.py +23 -0
@@ -439,6 +439,29 @@ class BBCooker: depend_tree["pn"][pn] = {} depend_tree["pn"][pn]["filename"] = fn depend_tree["pn"][pn]["version"] = version + + # This data is needed for webhob; + # it's here if the bitbake runs in server mode + try: + summary = self.recipecache.summary[fn] + lic = self.recipecache.license[fn] + section = self.recipecache.section[fn] + description = self.recipecache.description[fn] + homepage = self.recipecache.homepage[fn] + bugtracker = self.recipecache.bugtracker[fn] + inherits = self.recipecache.inherits.get(fn, None) + depend_tree["pn"][pn]["filename"] = fn + depend_tree["pn"][pn]["version"] = version + depend_tree["pn"][pn]["summary"] = summary + depend_tree["pn"][pn]["license"] = lic + depend_tree["pn"][pn]["section"] = section + depend_tree["pn"][pn]["description"] = description + depend_tree["pn"][pn]["inherits"] = inherits + depend_tree["pn"][pn]["homepage"] = homepage + depend_tree["pn"][pn]["bugtracker"] = bugtracker + except KeyError: + pass + for dep in rq.rqdata.runq_depends[task]: depfn = taskdata.fn_index[rq.rqdata.runq_fnid[dep]] deppn = self.recipecache.pkg_fn[depfn]
Signature · 1 line
1.8.1.2