Skip to contentSkip to conversation

webhob related changes in bitbake

PATCH In review Alex DAMIAN 2 participants 18 messages Sep 9, 2013 – Sep 13, 2013

lore
Share & tools
lore
#0 lore raw

cover webhob related changes in bitbake

From: Alexandru DAMIAN <[email protected]>

I'm sending in a set of patches that bring in changes that prepare bitbake for supporting WebHob-related tasks.

The changes focus on improving and standardizing the information sent through the event system, and cleaning up code in bitbake.

The following changes since commit 0ca5d1fb38157564a2b9452ade32391d18a41b09:

yocto-bsp: remove apm as a default MACHINE_FEATURE (2013-09-09 16:28:47 +0100)

are available in the git repository at:

git://git.yoctoproject.org/poky-contrib adamian/bitbake-core-changes-0909
http://git.yoctoproject.org/cgit.cgi/poky-contrib/log/?h=adamian/bitbake-core-changes-0909

Alexandru DAMIAN (13):
bitbake: cooker: clean up code and avoid duplication
bitbake: runqueue: Add runQueueTaskSkipped event
bitbake: runqueue: added prebuild detailed information
bitbake: runqueue, build: event data change
bitbake: runqueue: add sceneQueueTaskCompleted event
bitbake: build: add link to logfile for Task events
bitbake: cooker, command: add a command to return global data
bitbake: cooker: use prepareTreeData when computing environment
bitbake: event: do not abort on UI handlers error
bitbake: add SinglePackageInfo event
bitbake: event: adding an event to pass build file sizes
bitbake: xmlrpc: delete unused code BitBakeUIEventServer
bitbake: cooker: add extra recipe information

 bitbake/lib/bb/build.py         |  4 +++
 bitbake/lib/bb/command.py       |  7 ++++
 bitbake/lib/bb/cooker.py        | 74 +++++++++++++++++++++++++----------------
 bitbake/lib/bb/event.py         | 48 ++++++++++++++++++--------
 bitbake/lib/bb/runqueue.py      | 33 ++++++++++++++++--
 bitbake/lib/bb/server/xmlrpc.py | 45 -------------------------
 bitbake/lib/bb/ui/knotty.py     |  2 ++
 7 files changed, 123 insertions(+), 90 deletions(-)
Signature · 1 line
1.8.1.2
#1 lore raw

PATCH 1/13 bitbake: cooker: clean up code and avoid duplication

From: Alexandru DAMIAN <[email protected]>

Clean-up to avoid duplication and promote code reuse. We're using the prepareTreeData in all instances where this needs to be done.

When we're only testing target image contents (HOB), we do not abort with an error if the target is unbuildable.

---
 bitbake/lib/bb/cooker.py | 30 +++++++++---------------------
 1 file changed, 9 insertions(+), 21 deletions(-)
bitbake/lib/bb/cooker.py +9 -21
index f03f16b..9fd3417 100644
@@ -401,9 +401,7 @@ class BBCooker:
         localdata = data.createCopy(self.data)
         bb.data.update_data(localdata)
         bb.data.expandKeys(localdata)
-        # We set abort to False here to prevent unbuildable targets raising
-        # an exception when we're just generating data
-        taskdata = bb.taskdata.TaskData(False, skiplist=self.skiplist)
+        taskdata = bb.taskdata.TaskData(self.configuration.abort, skiplist=self.skiplist)
 
         runlist = []
         current = 0
@@ -495,7 +493,12 @@ class BBCooker:
         """
         Create a dependency tree of pkgs_to_build, returning the data.
         """
+
+        # We set abort to False here to prevent unbuildable targets raising
+        # an exception when we're just generating data
+        self.configuration.abort = False
         _, taskdata = self.prepareTreeData(pkgs_to_build, task)
+
         tasks_fnid = []
         if len(taskdata.tasks_name) != 0:
             for task in xrange(len(taskdata.tasks_name)):
@@ -1076,13 +1079,6 @@ class BBCooker:
         Attempt to build the targets specified
         """
 
-        # If we are told to do the NULL task then query the default task
-        if (task == None):
-            task = self.configuration.cmd
-
-        universe = ('universe' in targets)
-        targets = self.checkPackages(targets)
-
         def buildTargetsIdle(server, rq, abort):
             if abort or self.state == state.stop:
                 rq.finish_runqueue(True)
@@ -1109,21 +1105,13 @@ class BBCooker:
         self.buildSetVars()
 
         buildname = self.data.getVar("BUILDNAME")
-        bb.event.fire(bb.event.BuildStarted(buildname, targets), self.data)
-
-        localdata = data.createCopy(self.data)
-        bb.data.update_data(localdata)
-        bb.data.expandKeys(localdata)
 
-        taskdata = bb.taskdata.TaskData(self.configuration.abort, skiplist=self.skiplist)
+        bb.event.fire(bb.event.BuildStarted(buildname, targets), self.data)
 
-        runlist = []
-        for k in targets:
-            taskdata.add_provider(localdata, self.recipecache, k)
-            runlist.append([k, "do_%s" % task])
-        taskdata.add_unresolved(localdata, self.recipecache)
+        runlist, taskdata = self.prepareTreeData(targets, task)
 
         rq = bb.runqueue.RunQueue(self, self.data, self.recipecache, taskdata, runlist)
+        universe = ('universe' in targets)
         if universe:
             rq.rqdata.warn_multi_bb = True
 
Signature · 1 line
1.8.1.2
#2 lore raw

PATCH 2/13 bitbake: runqueue: Add runQueueTaskSkipped event

From: Alexandru DAMIAN <[email protected]>

Adding a runQueueTaskSkipped to notify all listeners about the tasks that are not run either because they are set-scened or they don't need an update (timestamps are ok)

---
 bitbake/lib/bb/runqueue.py | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)
bitbake/lib/bb/runqueue.py +16 -0
index a868332..f084407 100644
@@ -217,6 +217,12 @@ class RunQueueData:
             ret.extend([nam])
         return ret
 
+    def get_task_name(self, task):
+        return self.runq_task[task]
+
+    def get_task_package(self, task):
+        return self.taskData.fn_index[self.runq_fnid[task]]
+
     def get_user_idstring(self, task, task_name_suffix = ""):
         fn = self.taskData.fn_index[self.runq_fnid[task]]
         taskname = self.runq_task[task] + task_name_suffix
@@ -1328,6 +1334,7 @@ class RunQueueExecuteTasks(RunQueueExecute):
     def task_skip(self, task):
         self.runq_running[task] = 1
         self.runq_buildable[task] = 1
+        bb.event.fire(runQueueTaskSkipped(task, self.stats, self.rq), self.cfgData)
         self.task_completeoutright(task)
         self.stats.taskCompleted()
         self.stats.taskSkipped()
@@ -1828,6 +1835,15 @@ class runQueueTaskCompleted(runQueueEvent):
     Event notifing a task completed
     """
 
+class runQueueTaskSkipped(runQueueEvent):
+    """
+    Event notifing a task was skipped
+    """
+    def __init__(self, task, stats, rq):
+        runQueueEvent.__init__(self, task, stats, rq)
+        self._package = rq.rqdata.get_task_package(task)
+        self._task = rq.rqdata.get_task_name(task)
+
 class runQueuePipe():
     """
     Abstraction for a pipe between a worker thread and the server
Signature · 1 line
1.8.1.2
#3 lore raw

PATCH 3/13 bitbake: runqueue: added prebuild detailed information

In reply to Alex DAMIAN

From: Alexandru DAMIAN <[email protected]>

Adds why the prebuild tasks were skipped,
as to distinguish about covered/existing tasks.

Adds task name and task package to all runQueue
and sceneQueue events.

---
 bitbake/lib/bb/runqueue.py | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)
bitbake/lib/bb/runqueue.py +10 -5
index f084407..002bf76 100644
@@ -1331,10 +1331,10 @@ class RunQueueExecuteTasks(RunQueueExecute):
         if self.rqdata.taskData.abort:
             self.rq.state = runQueueCleanUp
 
-    def task_skip(self, task):
+    def task_skip(self, task, skip_reason = ""):
         self.runq_running[task] = 1
         self.runq_buildable[task] = 1
-        bb.event.fire(runQueueTaskSkipped(task, self.stats, self.rq), self.cfgData)
+        bb.event.fire(runQueueTaskSkipped(task, self.stats, self.rq, skip_reason), self.cfgData)
         self.task_completeoutright(task)
         self.stats.taskCompleted()
         self.stats.taskSkipped()
@@ -1359,13 +1359,13 @@ class RunQueueExecuteTasks(RunQueueExecute):
             if task in self.rq.scenequeue_covered:
                 logger.debug(2, "Setscene covered task %s (%s)", task,
                                 self.rqdata.get_user_idstring(task))
-                self.task_skip(task)
+                self.task_skip(task, "covered")
                 return True
 
             if self.rq.check_stamp_task(task, taskname, cache=self.stampcache):
                 logger.debug(2, "Stamp current task %s (%s)", task,
                                 self.rqdata.get_user_idstring(task))
-                self.task_skip(task)
+                self.task_skip(task, "existing")
                 return True
 
             taskdep = self.rqdata.dataCache.task_deps[fn]
@@ -1786,6 +1786,8 @@ class runQueueEvent(bb.event.Event):
     def __init__(self, task, stats, rq):
         self.taskid = task
         self.taskstring = rq.rqdata.get_user_idstring(task)
+        self.taskname = rq.rqdata.get_task_name(task)
+        self.taskpackage = rq.rqdata.get_task_package(task)
         self.stats = stats.copy()
         bb.event.Event.__init__(self)
 
@@ -1797,6 +1799,8 @@ class sceneQueueEvent(runQueueEvent):
         runQueueEvent.__init__(self, task, stats, rq)
         realtask = rq.rqdata.runq_setscene[task]
         self.taskstring = rq.rqdata.get_user_idstring(realtask, "_setscene")
+        self.taskname = rq.rqdata.get_task_name(realtask) + "_setscene"
+        self.taskpackage = rq.rqdata.get_task_package(realtask)
 
 class runQueueTaskStarted(runQueueEvent):
     """
@@ -1839,10 +1843,11 @@ class runQueueTaskSkipped(runQueueEvent):
     """
     Event notifing a task was skipped
     """
-    def __init__(self, task, stats, rq):
+    def __init__(self, task, stats, rq, skip_reason):
         runQueueEvent.__init__(self, task, stats, rq)
         self._package = rq.rqdata.get_task_package(task)
         self._task = rq.rqdata.get_task_name(task)
+        self._skip = skip_reason
 
 class runQueuePipe():
     """
Signature · 1 line
1.8.1.2
#4 lore raw

PATCH 4/13 bitbake: runqueue, build: event data change

In reply to Alex DAMIAN

From: Alexandru DAMIAN <[email protected]>

We modify event data to make sure we can match build events with runqueue events. This is important because the build events supply datastore-evaluated data while runqueue events supply bitbake-related data. We get the recipe name and version from the datastore, while the file paths and timing information are collected from the RunQueue.

We modify filling-in the information side to take advantage of the new info supplied.

---
 bitbake/lib/bb/build.py    | 3 +++
 bitbake/lib/bb/runqueue.py | 8 ++++----
 2 files changed, 7 insertions(+), 4 deletions(-)
bitbake/lib/bb/build.py +3 -0
index a53aba9..1265bdc 100644
@@ -72,6 +72,9 @@ class TaskBase(event.Event):
     def __init__(self, t, d ):
         self._task = t
         self._package = d.getVar("PF", True)
+        self._file = d.getVar("FILE", True)
+        self.taskfile = self._file
+        self.taskname = self._task
         event.Event.__init__(self)
         self._message = "recipe %s: task %s: %s" % (d.getVar("PF", True), t, self.getDisplayName())
 
bitbake/lib/bb/runqueue.py +4 -4
index 002bf76..73dd059 100644
@@ -220,7 +220,7 @@ class RunQueueData:
     def get_task_name(self, task):
         return self.runq_task[task]
 
-    def get_task_package(self, task):
+    def get_task_file(self, task):
         return self.taskData.fn_index[self.runq_fnid[task]]
 
     def get_user_idstring(self, task, task_name_suffix = ""):
@@ -1787,7 +1787,7 @@ class runQueueEvent(bb.event.Event):
         self.taskid = task
         self.taskstring = rq.rqdata.get_user_idstring(task)
         self.taskname = rq.rqdata.get_task_name(task)
-        self.taskpackage = rq.rqdata.get_task_package(task)
+        self.taskfile = rq.rqdata.get_task_file(task)
         self.stats = stats.copy()
         bb.event.Event.__init__(self)
 
@@ -1800,7 +1800,7 @@ class sceneQueueEvent(runQueueEvent):
         realtask = rq.rqdata.runq_setscene[task]
         self.taskstring = rq.rqdata.get_user_idstring(realtask, "_setscene")
         self.taskname = rq.rqdata.get_task_name(realtask) + "_setscene"
-        self.taskpackage = rq.rqdata.get_task_package(realtask)
+        self.taskfile = rq.rqdata.get_task_file(realtask)
 
 class runQueueTaskStarted(runQueueEvent):
     """
@@ -1845,7 +1845,7 @@ class runQueueTaskSkipped(runQueueEvent):
     """
     def __init__(self, task, stats, rq, skip_reason):
         runQueueEvent.__init__(self, task, stats, rq)
-        self._package = rq.rqdata.get_task_package(task)
+        self._file = rq.rqdata.get_task_file(task)
         self._task = rq.rqdata.get_task_name(task)
         self._skip = skip_reason
 
Signature · 1 line
1.8.1.2
#5 lore raw

Re: [PATCH 04/13] bitbake: runqueue, build: event data change

Quoted 64 lines · Alex DAMIAN

From: Alexandru DAMIAN <[email protected]>

We modify event data to make sure we can match build events with runqueue events. This is important because the build events supply datastore-evaluated data while runqueue events supply bitbake-related data. We get the recipe name and version from the datastore, while the file paths and timing information are collected from the RunQueue.

We modify filling-in the information side to take advantage of the new info supplied.

---
 bitbake/lib/bb/build.py    | 3 +++
 bitbake/lib/bb/runqueue.py | 8 ++++----
 2 files changed, 7 insertions(+), 4 deletions(-)
bitbake/lib/bb/build.py +3 -0
index a53aba9..1265bdc 100644
@@ -72,6 +72,9 @@ class TaskBase(event.Event):
     def __init__(self, t, d ):
         self._task = t
         self._package = d.getVar("PF", True)
+        self._file = d.getVar("FILE", True)
+        self.taskfile = self._file
+        self.taskname = self._task
         event.Event.__init__(self)
         self._message = "recipe %s: task %s: %s" % (d.getVar("PF", True), t, self.getDisplayName())
 
bitbake/lib/bb/runqueue.py +3 -3
index 002bf76..73dd059 100644
@@ -220,7 +220,7 @@ class RunQueueData:
     def get_task_name(self, task):
         return self.runq_task[task]
 
-    def get_task_package(self, task):
+    def get_task_file(self, task):
         return self.taskData.fn_index[self.runq_fnid[task]]
 
     def get_user_idstring(self, task, task_name_suffix = ""):
@@ -1787,7 +1787,7 @@ class runQueueEvent(bb.event.Event):
         self.taskid = task
         self.taskstring = rq.rqdata.get_user_idstring(task)
         self.taskname = rq.rqdata.get_task_name(task)
-        self.taskpackage = rq.rqdata.get_task_package(task)
+        self.taskfile = rq.rqdata.get_task_file(task)
         self.stats = stats.copy()
         bb.event.Event.__init__(self)
 
@@ -1800,7 +1800,7 @@ class sceneQueueEvent(runQueueEvent):
         realtask = rq.rqdata.runq_setscene[task]
         self.taskstring = rq.rqdata.get_user_idstring(realtask, "_setscene")
         self.taskname = rq.rqdata.get_task_name(realtask) + "_setscene"
-        self.taskpackage = rq.rqdata.get_task_package(realtask)
+        self.taskfile = rq.rqdata.get_task_file(realtask)
 
 class runQueueTaskStarted(runQueueEvent):
     """

bitbake/lib/bb/runqueue.py class runQueueTaskSkipped(runQueueEvent): · line 1851

Quoted 8 lines
bitbake/lib/bb/runqueue.py +1 -1
@@ -1845,7 +1845,7 @@ class runQueueTaskSkipped(runQueueEvent):
     """
     def __init__(self, task, stats, rq, skip_reason):
         runQueueEvent.__init__(self, task, stats, rq)
-        self._package = rq.rqdata.get_task_package(task)
+        self._file = rq.rqdata.get_task_file(task)
         self._task = rq.rqdata.get_task_name(task)
         self._skip = skip_reason
 

You added get_task_package() in 2/13, added calls to in 3/13, now we're getting rid of it. Is there not some way to clean up these patches so we just add the right things? Is get_task_package() still needed? Reviewing patches where things get added then removed again is rather confusing...

Cheers,

Richard

#6 lore raw

PATCH 5/13 bitbake: runqueue: add sceneQueueTaskCompleted event

In reply to Alex DAMIAN

From: Alexandru DAMIAN <[email protected]>

Adding an event to be fired when a scene task is completed. It is analogous to the run task completed event, and has been missing for some reason.

---
 bitbake/lib/bb/runqueue.py | 6 ++++++
 1 file changed, 6 insertions(+)
bitbake/lib/bb/runqueue.py +6 -0
index 73dd059..11ff193 100644
@@ -1652,6 +1652,7 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
 
     def task_complete(self, task):
         self.stats.taskCompleted()
+        bb.event.fire(sceneQueueTaskCompleted(task, self.stats, self.rq), self.cfgData)
         self.task_completeoutright(task)
 
     def task_fail(self, task, result):
@@ -1839,6 +1840,11 @@ class runQueueTaskCompleted(runQueueEvent):
     Event notifing a task completed
     """
 
+class sceneQueueTaskCompleted(sceneQueueEvent):
+    """
+    Event notifing a setscene task completed
+    """
+
 class runQueueTaskSkipped(runQueueEvent):
     """
     Event notifing a task was skipped
Signature · 1 line
1.8.1.2
#7 lore raw

PATCH 6/13 bitbake: build: add link to logfile for Task events

In reply to Alex DAMIAN

From: Alexandru DAMIAN <[email protected]>

We add a pointer to the logfile for Task events, so that we can trace that in the Bitbake database

---
 bitbake/lib/bb/build.py | 1 +
 1 file changed, 1 insertion(+)
bitbake/lib/bb/build.py +1 -0
index 1265bdc..4abbe48 100644
@@ -73,6 +73,7 @@ class TaskBase(event.Event):
         self._task = t
         self._package = d.getVar("PF", True)
         self._file = d.getVar("FILE", True)
+        self.logfile = d.getVar("BB_LOGFILE", True)
         self.taskfile = self._file
         self.taskname = self._task
         event.Event.__init__(self)
Signature · 1 line
1.8.1.2
#8 lore raw

Re: [PATCH 06/13] bitbake: build: add link to logfile for Task events

Quoted 15 lines · Alex DAMIAN

From: Alexandru DAMIAN <[email protected]>

We add a pointer to the logfile for Task events, so that we can trace that in the Bitbake database

---
 bitbake/lib/bb/build.py | 1 +
 1 file changed, 1 insertion(+)
bitbake/lib/bb/build.py +0 -0
index 1265bdc..4abbe48 100644

bitbake/lib/bb/build.py class TaskBase(event.Event): · line 79

Quoted 7 lines
bitbake/lib/bb/build.py +1 -0
@@ -73,6 +73,7 @@ class TaskBase(event.Event):
         self._task = t
         self._package = d.getVar("PF", True)
         self._file = d.getVar("FILE", True)
+        self.logfile = d.getVar("BB_LOGFILE", True)
         self.taskfile = self._file
         self.taskname = self._task
         event.Event.__init__(self)

This leaves the TaskFailed() events still manually passing in logfile as a separate parameter. I think we might as well adjust the code to always pass the logfile to the events instead of this, mainly as it makes it explictly clear where and when the logfile is uses. BB_LOGFILE is really meant more for external usage than for bitbake itself.

Cheers,

Richard

#9 lore raw

PATCH 7/13 bitbake: cooker, command: add a command to return global data

In reply to Alex DAMIAN

From: Alexandru DAMIAN <[email protected]>

Adding the 'getDataDump' read-only command that will return a dump of the global data state.

This will be used by the DSI to store the build configuration.

---
 bitbake/lib/bb/command.py |  7 +++++++
 bitbake/lib/bb/cooker.py  | 13 +++++++++++++
 2 files changed, 20 insertions(+)
bitbake/lib/bb/command.py +7 -0
index 3527ad0..4d36c4b 100644
@@ -145,6 +145,13 @@ class CommandsSync:
         """
         command.cooker.stop()
 
+    def getDataDump(self, command, params):
+        """
+        Returns a dump of the global state.
+        """
+        return command.cooker.getGlobalExpandedData()
+    getDataDump.readonly = True
+
     def getVariable(self, command, params):
         """
         Read the value of a variable from data
bitbake/lib/bb/cooker.py +13 -0
index 9fd3417..383c9c4 100644
@@ -1117,6 +1117,19 @@ class BBCooker:
 
         self.configuration.server_register_idlecallback(buildTargetsIdle, rq)
 
+
+    def getGlobalExpandedData(self):
+        dump = {}
+        for k in self.data.keys():
+            try:
+                v = self.data.getVar(k, True)
+                if not k.startswith("__") and not bool(self.data.getVarFlag(k, 'func')) and not isinstance(v, bb.data_smart.DataSmart):
+                    dump[k] = { 'v' : v, 'd' : self.data.getVarFlag(k, 'doc') }
+            except:
+                pass
+        return dump
+
+
     def generateNewImage(self, image, base_image, package_queue, timestamp, description):
         '''
         Create a new image with a "require"/"inherit" base_image statement
Signature · 1 line
1.8.1.2
#10 lore raw

Re: [PATCH 07/13] bitbake: cooker, command: add a command to return global data

Quoted 40 lines · Alex DAMIAN

From: Alexandru DAMIAN <[email protected]>

Adding the 'getDataDump' read-only command that will return a dump of the global data state.

This will be used by the DSI to store the build configuration.

---
 bitbake/lib/bb/command.py |  7 +++++++
 bitbake/lib/bb/cooker.py  | 13 +++++++++++++
 2 files changed, 20 insertions(+)
bitbake/lib/bb/command.py +7 -0
index 3527ad0..4d36c4b 100644
@@ -145,6 +145,13 @@ class CommandsSync:
         """
         command.cooker.stop()
 
+    def getDataDump(self, command, params):
+        """
+        Returns a dump of the global state.
+        """
+        return command.cooker.getGlobalExpandedData()
+    getDataDump.readonly = True
+
     def getVariable(self, command, params):
         """
         Read the value of a variable from data
bitbake/lib/bb/cooker.py +1 -0
index 9fd3417..383c9c4 100644
@@ -1117,6 +1117,19 @@ class BBCooker:
 
         self.configuration.server_register_idlecallback(buildTargetsIdle, rq)
 
+

bitbake/lib/bb/cooker.py class BBCooker: · line 1130

Quoted 10 lines
bitbake/lib/bb/cooker.py +10 -0
+    def getGlobalExpandedData(self):
+        dump = {}
+        for k in self.data.keys():
+            try:
+                v = self.data.getVar(k, True)
+                if not k.startswith("__") and not bool(self.data.getVarFlag(k, 'func')) and not isinstance(v, bb.data_smart.DataSmart):
+                    dump[k] = { 'v' : v, 'd' : self.data.getVarFlag(k, 'doc') }
+            except:
+                pass
+        return dump

There is a 'doc' hardcoded into there, perhaps this should be a better named command and take a parameter?

Cheers,

Richard

#11 lore raw

PATCH 8/13 bitbake: cooker: use prepareTreeData when computing environment

In reply to Alex DAMIAN

From: Alexandru DAMIAN <[email protected]>

Eliminate code duplication when building the task data and run list environment, using the prepareTreeData function that is used elsewhere.

---
 bitbake/lib/bb/cooker.py | 8 +-------
 1 file changed, 1 insertion(+), 7 deletions(-)
bitbake/lib/bb/cooker.py +1 -7
index 383c9c4..1eb1768 100644
@@ -348,13 +348,7 @@ class BBCooker:
             if pkgs_to_build[0] in set(ignore.split()):
                 bb.fatal("%s is in ASSUME_PROVIDED" % pkgs_to_build[0])
 
-            localdata = data.createCopy(self.data)
-            bb.data.update_data(localdata)
-            bb.data.expandKeys(localdata)
-
-            taskdata = bb.taskdata.TaskData(self.configuration.abort)
-            taskdata.add_provider(localdata, self.recipecache, pkgs_to_build[0])
-            taskdata.add_unresolved(localdata, self.recipecache)
+            runlist, taskdata = self.prepareTreeData(pkgs_to_build, None)
 
             targetid = taskdata.getbuild_id(pkgs_to_build[0])
             fnid = taskdata.build_targets[targetid][0]
Signature · 1 line
1.8.1.2
#12 lore raw

PATCH 9/13 bitbake: event: do not abort on UI handlers error

In reply to Alex DAMIAN

From: Alexandru DAMIAN <[email protected]>

Ignore runtime errors if UI handlers do something
funny, as exiting while the loop is running.

This allows the builder to continue even if something happens to the UI handlers, useful for remote bitbake server.

---
 bitbake/lib/bb/event.py | 32 ++++++++++++++++++--------------
 1 file changed, 18 insertions(+), 14 deletions(-)
bitbake/lib/bb/event.py +18 -14
index 67cfcea..1258471 100644
@@ -133,20 +133,24 @@ def fire_ui_handlers(event, d):
         return
 
     errors = []
-    for h in _ui_handlers:
-        #print "Sending event %s" % event
-        try:
-             if not _ui_logfilters[h].filter(event):
-                 continue
-             # We use pickle here since it better handles object instances
-             # which xmlrpc's marshaller does not. Events *must* be serializable
-             # by pickle.
-             if hasattr(_ui_handlers[h].event, "sendpickle"):
-                _ui_handlers[h].event.sendpickle((pickle.dumps(event)))
-             else:
-                _ui_handlers[h].event.send(event)
-        except:
-            errors.append(h)
+    try:
+        for h in _ui_handlers:
+            #print "Sending event %s" % event
+            try:
+                if not _ui_logfilters[h].filter(event):
+                    continue
+                # We use pickle here since it better handles object instances
+                # which xmlrpc's marshaller does not. Events *must* be serializable
+                # by pickle.
+                if hasattr(_ui_handlers[h].event, "sendpickle"):
+                    _ui_handlers[h].event.sendpickle((pickle.dumps(event)))
+                else:
+                    _ui_handlers[h].event.send(event)
+            except:
+                errors.append(h)
+    except RuntimeError:     # may happen that the ui handler simple disappears :D
+        errors.append(h)
+
     for h in errors:
         del _ui_handlers[h]
 
Signature · 1 line
1.8.1.2
#13 lore raw

Re: [PATCH 09/13] bitbake: event: do not abort on UI handlers error

Quoted 4 lines · Alex DAMIAN

From: Alexandru DAMIAN <[email protected]>

Ignore runtime errors if UI handlers do something
funny, as exiting while the loop is running.

This does look rather strange to me. How can a handler exit within the loop? The handler is either another process or something we're using an xmlrpc call too. Fine, the other end can disappear but our local end (and entry in _ui_handlers) should exist until we remove it ourselves?

Only the server itself can do the removal and the server is single threaded?

Cheers,

Richard

Quoted 52 lines

This allows the builder to continue even if something happens to the UI handlers, useful for remote bitbake server.

---
 bitbake/lib/bb/event.py | 32 ++++++++++++++++++--------------
 1 file changed, 18 insertions(+), 14 deletions(-)
bitbake/lib/bb/event.py +18 -14
index 67cfcea..1258471 100644
@@ -133,20 +133,24 @@ def fire_ui_handlers(event, d):
         return
 
     errors = []
-    for h in _ui_handlers:
-        #print "Sending event %s" % event
-        try:
-             if not _ui_logfilters[h].filter(event):
-                 continue
-             # We use pickle here since it better handles object instances
-             # which xmlrpc's marshaller does not. Events *must* be serializable
-             # by pickle.
-             if hasattr(_ui_handlers[h].event, "sendpickle"):
-                _ui_handlers[h].event.sendpickle((pickle.dumps(event)))
-             else:
-                _ui_handlers[h].event.send(event)
-        except:
-            errors.append(h)
+    try:
+        for h in _ui_handlers:
+            #print "Sending event %s" % event
+            try:
+                if not _ui_logfilters[h].filter(event):
+                    continue
+                # We use pickle here since it better handles object instances
+                # which xmlrpc's marshaller does not. Events *must* be serializable
+                # by pickle.
+                if hasattr(_ui_handlers[h].event, "sendpickle"):
+                    _ui_handlers[h].event.sendpickle((pickle.dumps(event)))
+                else:
+                    _ui_handlers[h].event.send(event)
+            except:
+                errors.append(h)
+    except RuntimeError:     # may happen that the ui handler simple disappears :D
+        errors.append(h)
+
     for h in errors:
         del _ui_handlers[h]
 
#14 lore raw

PATCH 10/13 bitbake: add SinglePackageInfo event

In reply to Alex DAMIAN

From: Alexandru DAMIAN <[email protected]>

We add a new event that will be triggered by the package-building class, containing the information about the currently built package.

---
 bitbake/lib/bb/event.py     | 8 ++++++++
 bitbake/lib/bb/ui/knotty.py | 1 +
 2 files changed, 9 insertions(+)
bitbake/lib/bb/event.py +8 -0
index 1258471..4be0e5b 100644
@@ -593,6 +593,14 @@ class PackageInfo(Event):
         Event.__init__(self)
         self._pkginfolist = pkginfolist
 
+class SinglePackageInfo(Event):
+    """
+    Single Package Information sent on emit_pkgdata
+    """
+    def __init__(self, data):
+        Event.__init__(self)
+        self._data = data
+
 class SanityCheck(Event):
     """
     Event to issue sanity check
bitbake/lib/bb/ui/knotty.py +1 -0
index 4520046..010530a 100644
@@ -473,6 +473,7 @@ def main(server, eventHandler, params, tf = TerminalFilter):
 
             # ignore
             if isinstance(event, (bb.event.BuildBase,
+                                  bb.event.SinglePackageInfo,
                                   bb.event.StampUpdate,
                                   bb.event.ConfigParsed,
                                   bb.event.RecipeParsed,
Signature · 1 line
1.8.1.2
#15 lore raw

PATCH 11/13 bitbake: event: adding an event to pass build file sizes

In reply to Alex DAMIAN

From: Alexandru DAMIAN <[email protected]>

Adding an event, ignored by default, that will pass information about the content of each package file that gets built.

---
 bitbake/lib/bb/event.py     | 8 ++++++++
 bitbake/lib/bb/ui/knotty.py | 1 +
 2 files changed, 9 insertions(+)
bitbake/lib/bb/event.py +8 -0
index 4be0e5b..781b8e7 100644
@@ -601,6 +601,14 @@ class SinglePackageInfo(Event):
         Event.__init__(self)
         self._data = data
 
+class PackageFileSizes(Event):
+    """
+    Event that contains information about the file sizes contained in a package.
+    """
+    def __init__(self, data):
+        Event.__init__(self)
+        self._data = data
+
 class SanityCheck(Event):
     """
     Event to issue sanity check
bitbake/lib/bb/ui/knotty.py +1 -0
index 010530a..1f09a55 100644
@@ -473,6 +473,7 @@ def main(server, eventHandler, params, tf = TerminalFilter):
 
             # ignore
             if isinstance(event, (bb.event.BuildBase,
+                                  bb.event.PackageFileSizes,
                                   bb.event.SinglePackageInfo,
                                   bb.event.StampUpdate,
                                   bb.event.ConfigParsed,
Signature · 1 line
1.8.1.2
#16 lore raw

PATCH 12/13 bitbake: xmlrpc: delete unused code BitBakeUIEventServer

In reply to Alex DAMIAN

From: Alexandru DAMIAN <[email protected]>

BitBakeUIEventServer is an unused class that pushes UI events over a separate thread.

The current version of XMLRPC server works just fine with the classic UI event handlers, so this class is not needed.

---
 bitbake/lib/bb/server/xmlrpc.py | 45 -----------------------------------------
 1 file changed, 45 deletions(-)
bitbake/lib/bb/server/xmlrpc.py +0 -45
index a28672c..8326623 100644
@@ -169,51 +169,6 @@ class BitBakeXMLRPCRequestHandler(SimpleXMLRPCRequestHandler):
         self.end_headers()
         self.wfile.write(response)
 
-class BitBakeUIEventServer(threading.Thread):
-    class EventAdapter():
-        """
-        Adapter to wrap our event queue since the caller (bb.event) expects to
-        call a send() method, but our actual queue only has put()
-        """
-        def __init__(self, notify):
-            self.queue = []
-            self.notify = notify
-            self.qlock = threading.Lock()
-
-        def send(self, event):
-            self.qlock.acquire()
-            self.queue.append(event)
-            self.qlock.release()
-            self.notify.set()
-
-        def get(self):
-            self.qlock.acquire()
-            if len(self.queue) == 0:
-                self.qlock.release()
-                return None
-            e = self.queue.pop(0)
-            if len(self.queue) == 0:
-                self.notify.clear()
-            self.qlock.release()
-            return e
-
-    def __init__(self, connection):
-        self.connection = connection
-        self.notify = threading.Event()
-        self.event = BitBakeUIEventServer.EventAdapter(self.notify)
-        self.quit = False
-        threading.Thread.__init__(self)
-
-    def terminateServer(self):
-        self.quit = True
-
-    def run(self):
-        while not self.quit:
-            self.notify.wait(0.1)
-            evt = self.event.get()
-            if evt:
-                self.connection.event.sendpickle(pickle.dumps(evt))
-
 
 class XMLRPCProxyServer(BaseImplServer):
     """ not a real working server, but a stub for a proxy server connection
Signature · 1 line
1.8.1.2
#17 lore raw

PATCH 13/13 bitbake: cooker: add extra recipe information

In reply to Alex DAMIAN

From: Alexandru DAMIAN <[email protected]>

Adding in the extra recipe information when creating the dependency tree information.

This works in server-mode because the HOB extra_cache is already enabled. In normal mode, it will do nothing with no performance impact.

---
 bitbake/lib/bb/cooker.py | 23 +++++++++++++++++++++++
 1 file changed, 23 insertions(+)
bitbake/lib/bb/cooker.py +23 -0
index 1eb1768..bad930b 100644
@@ -439,6 +439,29 @@ class BBCooker:
                 depend_tree["pn"][pn] = {}
                 depend_tree["pn"][pn]["filename"] = fn
                 depend_tree["pn"][pn]["version"] = version
+
+                # This data is needed for webhob;
+                # it's here if the bitbake runs in server mode
+                try:
+                    summary = self.recipecache.summary[fn]
+                    lic = self.recipecache.license[fn]
+                    section = self.recipecache.section[fn]
+                    description = self.recipecache.description[fn]
+                    homepage = self.recipecache.homepage[fn]
+                    bugtracker = self.recipecache.bugtracker[fn]
+                    inherits = self.recipecache.inherits.get(fn, None)
+                    depend_tree["pn"][pn]["filename"] = fn
+                    depend_tree["pn"][pn]["version"] = version
+                    depend_tree["pn"][pn]["summary"] = summary
+                    depend_tree["pn"][pn]["license"] = lic
+                    depend_tree["pn"][pn]["section"] = section
+                    depend_tree["pn"][pn]["description"] = description
+                    depend_tree["pn"][pn]["inherits"] = inherits
+                    depend_tree["pn"][pn]["homepage"] = homepage
+                    depend_tree["pn"][pn]["bugtracker"] = bugtracker
+                except KeyError:
+                    pass
+
             for dep in rq.rqdata.runq_depends[task]:
                 depfn = taskdata.fn_index[rq.rqdata.runq_fnid[dep]]
                 deppn = self.recipecache.pkg_fn[depfn]
Signature · 1 line
1.8.1.2