Skip to contentSkip to conversation

webhob related changes in bitbake, version 2

PATCH Superseded Alex DAMIAN 2 participants 11 messages Sep 16, 2013

v1 of 3 Superseded Latest is v3 →

lore
Share & tools
lore
#0 lore raw

cover webhob related changes in bitbake, version 2

From: Alexandru DAMIAN <[email protected]>

I'm sending in version 2 of the patches that bring in changes that prepare bitbake for supporting WebHob-related tasks.

The changes focus on improving and standardizing the information sent through the event system, and cleaning up code in bitbake.

The following changes since commit 5fca4d286eec240ad57d90defa43cd0ff24f88ed:

linux-yocto/3.10: genericx86 meta data factoring (2013-09-14 08:36:26 +0100)

are available in the git repository at:

git://git.yoctoproject.org/poky-contrib adamian/bitbake-core-changes-1609
http://git.yoctoproject.org/cgit.cgi/poky-contrib/log/?h=adamian/bitbake-core-changes-1609

Alexandru DAMIAN (8):
bitbake: cooker: clean up code and avoid duplication
bitbake: event: send the task dependency tree to UI
bitbake: runqueue: add runQueueTaskSkipped event
bitbake: cooker: add extra recipe information
bitbake: build: add logfile path for all Task events
bitbake: cooker, command: add a command to return global data
bitbake: event: adding events to allow bbclasses to push custom info
bitbake: runqueue: add task hash to Queue events

 bitbake/lib/bb/build.py     | 18 ++++++++---
 bitbake/lib/bb/command.py   |  8 +++++
 bitbake/lib/bb/cooker.py    | 79 +++++++++++++++++++++++++++++----------------
 bitbake/lib/bb/event.py     | 16 +++++++++
 bitbake/lib/bb/runqueue.py  | 35 ++++++++++++++++++--
 bitbake/lib/bb/ui/knotty.py |  5 +++
 6 files changed, 125 insertions(+), 36 deletions(-)
Signature · 1 line
1.8.1.2
#1 lore raw

PATCH 1/8 bitbake: cooker: clean up code and avoid duplication

From: Alexandru DAMIAN <[email protected]>

Clean-up to avoid duplication and promote code reuse. We're using the prepareTreeData in all instances where this needs to be done.

bb.event.BuildStarted now contains unexpanded package names, since the UI will show the original target names.

When we're only testing target image contents, we do not abort with an error if the target is unbuildable.

---
 bitbake/lib/bb/cooker.py | 38 ++++++++++----------------------------
 1 file changed, 10 insertions(+), 28 deletions(-)
bitbake/lib/bb/cooker.py +10 -28
index a07615b..3bfeefb 100644
@@ -345,13 +345,7 @@ class BBCooker:
             if pkgs_to_build[0] in set(ignore.split()):
                 bb.fatal("%s is in ASSUME_PROVIDED" % pkgs_to_build[0])
 
-            localdata = data.createCopy(self.data)
-            bb.data.update_data(localdata)
-            bb.data.expandKeys(localdata)
-
-            taskdata = bb.taskdata.TaskData(self.configuration.abort)
-            taskdata.add_provider(localdata, self.recipecache, pkgs_to_build[0])
-            taskdata.add_unresolved(localdata, self.recipecache)
+            runlist, taskdata = self.prepareTreeData(pkgs_to_build, None)
 
             targetid = taskdata.getbuild_id(pkgs_to_build[0])
             fnid = taskdata.build_targets[targetid][0]
@@ -398,9 +392,7 @@ class BBCooker:
         localdata = data.createCopy(self.data)
         bb.data.update_data(localdata)
         bb.data.expandKeys(localdata)
-        # We set abort to False here to prevent unbuildable targets raising
-        # an exception when we're just generating data
-        taskdata = bb.taskdata.TaskData(False, skiplist=self.skiplist)
+        taskdata = bb.taskdata.TaskData(self.configuration.abort, skiplist=self.skiplist)
 
         runlist = []
         current = 0
@@ -492,7 +484,12 @@ class BBCooker:
         """
         Create a dependency tree of pkgs_to_build, returning the data.
         """
+
+        # We set abort to False here to prevent unbuildable targets raising
+        # an exception when we're just generating data
+        self.configuration.abort = False
         _, taskdata = self.prepareTreeData(pkgs_to_build, task)
+
         tasks_fnid = []
         if len(taskdata.tasks_name) != 0:
             for task in xrange(len(taskdata.tasks_name)):
@@ -1073,13 +1070,6 @@ class BBCooker:
         Attempt to build the targets specified
         """
 
-        # If we are told to do the NULL task then query the default task
-        if (task == None):
-            task = self.configuration.cmd
-
-        universe = ('universe' in targets)
-        targets = self.checkPackages(targets)
-
         def buildTargetsIdle(server, rq, abort):
             if abort or self.state == state.forceshutdown:
                 rq.finish_runqueue(True)
@@ -1106,21 +1096,13 @@ class BBCooker:
         self.buildSetVars()
 
         buildname = self.data.getVar("BUILDNAME")
-        bb.event.fire(bb.event.BuildStarted(buildname, targets), self.data)
-
-        localdata = data.createCopy(self.data)
-        bb.data.update_data(localdata)
-        bb.data.expandKeys(localdata)
 
-        taskdata = bb.taskdata.TaskData(self.configuration.abort, skiplist=self.skiplist)
+        bb.event.fire(bb.event.BuildStarted(buildname, targets), self.data)
 
-        runlist = []
-        for k in targets:
-            taskdata.add_provider(localdata, self.recipecache, k)
-            runlist.append([k, "do_%s" % task])
-        taskdata.add_unresolved(localdata, self.recipecache)
+        runlist, taskdata = self.prepareTreeData(targets, task)
 
         rq = bb.runqueue.RunQueue(self, self.data, self.recipecache, taskdata, runlist)
+        universe = ('universe' in targets)
         if universe:
             rq.rqdata.warn_multi_bb = True
 
Signature · 1 line
1.8.1.2
#2 lore raw

PATCH 2/8 bitbake: event: send the task dependency tree to UI

From: Alexandru DAMIAN <[email protected]>

We add an event that will send the dependency tree graph to the UI client once it is computed.

This will allow the clients to display dependency data in an efficient manner, and not redo the runqueue computation specifically to get the dependency data.

---
 bitbake/lib/bb/cooker.py    | 3 +++
 bitbake/lib/bb/runqueue.py  | 3 +++
 bitbake/lib/bb/ui/knotty.py | 3 +++
 3 files changed, 9 insertions(+)
bitbake/lib/bb/cooker.py +3 -0
index 3bfeefb..68a3c01 100644
@@ -415,7 +415,10 @@ class BBCooker:
         runlist, taskdata = self.prepareTreeData(pkgs_to_build, task)
         rq = bb.runqueue.RunQueue(self, self.data, self.recipecache, taskdata, runlist)
         rq.rqdata.prepare()
+        return self.buildDependTree(rq, taskdata)
 
+
+    def buildDependTree(self, rq, taskdata):
         seen_fnids = []
         depend_tree = {}
         depend_tree["depends"] = {}
bitbake/lib/bb/runqueue.py +3 -0
index 25f1ab5..aa2f147 100644
@@ -999,6 +999,9 @@ class RunQueue:
             else:
                 self.state = runQueueSceneInit
 
+                depgraph = self.cooker.buildDependTree(self, self.rqdata.taskData)
+                bb.event.fire(bb.event.DepTreeGenerated(depgraph), self.cooker.data)
+
         if self.state is runQueueSceneInit:
             if self.cooker.configuration.dump_signatures:
                 self.dump_signatures()
bitbake/lib/bb/ui/knotty.py +3 -0
index 35590a2..0211b50 100644
@@ -472,6 +472,9 @@ def main(server, eventHandler, params, tf = TerminalFilter):
                              event.taskid, event.taskstring, event.exitcode)
                 continue
 
+            if isinstance(event, bb.event.DepTreeGenerated):
+                continue
+
             # ignore
             if isinstance(event, (bb.event.BuildBase,
                                   bb.event.StampUpdate,
Signature · 1 line
1.8.1.2
#3 lore raw

PATCH 3/8 bitbake: runqueue: add runQueueTaskSkipped event

In reply to Alex DAMIAN

From: Alexandru DAMIAN <[email protected]>

Adding a runQueueTaskSkipped to notify all listeners about the tasks that are not run either because they are set-scened or they don't need an update (timestamps are ok)

Adds RunQueueData functions to get the task name and task file for usage with the runQueue* events.

---
 bitbake/lib/bb/build.py    |  3 +++
 bitbake/lib/bb/runqueue.py | 27 ++++++++++++++++++++++++---
 2 files changed, 27 insertions(+), 3 deletions(-)
bitbake/lib/bb/build.py +3 -0
index a53aba9..1265bdc 100644
@@ -72,6 +72,9 @@ class TaskBase(event.Event):
     def __init__(self, t, d ):
         self._task = t
         self._package = d.getVar("PF", True)
+        self._file = d.getVar("FILE", True)
+        self.taskfile = self._file
+        self.taskname = self._task
         event.Event.__init__(self)
         self._message = "recipe %s: task %s: %s" % (d.getVar("PF", True), t, self.getDisplayName())
 
bitbake/lib/bb/runqueue.py +24 -3
index aa2f147..2ab4405 100644
@@ -217,6 +217,12 @@ class RunQueueData:
             ret.extend([nam])
         return ret
 
+    def get_task_name(self, task):
+        return self.runq_task[task]
+
+    def get_task_file(self, task):
+        return self.taskData.fn_index[self.runq_fnid[task]]
+
     def get_user_idstring(self, task, task_name_suffix = ""):
         fn = self.taskData.fn_index[self.runq_fnid[task]]
         taskname = self.runq_task[task] + task_name_suffix
@@ -1328,9 +1334,10 @@ class RunQueueExecuteTasks(RunQueueExecute):
         if self.rqdata.taskData.abort:
             self.rq.state = runQueueCleanUp
 
-    def task_skip(self, task):
+    def task_skip(self, task, skip_reason = ""):
         self.runq_running[task] = 1
         self.runq_buildable[task] = 1
+        bb.event.fire(runQueueTaskSkipped(task, self.stats, self.rq, skip_reason), self.cfgData)
         self.task_completeoutright(task)
         self.stats.taskCompleted()
         self.stats.taskSkipped()
@@ -1355,13 +1362,13 @@ class RunQueueExecuteTasks(RunQueueExecute):
             if task in self.rq.scenequeue_covered:
                 logger.debug(2, "Setscene covered task %s (%s)", task,
                                 self.rqdata.get_user_idstring(task))
-                self.task_skip(task)
+                self.task_skip(task, "covered")
                 return True
 
             if self.rq.check_stamp_task(task, taskname, cache=self.stampcache):
                 logger.debug(2, "Stamp current task %s (%s)", task,
                                 self.rqdata.get_user_idstring(task))
-                self.task_skip(task)
+                self.task_skip(task, "existing")
                 return True
 
             taskdep = self.rqdata.dataCache.task_deps[fn]
@@ -1783,6 +1790,8 @@ class runQueueEvent(bb.event.Event):
     def __init__(self, task, stats, rq):
         self.taskid = task
         self.taskstring = rq.rqdata.get_user_idstring(task)
+        self.taskname = rq.rqdata.get_task_name(task)
+        self.taskfile = rq.rqdata.get_task_file(task)
         self.stats = stats.copy()
         bb.event.Event.__init__(self)
 
@@ -1794,6 +1803,8 @@ class sceneQueueEvent(runQueueEvent):
         runQueueEvent.__init__(self, task, stats, rq)
         realtask = rq.rqdata.runq_setscene[task]
         self.taskstring = rq.rqdata.get_user_idstring(realtask, "_setscene")
+        self.taskname = rq.rqdata.get_task_name(realtask) + "_setscene"
+        self.taskfile = rq.rqdata.get_task_file(realtask)
 
 class runQueueTaskStarted(runQueueEvent):
     """
@@ -1837,6 +1848,16 @@ class sceneQueueTaskCompleted(sceneQueueEvent):
     Event notifing a setscene task completed
     """
 
+class runQueueTaskSkipped(runQueueEvent):
+    """
+    Event notifing a task was skipped
+    """
+    def __init__(self, task, stats, rq, skip_reason):
+        runQueueEvent.__init__(self, task, stats, rq)
+        self._file = rq.rqdata.get_task_file(task)
+        self._task = rq.rqdata.get_task_name(task)
+        self._skip = skip_reason
+
 class runQueuePipe():
     """
     Abstraction for a pipe between a worker thread and the server
Signature · 1 line
1.8.1.2
#4 lore raw

PATCH 4/8 bitbake: cooker: add extra recipe information

In reply to Alex DAMIAN

From: Alexandru DAMIAN <[email protected]>

Adding in the extra recipe information when creating the dependency tree information.

This works in server-mode because the HOB extra_cache is already enabled. In normal mode, it will do nothing with no performance impact.

---
 bitbake/lib/bb/cooker.py | 23 +++++++++++++++++++++++
 1 file changed, 23 insertions(+)
bitbake/lib/bb/cooker.py +23 -0
index 68a3c01..cb0e3e5 100644
@@ -439,6 +439,29 @@ class BBCooker:
                 depend_tree["pn"][pn] = {}
                 depend_tree["pn"][pn]["filename"] = fn
                 depend_tree["pn"][pn]["version"] = version
+
+                # This data is needed for webhob;
+                # it's here if the bitbake runs in server mode
+                try:
+                    summary = self.recipecache.summary[fn]
+                    lic = self.recipecache.license[fn]
+                    section = self.recipecache.section[fn]
+                    description = self.recipecache.description[fn]
+                    homepage = self.recipecache.homepage[fn]
+                    bugtracker = self.recipecache.bugtracker[fn]
+                    inherits = self.recipecache.inherits.get(fn, None)
+                    depend_tree["pn"][pn]["filename"] = fn
+                    depend_tree["pn"][pn]["version"] = version
+                    depend_tree["pn"][pn]["summary"] = summary
+                    depend_tree["pn"][pn]["license"] = lic
+                    depend_tree["pn"][pn]["section"] = section
+                    depend_tree["pn"][pn]["description"] = description
+                    depend_tree["pn"][pn]["inherits"] = inherits
+                    depend_tree["pn"][pn]["homepage"] = homepage
+                    depend_tree["pn"][pn]["bugtracker"] = bugtracker
+                except:
+                    pass
+
             for dep in rq.rqdata.runq_depends[task]:
                 depfn = taskdata.fn_index[rq.rqdata.runq_fnid[dep]]
                 deppn = self.recipecache.pkg_fn[depfn]
Signature · 1 line
1.8.1.2
#5 lore raw

PATCH 5/8 bitbake: build: add logfile path for all Task events

In reply to Alex DAMIAN

From: Alexandru DAMIAN <[email protected]>

We add the path to the logfile for all Task events, so that we can record the logfile locations during the run.

This is needed because the logfile symlinks will point only to the last build logs, and there is no other link between a certain build and the log file it produces.

---
 bitbake/lib/bb/build.py | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)
bitbake/lib/bb/build.py +10 -5
index 1265bdc..8d001b1 100644
@@ -91,13 +91,18 @@ class TaskBase(event.Event):
 
 class TaskStarted(TaskBase):
     """Task execution started"""
+    def __init__(self, task, logfile, metadata):
+        self.logfile = logfile
+        super(TaskStarted, self).__init__(task, metadata)
 
 class TaskSucceeded(TaskBase):
     """Task execution completed"""
+    def __init__(self, task, logfile, metadata):
+        self.logfile = logfile
+        super(TaskSucceeded, self).__init__(task, metadata)
 
 class TaskFailed(TaskBase):
     """Task execution failed"""
-
     def __init__(self, task, logfile, metadata, errprinted = False):
         self.logfile = logfile
         self.errprinted = errprinted
@@ -114,8 +119,8 @@ class TaskFailedSilent(TaskBase):
         return "Failed"
 
 class TaskInvalid(TaskBase):
-
-    def __init__(self, task, metadata):
+    def __init__(self, task, logfile, metadata):
+        self.logfile = logfile
         super(TaskInvalid, self).__init__(task, metadata)
         self._message = "No such task '%s'" % task
 
@@ -419,7 +424,7 @@ def _exec_task(fn, task, d, quieterr):
     localdata.setVar('BB_LOGFILE', logfn)
     localdata.setVar('BB_RUNTASK', task)
 
-    event.fire(TaskStarted(task, localdata), localdata)
+    event.fire(TaskStarted(task, logfn, localdata), localdata)
     try:
         for func in (prefuncs or '').split():
             exec_func(func, localdata)
@@ -456,7 +461,7 @@ def _exec_task(fn, task, d, quieterr):
             logger.debug(2, "Zero size logfn %s, removing", logfn)
             bb.utils.remove(logfn)
             bb.utils.remove(loglink)
-    event.fire(TaskSucceeded(task, localdata), localdata)
+    event.fire(TaskSucceeded(task, logfn, localdata), localdata)
 
     if not localdata.getVarFlag(task, 'nostamp') and not localdata.getVarFlag(task, 'selfstamp'):
         make_stamp(task, localdata)
Signature · 1 line
1.8.1.2
#6 lore raw

Re: [PATCH 5/8] bitbake: build: add logfile path for all Task events

Quoted 5 lines · Alex DAMIAN

From: Alexandru DAMIAN <[email protected]>

We add the path to the logfile for all Task events, so that we can record the logfile locations during the run.

Quoted 8 lines

This is needed because the logfile symlinks will point only to the last build logs, and there is no other link between a certain build and the log file it produces.

---
 bitbake/lib/bb/build.py | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

a) you might as well modify TaskBase intead of each Task
b) You didn't fix up all the original Task* references e.g. TaskInvalid.

I gave up and merged my own version of this.

Cheers,

Richard

#7 lore raw

PATCH 6/8 bitbake: cooker, command: add a command to return global data

In reply to Alex DAMIAN

From: Alexandru DAMIAN <[email protected]>

Adding the 'getDataDump' read-only command that will return a dump of the global data state.

This will be used by UI clients to get the build configuration.

---
 bitbake/lib/bb/command.py |  8 ++++++++
 bitbake/lib/bb/cooker.py  | 15 +++++++++++++++
 2 files changed, 23 insertions(+)
bitbake/lib/bb/command.py +8 -0
index f1abaf7..84136f1 100644
@@ -145,6 +145,14 @@ class CommandsSync:
         """
         command.cooker.shutdown(True)
 
+    def getDataDump(self, command, params):
+        """
+        Returns a dump of the global state. Call with
+        variable flags to be retrieved as params.
+        """
+        return command.cooker.getGlobalExpandedData(params)
+    getDataDump.readonly = True
+
     def getVariable(self, command, params):
         """
         Read the value of a variable from data
bitbake/lib/bb/cooker.py +15 -0
index cb0e3e5..12155b9 100644
@@ -1134,6 +1134,21 @@ class BBCooker:
 
         self.configuration.server_register_idlecallback(buildTargetsIdle, rq)
 
+
+    def getGlobalExpandedData(self, flags):
+        dump = {}
+        for k in self.data.keys():
+            try:
+                v = self.data.getVar(k, True)
+                if not k.startswith("__") and not bool(self.data.getVarFlag(k, 'func')) and not isinstance(v, bb.data_smart.DataSmart):
+                    dump[k] = { 'v' : v }
+                    for d in flags:
+                        dump[k][d] = self.data.getVarFlag(k, d)
+            except:
+                pass
+        return dump
+
+
     def generateNewImage(self, image, base_image, package_queue, timestamp, description):
         '''
         Create a new image with a "require"/"inherit" base_image statement
Signature · 1 line
1.8.1.2
#8 lore raw

Re: [PATCH 6/8] bitbake: cooker, command: add a command to return global data

Quoted 18 lines · Alex DAMIAN

From: Alexandru DAMIAN <[email protected]>

Adding the 'getDataDump' read-only command that will return a dump of the global data state.

This will be used by UI clients to get the build configuration.

---
 bitbake/lib/bb/command.py |  8 ++++++++
 bitbake/lib/bb/cooker.py  | 15 +++++++++++++++
 2 files changed, 23 insertions(+)
bitbake/lib/bb/command.py +0 -0
index f1abaf7..84136f1 100644

bitbake/lib/bb/command.py class CommandsSync: · line 153

Quoted 9 lines
bitbake/lib/bb/command.py +6 -0
@@ -145,6 +145,14 @@ class CommandsSync:
         """
         command.cooker.shutdown(True)
 
+    def getDataDump(self, command, params):
+        """
+        Returns a dump of the global state. Call with
+        variable flags to be retrieved as params.
+        """
+        return command.cooker.getGlobalExpandedData(params)

To be blunt, this *sucks* as an API call. "getDataDump" tells me nothing about what it does. "getGlobalExpandedData" doesn't do much better.

"getAllKeyFlags"?

and please pass in a list as param[0] so there is a:

flaglist = params[0]

in there. This means we can add other parameters if we ever need to and we don't have complete lock in to the API.

Cheers,

Richard

#9 lore raw

PATCH 7/8 bitbake: event: adding events to allow bbclasses to push custom info

In reply to Alex DAMIAN

From: Alexandru DAMIAN <[email protected]>

We add a new event that will be triggered by the package-building class, containing the information about the currently built package.

Adding an event that will passi information about the content of each package file that gets built.

Knotty UI will ignore these events by default.

Maybe a mechanism for using a single generic event is in order ?

---
 bitbake/lib/bb/event.py     | 16 ++++++++++++++++
 bitbake/lib/bb/ui/knotty.py |  2 ++
 2 files changed, 18 insertions(+)
bitbake/lib/bb/event.py +16 -0
index 67cfcea..7dd9236 100644
@@ -589,6 +589,22 @@ class PackageInfo(Event):
         Event.__init__(self)
         self._pkginfolist = pkginfolist
 
+class SinglePackageInfo(Event):
+    """
+    Single Package Information sent on emit_pkgdata
+    """
+    def __init__(self, data):
+        Event.__init__(self)
+        self._data = data
+
+class PackageFileSizes(Event):
+    """
+    Event that contains information about the file sizes contained in a package.
+    """
+    def __init__(self, data):
+        Event.__init__(self)
+        self._data = data
+
 class SanityCheck(Event):
     """
     Event to issue sanity check
bitbake/lib/bb/ui/knotty.py +2 -0
index 0211b50..03baa75 100644
@@ -477,6 +477,8 @@ def main(server, eventHandler, params, tf = TerminalFilter):
 
             # ignore
             if isinstance(event, (bb.event.BuildBase,
+                                  bb.event.PackageFileSizes,
+                                  bb.event.SinglePackageInfo,
                                   bb.event.StampUpdate,
                                   bb.event.ConfigParsed,
                                   bb.event.RecipeParsed,
Signature · 1 line
1.8.1.2
#10 lore raw

PATCH 8/8 bitbake: runqueue: add task hash to Queue events

In reply to Alex DAMIAN

From: Alexandru DAMIAN <[email protected]>

Adding the sstate-related hash for all runqueue and scenequeue tasks, as it's needed in the WebHob data.

---
 bitbake/lib/bb/runqueue.py | 5 +++++
 1 file changed, 5 insertions(+)
bitbake/lib/bb/runqueue.py +5 -0
index 2ab4405..18ebce0 100644
@@ -223,6 +223,9 @@ class RunQueueData:
     def get_task_file(self, task):
         return self.taskData.fn_index[self.runq_fnid[task]]
 
+    def get_task_hash(self, task):
+        return self.runq_hash[task]
+
     def get_user_idstring(self, task, task_name_suffix = ""):
         fn = self.taskData.fn_index[self.runq_fnid[task]]
         taskname = self.runq_task[task] + task_name_suffix
@@ -1792,6 +1795,7 @@ class runQueueEvent(bb.event.Event):
         self.taskstring = rq.rqdata.get_user_idstring(task)
         self.taskname = rq.rqdata.get_task_name(task)
         self.taskfile = rq.rqdata.get_task_file(task)
+        self.taskhash = rq.rqdata.get_task_hash(task)
         self.stats = stats.copy()
         bb.event.Event.__init__(self)
 
@@ -1805,6 +1809,7 @@ class sceneQueueEvent(runQueueEvent):
         self.taskstring = rq.rqdata.get_user_idstring(realtask, "_setscene")
         self.taskname = rq.rqdata.get_task_name(realtask) + "_setscene"
         self.taskfile = rq.rqdata.get_task_file(realtask)
+        self.taskhash = rq.rqdata.get_task_hash(task)
 
 class runQueueTaskStarted(runQueueEvent):
     """
Signature · 1 line
1.8.1.2