webhob related changes in bitbake, version 2
cover webhob related changes in bitbake, version 2
From: Alexandru DAMIAN <[email protected]>
I'm sending in version 2 of the patches that bring in changes that prepare bitbake for supporting WebHob-related tasks.
The changes focus on improving and standardizing the information sent through the event system, and cleaning up code in bitbake.
The following changes since commit 5fca4d286eec240ad57d90defa43cd0ff24f88ed:
linux-yocto/3.10: genericx86 meta data factoring (2013-09-14 08:36:26 +0100)
are available in the git repository at:
git://git.yoctoproject.org/poky-contrib adamian/bitbake-core-changes-1609
http://git.yoctoproject.org/cgit.cgi/poky-contrib/log/?h=adamian/bitbake-core-changes-1609
Alexandru DAMIAN (8):
bitbake: cooker: clean up code and avoid duplication
bitbake: event: send the task dependency tree to UI
bitbake: runqueue: add runQueueTaskSkipped event
bitbake: cooker: add extra recipe information
bitbake: build: add logfile path for all Task events
bitbake: cooker, command: add a command to return global data
bitbake: event: adding events to allow bbclasses to push custom info
bitbake: runqueue: add task hash to Queue events
bitbake/lib/bb/build.py | 18 ++++++++--- bitbake/lib/bb/command.py | 8 +++++ bitbake/lib/bb/cooker.py | 79 +++++++++++++++++++++++++++++---------------- bitbake/lib/bb/event.py | 16 +++++++++ bitbake/lib/bb/runqueue.py | 35 ++++++++++++++++++-- bitbake/lib/bb/ui/knotty.py | 5 +++ 6 files changed, 125 insertions(+), 36 deletions(-)
Signature · 1 line
1.8.1.2
PATCH 1/8 bitbake: cooker: clean up code and avoid duplication
From: Alexandru DAMIAN <[email protected]>
Clean-up to avoid duplication and promote code reuse. We're using the prepareTreeData in all instances where this needs to be done.
bb.event.BuildStarted now contains unexpanded package names, since the UI will show the original target names.
When we're only testing target image contents, we do not abort with an error if the target is unbuildable.
--- bitbake/lib/bb/cooker.py | 38 ++++++++++---------------------------- 1 file changed, 10 insertions(+), 28 deletions(-)
bitbake/lib/bb/cooker.py +10 -28
@@ -345,13 +345,7 @@ class BBCooker: if pkgs_to_build[0] in set(ignore.split()): bb.fatal("%s is in ASSUME_PROVIDED" % pkgs_to_build[0]) - localdata = data.createCopy(self.data) - bb.data.update_data(localdata) - bb.data.expandKeys(localdata) - - taskdata = bb.taskdata.TaskData(self.configuration.abort) - taskdata.add_provider(localdata, self.recipecache, pkgs_to_build[0]) - taskdata.add_unresolved(localdata, self.recipecache) + runlist, taskdata = self.prepareTreeData(pkgs_to_build, None) targetid = taskdata.getbuild_id(pkgs_to_build[0]) fnid = taskdata.build_targets[targetid][0]
@@ -398,9 +392,7 @@ class BBCooker: localdata = data.createCopy(self.data) bb.data.update_data(localdata) bb.data.expandKeys(localdata) - # We set abort to False here to prevent unbuildable targets raising - # an exception when we're just generating data - taskdata = bb.taskdata.TaskData(False, skiplist=self.skiplist) + taskdata = bb.taskdata.TaskData(self.configuration.abort, skiplist=self.skiplist) runlist = [] current = 0
@@ -492,7 +484,12 @@ class BBCooker: """ Create a dependency tree of pkgs_to_build, returning the data. """ + + # We set abort to False here to prevent unbuildable targets raising + # an exception when we're just generating data + self.configuration.abort = False _, taskdata = self.prepareTreeData(pkgs_to_build, task) + tasks_fnid = [] if len(taskdata.tasks_name) != 0: for task in xrange(len(taskdata.tasks_name)):
@@ -1073,13 +1070,6 @@ class BBCooker: Attempt to build the targets specified """ - # If we are told to do the NULL task then query the default task - if (task == None): - task = self.configuration.cmd - - universe = ('universe' in targets) - targets = self.checkPackages(targets) - def buildTargetsIdle(server, rq, abort): if abort or self.state == state.forceshutdown: rq.finish_runqueue(True)
@@ -1106,21 +1096,13 @@ class BBCooker: self.buildSetVars() buildname = self.data.getVar("BUILDNAME") - bb.event.fire(bb.event.BuildStarted(buildname, targets), self.data) - - localdata = data.createCopy(self.data) - bb.data.update_data(localdata) - bb.data.expandKeys(localdata) - taskdata = bb.taskdata.TaskData(self.configuration.abort, skiplist=self.skiplist) + bb.event.fire(bb.event.BuildStarted(buildname, targets), self.data) - runlist = [] - for k in targets: - taskdata.add_provider(localdata, self.recipecache, k) - runlist.append([k, "do_%s" % task]) - taskdata.add_unresolved(localdata, self.recipecache) + runlist, taskdata = self.prepareTreeData(targets, task) rq = bb.runqueue.RunQueue(self, self.data, self.recipecache, taskdata, runlist) + universe = ('universe' in targets) if universe: rq.rqdata.warn_multi_bb = True
Signature · 1 line
1.8.1.2
PATCH 2/8 bitbake: event: send the task dependency tree to UI
From: Alexandru DAMIAN <[email protected]>
We add an event that will send the dependency tree graph to the UI client once it is computed.
This will allow the clients to display dependency data in an efficient manner, and not redo the runqueue computation specifically to get the dependency data.
--- bitbake/lib/bb/cooker.py | 3 +++ bitbake/lib/bb/runqueue.py | 3 +++ bitbake/lib/bb/ui/knotty.py | 3 +++ 3 files changed, 9 insertions(+)
bitbake/lib/bb/cooker.py +3 -0
@@ -415,7 +415,10 @@ class BBCooker: runlist, taskdata = self.prepareTreeData(pkgs_to_build, task) rq = bb.runqueue.RunQueue(self, self.data, self.recipecache, taskdata, runlist) rq.rqdata.prepare() + return self.buildDependTree(rq, taskdata) + + def buildDependTree(self, rq, taskdata): seen_fnids = [] depend_tree = {} depend_tree["depends"] = {}
bitbake/lib/bb/runqueue.py +3 -0
@@ -999,6 +999,9 @@ class RunQueue: else: self.state = runQueueSceneInit + depgraph = self.cooker.buildDependTree(self, self.rqdata.taskData) + bb.event.fire(bb.event.DepTreeGenerated(depgraph), self.cooker.data) + if self.state is runQueueSceneInit: if self.cooker.configuration.dump_signatures: self.dump_signatures()
bitbake/lib/bb/ui/knotty.py +3 -0
@@ -472,6 +472,9 @@ def main(server, eventHandler, params, tf = TerminalFilter): event.taskid, event.taskstring, event.exitcode) continue + if isinstance(event, bb.event.DepTreeGenerated): + continue + # ignore if isinstance(event, (bb.event.BuildBase, bb.event.StampUpdate,
Signature · 1 line
1.8.1.2
PATCH 3/8 bitbake: runqueue: add runQueueTaskSkipped event
In reply to Alex DAMIANFrom: Alexandru DAMIAN <[email protected]>
Adding a runQueueTaskSkipped to notify all listeners about the tasks that are not run either because they are set-scened or they don't need an update (timestamps are ok)
Adds RunQueueData functions to get the task name and task file for usage with the runQueue* events.
--- bitbake/lib/bb/build.py | 3 +++ bitbake/lib/bb/runqueue.py | 27 ++++++++++++++++++++++++--- 2 files changed, 27 insertions(+), 3 deletions(-)
bitbake/lib/bb/build.py +3 -0
@@ -72,6 +72,9 @@ class TaskBase(event.Event): def __init__(self, t, d ): self._task = t self._package = d.getVar("PF", True) + self._file = d.getVar("FILE", True) + self.taskfile = self._file + self.taskname = self._task event.Event.__init__(self) self._message = "recipe %s: task %s: %s" % (d.getVar("PF", True), t, self.getDisplayName())
bitbake/lib/bb/runqueue.py +24 -3
@@ -217,6 +217,12 @@ class RunQueueData: ret.extend([nam]) return ret + def get_task_name(self, task): + return self.runq_task[task] + + def get_task_file(self, task): + return self.taskData.fn_index[self.runq_fnid[task]] + def get_user_idstring(self, task, task_name_suffix = ""): fn = self.taskData.fn_index[self.runq_fnid[task]] taskname = self.runq_task[task] + task_name_suffix
@@ -1328,9 +1334,10 @@ class RunQueueExecuteTasks(RunQueueExecute): if self.rqdata.taskData.abort: self.rq.state = runQueueCleanUp - def task_skip(self, task): + def task_skip(self, task, skip_reason = ""): self.runq_running[task] = 1 self.runq_buildable[task] = 1 + bb.event.fire(runQueueTaskSkipped(task, self.stats, self.rq, skip_reason), self.cfgData) self.task_completeoutright(task) self.stats.taskCompleted() self.stats.taskSkipped()
@@ -1355,13 +1362,13 @@ class RunQueueExecuteTasks(RunQueueExecute): if task in self.rq.scenequeue_covered: logger.debug(2, "Setscene covered task %s (%s)", task, self.rqdata.get_user_idstring(task)) - self.task_skip(task) + self.task_skip(task, "covered") return True if self.rq.check_stamp_task(task, taskname, cache=self.stampcache): logger.debug(2, "Stamp current task %s (%s)", task, self.rqdata.get_user_idstring(task)) - self.task_skip(task) + self.task_skip(task, "existing") return True taskdep = self.rqdata.dataCache.task_deps[fn]
@@ -1783,6 +1790,8 @@ class runQueueEvent(bb.event.Event): def __init__(self, task, stats, rq): self.taskid = task self.taskstring = rq.rqdata.get_user_idstring(task) + self.taskname = rq.rqdata.get_task_name(task) + self.taskfile = rq.rqdata.get_task_file(task) self.stats = stats.copy() bb.event.Event.__init__(self)
@@ -1794,6 +1803,8 @@ class sceneQueueEvent(runQueueEvent): runQueueEvent.__init__(self, task, stats, rq) realtask = rq.rqdata.runq_setscene[task] self.taskstring = rq.rqdata.get_user_idstring(realtask, "_setscene") + self.taskname = rq.rqdata.get_task_name(realtask) + "_setscene" + self.taskfile = rq.rqdata.get_task_file(realtask) class runQueueTaskStarted(runQueueEvent): """
@@ -1837,6 +1848,16 @@ class sceneQueueTaskCompleted(sceneQueueEvent): Event notifing a setscene task completed """ +class runQueueTaskSkipped(runQueueEvent): + """ + Event notifing a task was skipped + """ + def __init__(self, task, stats, rq, skip_reason): + runQueueEvent.__init__(self, task, stats, rq) + self._file = rq.rqdata.get_task_file(task) + self._task = rq.rqdata.get_task_name(task) + self._skip = skip_reason + class runQueuePipe(): """ Abstraction for a pipe between a worker thread and the server
Signature · 1 line
1.8.1.2
PATCH 4/8 bitbake: cooker: add extra recipe information
In reply to Alex DAMIANFrom: Alexandru DAMIAN <[email protected]>
Adding in the extra recipe information when creating the dependency tree information.
This works in server-mode because the HOB extra_cache is already enabled. In normal mode, it will do nothing with no performance impact.
--- bitbake/lib/bb/cooker.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+)
bitbake/lib/bb/cooker.py +23 -0
@@ -439,6 +439,29 @@ class BBCooker: depend_tree["pn"][pn] = {} depend_tree["pn"][pn]["filename"] = fn depend_tree["pn"][pn]["version"] = version + + # This data is needed for webhob; + # it's here if the bitbake runs in server mode + try: + summary = self.recipecache.summary[fn] + lic = self.recipecache.license[fn] + section = self.recipecache.section[fn] + description = self.recipecache.description[fn] + homepage = self.recipecache.homepage[fn] + bugtracker = self.recipecache.bugtracker[fn] + inherits = self.recipecache.inherits.get(fn, None) + depend_tree["pn"][pn]["filename"] = fn + depend_tree["pn"][pn]["version"] = version + depend_tree["pn"][pn]["summary"] = summary + depend_tree["pn"][pn]["license"] = lic + depend_tree["pn"][pn]["section"] = section + depend_tree["pn"][pn]["description"] = description + depend_tree["pn"][pn]["inherits"] = inherits + depend_tree["pn"][pn]["homepage"] = homepage + depend_tree["pn"][pn]["bugtracker"] = bugtracker + except: + pass + for dep in rq.rqdata.runq_depends[task]: depfn = taskdata.fn_index[rq.rqdata.runq_fnid[dep]] deppn = self.recipecache.pkg_fn[depfn]
Signature · 1 line
1.8.1.2
PATCH 5/8 bitbake: build: add logfile path for all Task events
In reply to Alex DAMIANFrom: Alexandru DAMIAN <[email protected]>
We add the path to the logfile for all Task events, so that we can record the logfile locations during the run.
This is needed because the logfile symlinks will point only to the last build logs, and there is no other link between a certain build and the log file it produces.
--- bitbake/lib/bb/build.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-)
bitbake/lib/bb/build.py +10 -5
@@ -91,13 +91,18 @@ class TaskBase(event.Event): class TaskStarted(TaskBase): """Task execution started""" + def __init__(self, task, logfile, metadata): + self.logfile = logfile + super(TaskStarted, self).__init__(task, metadata) class TaskSucceeded(TaskBase): """Task execution completed""" + def __init__(self, task, logfile, metadata): + self.logfile = logfile + super(TaskSucceeded, self).__init__(task, metadata) class TaskFailed(TaskBase): """Task execution failed""" - def __init__(self, task, logfile, metadata, errprinted = False): self.logfile = logfile self.errprinted = errprinted
@@ -114,8 +119,8 @@ class TaskFailedSilent(TaskBase): return "Failed" class TaskInvalid(TaskBase): - - def __init__(self, task, metadata): + def __init__(self, task, logfile, metadata): + self.logfile = logfile super(TaskInvalid, self).__init__(task, metadata) self._message = "No such task '%s'" % task
@@ -419,7 +424,7 @@ def _exec_task(fn, task, d, quieterr): localdata.setVar('BB_LOGFILE', logfn) localdata.setVar('BB_RUNTASK', task) - event.fire(TaskStarted(task, localdata), localdata) + event.fire(TaskStarted(task, logfn, localdata), localdata) try: for func in (prefuncs or '').split(): exec_func(func, localdata)
@@ -456,7 +461,7 @@ def _exec_task(fn, task, d, quieterr): logger.debug(2, "Zero size logfn %s, removing", logfn) bb.utils.remove(logfn) bb.utils.remove(loglink) - event.fire(TaskSucceeded(task, localdata), localdata) + event.fire(TaskSucceeded(task, logfn, localdata), localdata) if not localdata.getVarFlag(task, 'nostamp') and not localdata.getVarFlag(task, 'selfstamp'): make_stamp(task, localdata)
Signature · 1 line
1.8.1.2
Re: [PATCH 5/8] bitbake: build: add logfile path for all Task events
Quoted 5 lines · Alex DAMIAN
From: Alexandru DAMIAN <[email protected]>
We add the path to the logfile for all Task events, so that we can record the logfile locations during the run.
Quoted 8 lines
This is needed because the logfile symlinks will point only to the last build logs, and there is no other link between a certain build and the log file it produces.
Signed-off-by Alexandru DAMIAN <[email protected]>--- bitbake/lib/bb/build.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-)
a) you might as well modify TaskBase intead of each Task
b) You didn't fix up all the original Task* references e.g. TaskInvalid.
I gave up and merged my own version of this.
Cheers,
Richard
PATCH 6/8 bitbake: cooker, command: add a command to return global data
In reply to Alex DAMIANFrom: Alexandru DAMIAN <[email protected]>
Adding the 'getDataDump' read-only command that will return a dump of the global data state.
This will be used by UI clients to get the build configuration.
--- bitbake/lib/bb/command.py | 8 ++++++++ bitbake/lib/bb/cooker.py | 15 +++++++++++++++ 2 files changed, 23 insertions(+)
bitbake/lib/bb/command.py +8 -0
@@ -145,6 +145,14 @@ class CommandsSync: """ command.cooker.shutdown(True) + def getDataDump(self, command, params): + """ + Returns a dump of the global state. Call with + variable flags to be retrieved as params. + """ + return command.cooker.getGlobalExpandedData(params) + getDataDump.readonly = True + def getVariable(self, command, params): """ Read the value of a variable from data
bitbake/lib/bb/cooker.py +15 -0
@@ -1134,6 +1134,21 @@ class BBCooker: self.configuration.server_register_idlecallback(buildTargetsIdle, rq) + + def getGlobalExpandedData(self, flags): + dump = {} + for k in self.data.keys(): + try: + v = self.data.getVar(k, True) + if not k.startswith("__") and not bool(self.data.getVarFlag(k, 'func')) and not isinstance(v, bb.data_smart.DataSmart): + dump[k] = { 'v' : v } + for d in flags: + dump[k][d] = self.data.getVarFlag(k, d) + except: + pass + return dump + + def generateNewImage(self, image, base_image, package_queue, timestamp, description): ''' Create a new image with a "require"/"inherit" base_image statement
Signature · 1 line
1.8.1.2
Re: [PATCH 6/8] bitbake: cooker, command: add a command to return global data
Quoted 18 lines · Alex DAMIAN
From: Alexandru DAMIAN <[email protected]>
Adding the 'getDataDump' read-only command that will return a dump of the global data state.
This will be used by UI clients to get the build configuration.
Signed-off-by Alexandru DAMIAN <[email protected]>--- bitbake/lib/bb/command.py | 8 ++++++++ bitbake/lib/bb/cooker.py | 15 +++++++++++++++ 2 files changed, 23 insertions(+)bitbake/lib/bb/command.py +0
-0
bitbake/lib/bb/command.py class CommandsSync: · line 153
Quoted 9 lines
bitbake/lib/bb/command.py +6
-0@@ -145,6 +145,14 @@ class CommandsSync: """ command.cooker.shutdown(True) + def getDataDump(self, command, params): + """ + Returns a dump of the global state. Call with + variable flags to be retrieved as params. + """ + return command.cooker.getGlobalExpandedData(params)
To be blunt, this *sucks* as an API call. "getDataDump" tells me nothing about what it does. "getGlobalExpandedData" doesn't do much better.
"getAllKeyFlags"?
and please pass in a list as param[0] so there is a:
flaglist = params[0]
in there. This means we can add other parameters if we ever need to and we don't have complete lock in to the API.
Cheers,
Richard
PATCH 7/8 bitbake: event: adding events to allow bbclasses to push custom info
In reply to Alex DAMIANFrom: Alexandru DAMIAN <[email protected]>
We add a new event that will be triggered by the package-building class, containing the information about the currently built package.
Adding an event that will passi information about the content of each package file that gets built.
Knotty UI will ignore these events by default.
Maybe a mechanism for using a single generic event is in order ?
--- bitbake/lib/bb/event.py | 16 ++++++++++++++++ bitbake/lib/bb/ui/knotty.py | 2 ++ 2 files changed, 18 insertions(+)
bitbake/lib/bb/event.py +16 -0
@@ -589,6 +589,22 @@ class PackageInfo(Event): Event.__init__(self) self._pkginfolist = pkginfolist +class SinglePackageInfo(Event): + """ + Single Package Information sent on emit_pkgdata + """ + def __init__(self, data): + Event.__init__(self) + self._data = data + +class PackageFileSizes(Event): + """ + Event that contains information about the file sizes contained in a package. + """ + def __init__(self, data): + Event.__init__(self) + self._data = data + class SanityCheck(Event): """ Event to issue sanity check
bitbake/lib/bb/ui/knotty.py +2 -0
@@ -477,6 +477,8 @@ def main(server, eventHandler, params, tf = TerminalFilter): # ignore if isinstance(event, (bb.event.BuildBase, + bb.event.PackageFileSizes, + bb.event.SinglePackageInfo, bb.event.StampUpdate, bb.event.ConfigParsed, bb.event.RecipeParsed,
Signature · 1 line
1.8.1.2
PATCH 8/8 bitbake: runqueue: add task hash to Queue events
In reply to Alex DAMIANFrom: Alexandru DAMIAN <[email protected]>
Adding the sstate-related hash for all runqueue and scenequeue tasks, as it's needed in the WebHob data.
--- bitbake/lib/bb/runqueue.py | 5 +++++ 1 file changed, 5 insertions(+)
bitbake/lib/bb/runqueue.py +5 -0
@@ -223,6 +223,9 @@ class RunQueueData: def get_task_file(self, task): return self.taskData.fn_index[self.runq_fnid[task]] + def get_task_hash(self, task): + return self.runq_hash[task] + def get_user_idstring(self, task, task_name_suffix = ""): fn = self.taskData.fn_index[self.runq_fnid[task]] taskname = self.runq_task[task] + task_name_suffix
@@ -1792,6 +1795,7 @@ class runQueueEvent(bb.event.Event): self.taskstring = rq.rqdata.get_user_idstring(task) self.taskname = rq.rqdata.get_task_name(task) self.taskfile = rq.rqdata.get_task_file(task) + self.taskhash = rq.rqdata.get_task_hash(task) self.stats = stats.copy() bb.event.Event.__init__(self)
@@ -1805,6 +1809,7 @@ class sceneQueueEvent(runQueueEvent): self.taskstring = rq.rqdata.get_user_idstring(realtask, "_setscene") self.taskname = rq.rqdata.get_task_name(realtask) + "_setscene" self.taskfile = rq.rqdata.get_task_file(realtask) + self.taskhash = rq.rqdata.get_task_hash(task) class runQueueTaskStarted(runQueueEvent): """
Signature · 1 line
1.8.1.2