Switch hob from buildFiles to buildTargets
cover Switch hob from buildFiles to buildTargets
The initial hob merge came with the caveat that it used buildFiles and therefore a build in a new builddir wouldn't have the required native tools to actually generate the rootfs.
The key patch in this series switches to using buildTargets by injecting the directory the generated image recipe is saved to into BBPATH and a BBFILES pattern then reparsing the files before building.
Further this series includes a bug fix or two and a patch that got lost in a branch somewhere.
The following changes since commit 5c8eeefc79455f058dda8f04cf4c12dc5418e00f:
cooker: only return *Found events if something was actually found (2011-07-14 15:12:40 +0100)
are available in the git repository at:
git://github.com/incandescant/bitbake hob
https://github.com/incandescant/bitbake/tree/hob
Joshua Lock (6):
ui/hob: replace the ugly static command map ui/crumbs/hobprefs: add missing import
command|cooker: Add reparseFiles command ui/crumbs/hobeventhandler: reparse files before running other
commands
ui/crumbs/tasklistmodel: fix saving recipes
ui/hob: switch from buildFile to buildTargets for custom image builds
lib/bb/command.py | 8 ++ lib/bb/cooker.py | 54 ++++++++----- lib/bb/ui/crumbs/configurator.py | 18 ++++ lib/bb/ui/crumbs/hobeventhandler.py | 147 +++++++++++++++++++++-------------- lib/bb/ui/crumbs/hobprefs.py | 1 + lib/bb/ui/crumbs/tasklistmodel.py | 3 +- lib/bb/ui/hob.py | 26 +++---- 7 files changed, 161 insertions(+), 96 deletions(-)
Signature · 1 line
1.7.6
PATCH 1/6 ui/hob: replace the ugly static command map
The command_map was never a good idea, what's implemented here is a fraction less ugly but a significant factor more readable and therefore easy to maintain.
The method implemented in this patch also has the advantage of not being static meaning we can determine the desired runCommand arguments dynamically at call time.
--- lib/bb/ui/crumbs/hobeventhandler.py | 63 ++++++++++++++++++++-------------- lib/bb/ui/hob.py | 2 +- 2 files changed, 38 insertions(+), 27 deletions(-)
lib/bb/ui/crumbs/hobeventhandler.py +37 -26
@@ -61,8 +61,11 @@ class HobHandler(gobject.GObject): gobject.TYPE_STRING)), } + (CFG_PATH_LOCAL, CFG_PATH_HOB, CFG_PATH_LAYERS, CFG_FILES_DISTRO, CFG_FILES_MACH, CFG_FILES_SDK, FILES_MATCH_CLASS, GENERATE_TGTS) = range(8) + def __init__(self, taskmodel, server): gobject.GObject.__init__(self) + self.current_command = None self.building = None self.gplv3_excluded = False
@@ -74,30 +77,40 @@ class HobHandler(gobject.GObject): self.model = taskmodel self.server = server - self.command_map = { - "findConfigFilePathLocal" : ("findConfigFilePath", ["hob.local.conf"], "findConfigFilePathHobLocal"), - "findConfigFilePathHobLocal" : ("findConfigFilePath", ["bblayers.conf"], "findConfigFilePathLayers"), - "findConfigFilePathLayers" : ("findConfigFiles", ["DISTRO"], "findConfigFilesDistro"), - "findConfigFilesDistro" : ("findConfigFiles", ["MACHINE"], "findConfigFilesMachine"), - "findConfigFilesMachine" : ("findConfigFiles", ["MACHINE-SDK"], "findConfigFilesSdkMachine"), - "findConfigFilesSdkMachine" : ("findFilesMatchingInDir", ["rootfs_", "classes"], "findFilesMatchingPackage"), - "findFilesMatchingPackage" : ("generateTargetsTree", ["classes/image.bbclass"], None), - "generateTargetsTree" : (None, [], None), - } - def run_next_command(self): - # FIXME: this is ugly and I *will* replace it - if self.current_command: - if not self.generating: - self.emit("generating-data") - self.generating = True - next_cmd = self.command_map[self.current_command] - command = next_cmd[0] - argument = next_cmd[1] - self.current_command = next_cmd[2] - args = [command] - args.extend(argument) - self.server.runCommand(args) + if self.current_command and not self.generating: + self.emit("generating-data") + self.generating = True + + if self.current_command == self.CFG_PATH_LOCAL: + self.current_command = self.CFG_PATH_HOB + self.server.runCommand(["findConfigFilePath", "hob.local.conf"]) + elif self.current_command == self.CFG_PATH_HOB: + self.current_command = self.CFG_PATH_LAYERS + self.server.runCommand(["findConfigFilePath", "bblayers.conf"]) + elif self.current_command == self.CFG_PATH_LAYERS: + self.current_command = self.CFG_FILES_DISTRO + self.server.runCommand(["findConfigFiles", "DISTRO"]) + elif self.current_command == self.CFG_FILES_DISTRO: + self.current_command = self.CFG_FILES_MACH + self.server.runCommand(["findConfigFiles", "MACHINE"]) + elif self.current_command == self.CFG_FILES_MACH: + self.current_command = self.CFG_FILES_SDK + self.server.runCommand(["findConfigFiles", "MACHINE-SDK"]) + elif self.current_command == self.CFG_FILES_SDK: + self.current_command = self.FILES_MATCH_CLASS + self.server.runCommand(["findFilesMatchingInDir", "rootfs_", "classes"]) + elif self.current_command == self.FILES_MATCH_CLASS: + self.current_command = self.GENERATE_TGTS + self.server.runCommand(["generateTargetsTree", "classes/image.bbclass"]) + elif self.current_command == self.GENERATE_TGTS: + if self.generating: + self.emit("data-generated") + self.generating = False + self.current_command = None + else: + # No command? + pass def handle_event(self, event, running_build, pbar): if not event:
@@ -107,8 +120,6 @@ class HobHandler(gobject.GObject): if self.building: running_build.handle_event(event) elif isinstance(event, bb.event.TargetsTreeGenerated): - self.emit("data-generated") - self.generating = False if event._model: self.model.populate(event._model) elif isinstance(event, bb.event.ConfigFilesFound):
@@ -186,7 +197,7 @@ class HobHandler(gobject.GObject): selected_packages, _ = self.model.get_selected_packages() self.emit("reload-triggered", img, " ".join(selected_packages)) self.server.runCommand(["reparseFiles"]) - self.current_command = "findConfigFilePathLayers" + self.current_command = self.CFG_PATH_LAYERS self.run_next_command() def set_bbthreads(self, threads):
lib/bb/ui/hob.py +1 -1
@@ -906,7 +906,7 @@ def main (server, eventHandler): try: # kick the while thing off - handler.current_command = "findConfigFilePathLocal" + handler.current_command = handler.CFG_PATH_LOCAL server.runCommand(["findConfigFilePath", "local.conf"]) except xmlrpclib.Fault: print("XMLRPC Fault getting commandline:\n %s" % x)
Signature · 1 line
1.7.6
PATCH 2/6 ui/crumbs/hobprefs: add missing import
In reply to Joshua Lockglib.idle_add is used so the glib module must be imported
Fixes [YOCTO #1248]
--- lib/bb/ui/crumbs/hobprefs.py | 1 + 1 files changed, 1 insertions(+), 0 deletions(-)
lib/bb/ui/crumbs/hobprefs.py +1 -0
@@ -19,6 +19,7 @@ # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. import gtk +import glib from bb.ui.crumbs.configurator import Configurator class HobPrefs(gtk.Dialog):
Signature · 1 line
1.7.6
PATCH 3/6 command|cooker: Add reparseFiles command
In reply to Joshua LockAdd command reparseFiles to reparse bb files in a running cooker instance.
Fixes [YOCTO #1249]
--- lib/bb/command.py | 8 +++++++ lib/bb/cooker.py | 54 +++++++++++++++++++++++++++++++++------------------- 2 files changed, 42 insertions(+), 20 deletions(-)
lib/bb/command.py +8 -0
@@ -311,6 +311,14 @@ class CommandsAsync: command.finishAsyncCommand() parseFiles.needcache = True + def reparseFiles(self, command, params): + """ + Reparse .bb files + """ + command.cooker.reparseFiles() + command.finishAsyncCommand() + reparseFiles.needcache = True + def compareRevisions(self, command, params): """ Parse the .bb files
lib/bb/cooker.py +34 -20
102 lines not on this page · raw message
Signature · 1 line
1.7.6
Re: [PATCH 3/6] command|cooker: Add reparseFiles command
Quoted 40 lines · Joshua Lock
Add command reparseFiles to reparse bb files in a running cooker instance.
Fixes [YOCTO #1249]
Signed-off-by Joshua Lock <[email protected]>--- lib/bb/command.py | 8 +++++++ lib/bb/cooker.py | 54 +++++++++++++++++++++++++++++++++------------------- 2 files changed, 42 insertions(+), 20 deletions(-)lib/bb/cooker.py +2
-15@@ -122,21 +122,8 @@ class BBCooker: logger.critical("Unable to import extra RecipeInfo '%s' from '%s': %s" % (cache_name, module_name, exc)) sys.exit("FATAL: Failed to import extra cache class '%s'." % cache_name) - self.configuration.data = bb.data.init() - - bb.data.inheritFromOS(self.configuration.data) - - try: - self.parseConfigurationFiles(self.configuration.prefile, - self.configuration.postfile) - except SyntaxError: - sys.exit(1) - except Exception: - logger.exception("Error parsing configuration files") - sys.exit(1) - - if not self.configuration.cmd: - self.configuration.cmd = bb.data.getVar("BB_DEFAULT_TASK", self.configuration.data, True) or "build" + self.configuration.data = None + self.loadConfigurationData() bbpkgs = bb.data.getVar('BBPKGS', self.configuration.data, True) if bbpkgs and len(self.configuration.pkgs_to_build) == 0:
lib/bb/cooker.py class BBCooker: · line 157
Quoted 8 lines
lib/bb/cooker.py +5
-0@@ -163,6 +150,26 @@ class BBCooker: self.parser = None + def loadConfigurationData(self): + self.configuration.data = bb.data.init() + + if not self.server_registration_cb: + bb.data.setVar("BB_WORKERCONTEXT", "1", self.configuration.data)
This looks like one of the last remaining differences between the bitbake in poky and the upstream. See:
but bottom line is this shouldn't be here.
lib/bb/cooker.py class BBCooker: · line 159
Quoted 1 line
lib/bb/cooker.py +1
-0+ bb.data.inheritFromOS(self.configuration.data)
and this line is the one that worries me since it pokes around the environment. If we do this later, we're likely using a different environment and I'm not happy this is going to give good results.
This actually ties into the terminal bbclass issues Chris has been proposing recently. I think we need to store off the original environment into a datastore and then leave it somewhere fixed we can recall it later for the likes of this function and the terminal class code.
Quoted 54 lines
lib/bb/cooker.py +18
-4+ try: + self.parseConfigurationFiles(self.configuration.prefile, + self.configuration.postfile) + except SyntaxError: + sys.exit(1) + except Exception: + logger.exception("Error parsing configuration files") + sys.exit(1) + + if not self.configuration.cmd: + self.configuration.cmd = bb.data.getVar("BB_DEFAULT_TASK", self.configuration.data, True) or "build" + def parseConfiguration(self):@@ -508,7 +515,7 @@ class BBCooker: bb.data.expandKeys(localdata) # Handle PREFERRED_PROVIDERS - for p in (bb.data.getVar('PREFERRED_PROVIDERS', localdata, 1) or "").split(): + for p in (bb.data.getVar('PREFERRED_PROVIDERS', localdata, True) or "").split(): try: (providee, provider) = p.split(':') except:@@ -999,8 +1006,8 @@ class BBCooker: self.server_registration_cb(buildTargetsIdle, rq) - def updateCache(self): - if self.state == state.running: + def updateCache(self, force=False): + if self.state == state.running and not force: return if self.state in (state.shutdown, state.stop):@@ -1010,6 +1017,8 @@ class BBCooker: if self.state != state.parsing: self.parseConfiguration () + if self.status: + del self.status self.status = bb.cache.CacheData(self.caches_array) ignore = bb.data.getVar("ASSUME_PROVIDED", self.configuration.data, 1) or ""@@ -1083,7 +1092,7 @@ class BBCooker: collectlog.debug(1, "collecting .bb files") - files = (data.getVar( "BBFILES", self.configuration.data, 1 ) or "").split() + files = (data.getVar( "BBFILES", self.configuration.data, True) or "").split() data.setVar("BBFILES", " ".join(files), self.configuration.data) # Sort files by priority
lib/bb/cooker.py class BBCooker: · line 1153
Quoted 6 lines
lib/bb/cooker.py +2
-1@@ -1140,7 +1149,8 @@ class BBCooker: base = os.path.basename(f).replace('.bbappend', '.bb') if not base in self.appendlist: self.appendlist[base] = [] - self.appendlist[base].append(f) + if f not in self.appendlist[base]: + self.appendlist[base].append(f)
append files only get appended once. Shouldn't that be a separate patch?
Otherwise the patch and the idea are ok, we just need to be careful to get this right...
Cheers,
Richard
Re: [PATCH 3/6] command|cooker: Add reparseFiles command
Quoted 53 lines · Richard Purdie
Quoted 40 lines · Joshua Lock
Add command reparseFiles to reparse bb files in a running cooker instance.
Fixes [YOCTO #1249]
Signed-off-by Joshua Lock <[email protected]>--- lib/bb/command.py | 8 +++++++ lib/bb/cooker.py | 54 +++++++++++++++++++++++++++++++++------------------- 2 files changed, 42 insertions(+), 20 deletions(-)lib/bb/cooker.py +2
-15@@ -122,21 +122,8 @@ class BBCooker: logger.critical("Unable to import extra RecipeInfo '%s' from '%s': %s" % (cache_name, module_name, exc)) sys.exit("FATAL: Failed to import extra cache class '%s'." % cache_name) - self.configuration.data = bb.data.init() - - bb.data.inheritFromOS(self.configuration.data) - - try: - self.parseConfigurationFiles(self.configuration.prefile, - self.configuration.postfile) - except SyntaxError: - sys.exit(1) - except Exception: - logger.exception("Error parsing configuration files") - sys.exit(1) - - if not self.configuration.cmd: - self.configuration.cmd = bb.data.getVar("BB_DEFAULT_TASK", self.configuration.data, True) or "build" + self.configuration.data = None + self.loadConfigurationData() bbpkgs = bb.data.getVar('BBPKGS', self.configuration.data, True) if bbpkgs and len(self.configuration.pkgs_to_build) == 0:lib/bb/cooker.py class BBCooker: · line 157
Quoted 8 lines
lib/bb/cooker.py +5
-0@@ -163,6 +150,26 @@ class BBCooker: self.parser = None + def loadConfigurationData(self): + self.configuration.data = bb.data.init() + + if not self.server_registration_cb: + bb.data.setVar("BB_WORKERCONTEXT", "1", self.configuration.data)This looks like one of the last remaining differences between the bitbake in poky and the upstream. See:
Quoted 3 lines
but bottom line is this shouldn't be here.
Whoops.
Quoted 7 lines
lib/bb/cooker.py class BBCooker: · line 159
Quoted 2 lines
lib/bb/cooker.py +1
-0+ bb.data.inheritFromOS(self.configuration.data)and this line is the one that worries me since it pokes around the environment. If we do this later, we're likely using a different environment and I'm not happy this is going to give good results.
Quoted 5 lines
This actually ties into the terminal bbclass issues Chris has been proposing recently. I think we need to store off the original environment into a datastore and then leave it somewhere fixed we can recall it later for the likes of this function and the terminal class code.
Ah, that's a better idea. I'll take a stab at something to do that.
Quoted 53 lines
Quoted 53 lines
lib/bb/cooker.py +18
-4+ try: + self.parseConfigurationFiles(self.configuration.prefile, + self.configuration.postfile) + except SyntaxError: + sys.exit(1) + except Exception: + logger.exception("Error parsing configuration files") + sys.exit(1) + + if not self.configuration.cmd: + self.configuration.cmd = bb.data.getVar("BB_DEFAULT_TASK", self.configuration.data, True) or "build" + def parseConfiguration(self):@@ -508,7 +515,7 @@ class BBCooker: bb.data.expandKeys(localdata) # Handle PREFERRED_PROVIDERS - for p in (bb.data.getVar('PREFERRED_PROVIDERS', localdata, 1) or "").split(): + for p in (bb.data.getVar('PREFERRED_PROVIDERS', localdata, True) or "").split(): try: (providee, provider) = p.split(':') except:@@ -999,8 +1006,8 @@ class BBCooker: self.server_registration_cb(buildTargetsIdle, rq) - def updateCache(self): - if self.state == state.running: + def updateCache(self, force=False): + if self.state == state.running and not force: return if self.state in (state.shutdown, state.stop):@@ -1010,6 +1017,8 @@ class BBCooker: if self.state != state.parsing: self.parseConfiguration () + if self.status: + del self.status self.status = bb.cache.CacheData(self.caches_array) ignore = bb.data.getVar("ASSUME_PROVIDED", self.configuration.data, 1) or ""@@ -1083,7 +1092,7 @@ class BBCooker: collectlog.debug(1, "collecting .bb files") - files = (data.getVar( "BBFILES", self.configuration.data, 1 ) or "").split() + files = (data.getVar( "BBFILES", self.configuration.data, True) or "").split() data.setVar("BBFILES", " ".join(files), self.configuration.data)
Quoted 10 lines
lib/bb/cooker.py class BBCooker: · line 1153
Quoted 8 lines
lib/bb/cooker.py +2
-1# Sort files by priority@@ -1140,7 +1149,8 @@ class BBCooker: base = os.path.basename(f).replace('.bbappend', '.bb') if not base in self.appendlist: self.appendlist[base] = [] - self.appendlist[base].append(f) + if f not in self.appendlist[base]: + self.appendlist[base].append(f)append files only get appended once. Shouldn't that be a separate patch?
Quite possibly. I have a bad habit of spotting things and changing them when I should be thinking about something else. I'll split this change out next time. Must have missed it this try.
Quoted 3 lines
Otherwise the patch and the idea are ok, we just need to be careful to get this right...
Thanks for the detailed review. I'll work on a new revision to address your concerns.
Regards,
Joshua
Signature · 3 lines
Joshua Lock
Yocto Project "Johannes factotum"
Intel Open Source Technology CentrePATCH 4/6 ui/crumbs/hobeventhandler: reparse files before running other commands
In reply to Joshua LockIntegrate reparseFiles into the run_next_command() method rather than calling reparseFiles on the server and immediately calling other methods.
--- lib/bb/ui/crumbs/hobeventhandler.py | 8 +++++--- 1 files changed, 5 insertions(+), 3 deletions(-)
lib/bb/ui/crumbs/hobeventhandler.py +5 -3
@@ -61,7 +61,7 @@ class HobHandler(gobject.GObject): gobject.TYPE_STRING)), } - (CFG_PATH_LOCAL, CFG_PATH_HOB, CFG_PATH_LAYERS, CFG_FILES_DISTRO, CFG_FILES_MACH, CFG_FILES_SDK, FILES_MATCH_CLASS, GENERATE_TGTS) = range(8) + (CFG_PATH_LOCAL, CFG_PATH_HOB, CFG_PATH_LAYERS, CFG_FILES_DISTRO, CFG_FILES_MACH, CFG_FILES_SDK, FILES_MATCH_CLASS, GENERATE_TGTS, REPARSE_FILES) = range(9) def __init__(self, taskmodel, server): gobject.GObject.__init__(self)
@@ -108,6 +108,9 @@ class HobHandler(gobject.GObject): self.emit("data-generated") self.generating = False self.current_command = None + elif self.current_command == self.REPARSE_FILES: + self.current_command = self.CFG_PATH_LAYERS + self.server.runCommand(["reparseFiles"]) else: # No command? pass
@@ -196,8 +199,7 @@ class HobHandler(gobject.GObject): img = self.model.selected_image selected_packages, _ = self.model.get_selected_packages() self.emit("reload-triggered", img, " ".join(selected_packages)) - self.server.runCommand(["reparseFiles"]) - self.current_command = self.CFG_PATH_LAYERS + self.current_command = self.REPARSE_FILES self.run_next_command() def set_bbthreads(self, threads):
Signature · 1 line
1.7.6
PATCH 5/6 ui/crumbs/tasklistmodel: fix saving recipes
In reply to Joshua LockAfter switching to dynamically finding the relative path for the recipe file it's no longer to append .bb when inserting the require line into the saved recipe.
Fixes [YOCTO #1247]
--- lib/bb/ui/crumbs/tasklistmodel.py | 3 +-- 1 files changed, 1 insertions(+), 2 deletions(-)
lib/bb/ui/crumbs/tasklistmodel.py +1 -2
@@ -53,11 +53,10 @@ class BuildRep(gobject.GObject): self.userpkgs = packages def writeRecipe(self, writepath, model): - # FIXME: Need a better way to determine meta_path... template = """ # Recipe generated by the HOB -require %s.bb +require %s IMAGE_INSTALL += "%s" """
Signature · 1 line
1.7.6
PATCH 6/6 ui/hob: switch from buildFile to buildTargets for custom image builds
In reply to Joshua LockWe need to use the buildTargets command to ensure dependencies, such as native tools to build the rootfs, are correctly included. This patch achieves this by modifying BBPATH and BBFILES to include matches for the location of the generated recipe file and reparsing the metadata before calling buildTargets.
Fixes [YOCTO #1228]
--- lib/bb/ui/crumbs/configurator.py | 18 +++++++ lib/bb/ui/crumbs/hobeventhandler.py | 84 +++++++++++++++++++++-------------- lib/bb/ui/hob.py | 24 ++++------ 3 files changed, 78 insertions(+), 48 deletions(-)
lib/bb/ui/crumbs/configurator.py +18 -0
@@ -240,6 +240,24 @@ class Configurator(gobject.GObject): del self.orig_config self.orig_config = copy.deepcopy(self.config) + def insertTempBBPath(self, bbpath, bbfiles): + # Create a backup of the local.conf + bkup = "%s~" % self.local + os.rename(self.local, bkup) + + # read the original conf into a list + with open(bkup, 'r') as config: + config_lines = config.readlines() + + if bbpath: + config_lines.append("BBPATH := \"${BBPATH}:%s\"\n" % bbpath) + if bbfiles: + config_lines.append("BBFILES := \"${BBFILES} %s\"\n" % bbfiles) + + # Write the updated lines list object to the local.conf + with open(self.local, "w") as n: + n.write("".join(config_lines)) + def writeLayerConf(self): # If we've not added/removed new layers don't write if not self._isLayerConfDirty():
lib/bb/ui/crumbs/hobeventhandler.py +50 -34
123 lines not on this page · raw message
lib/bb/ui/hob.py +10 -14
@@ -64,10 +64,8 @@ class MainWindow (gtk.Window): self.build = RunningBuild() self.build.connect("build-failed", self.running_build_failed_cb) - self.build.connect("build-complete", self.handler.build_complete_cb) self.build.connect("build-started", self.build_started_cb) - - self.handler.connect("build-complete", self.build_complete_cb) + self.build.connect("build-complete", self.build_complete_cb) vbox = gtk.VBox(False, 0) vbox.set_border_width(0)
@@ -371,16 +369,15 @@ class MainWindow (gtk.Window): dialog.destroy() if response == gtk.RESPONSE_CANCEL: return - else: - # TODO: show a confirmation dialog ? - if not self.save_path: - import tempfile, datetime - image_name = "hob-%s-variant-%s.bb" % (rep.base_image, datetime.date.today().isoformat()) - image_dir = os.path.join(tempfile.gettempdir(), 'hob-images') - bb.utils.mkdirhier(image_dir) - recipepath = os.path.join(image_dir, image_name) else: - recipepath = self.save_path + self.handler.build_packages(rep.allpkgs.split(" ")) + else: + import tempfile, datetime + image_name = "hob-%s-variant-%s" % (rep.base_image, datetime.date.today().isoformat()) + image_file = "%s.bb" % (image_name) + image_dir = os.path.join(tempfile.gettempdir(), 'hob-images') + bb.utils.mkdirhier(image_dir) + recipepath = os.path.join(image_dir, image_file) rep.writeRecipe(recipepath, self.model) # In the case where we saved the file for the purpose of building
@@ -389,9 +386,8 @@ class MainWindow (gtk.Window): if not self.save_path: self.files_to_clean.append(recipepath) - self.handler.queue_image_recipe_path(recipepath) + self.handler.build_image(image_name, image_dir, self.configurator) - self.handler.build_packages(rep.allpkgs.split(" ")) self.nb.set_current_page(1) def back_button_clicked_cb(self, button):
Signature · 1 line
1.7.6