Layer tooling improvements
cover Layer tooling improvements
The following patches add dependencies between layers, and add the ability to query "overlayed" recipes. The latter functionality will hopefully be utilised by future changes for "flattening" multi-layer setups also. Please review.
The changes are available (against Poky, but apply cleanly against bitbake master with -p2) at:
git://git.pokylinux.org/poky-contrib paule/layertooling-1
http://git.pokylinux.org/cgit.cgi/poky-contrib/log/?h=paule/layertooling-1
Paul Eggleton (3):
bitbake: track 'overlayed' recipes
bitbake-layers: add show_overlayed action
bitbake/cooker: implement layer dependencies, make priority optional
bitbake/bin/bitbake-layers | 10 +++++ bitbake/lib/bb/cooker.py | 92 +++++++++++++++++++++++++++++++++++++++---- 2 files changed, 93 insertions(+), 9 deletions(-)
Signature · 1 line
1.7.4.1
RFC PATCH 1/3 bitbake: track 'overlayed' recipes
Recipes that have been 'overlayed' (where there is a recipe in another layer where that layer has a higher priority) are now listed within cooker.overlayedlist for use in bitbake-layers. This is a dict with keys of the topmost (highest priority) recipe file.
--- bitbake/lib/bb/cooker.py | 13 +++++++++++++ 1 files changed, 13 insertions(+), 0 deletions(-)
bitbake/lib/bb/cooker.py +13 -0
@@ -33,6 +33,7 @@ import threading from cStringIO import StringIO from contextlib import closing from functools import wraps +from collections import defaultdict import bb, bb.exceptions from bb import utils, data, parse, event, cache, providers, taskdata, command, runqueue
@@ -1053,6 +1054,18 @@ class BBCooker: self.appendlist[base] = [] self.appendlist[base].append(f) + # Find overlayed recipes + # bbfiles will be in priority order which makes this easy + bbfile_seen = dict() + self.overlayedlist = defaultdict(list) + for f in reversed(bbfiles): + base = os.path.basename(f) + if base not in bbfile_seen: + bbfile_seen[base] = f + else: + topfile = bbfile_seen[base] + self.overlayedlist[topfile].append(f) + return (bbfiles, masked) def get_file_appends(self, fn):
Signature · 1 line
1.7.4.1
RFC PATCH 2/3 bitbake-layers: add show_overlayed action
In reply to Paul EggletonAdd a show_overlayed action to list overlayed recipes.
--- bitbake/bin/bitbake-layers | 10 ++++++++++ 1 files changed, 10 insertions(+), 0 deletions(-)
bitbake/bin/bitbake-layers +10 -0
@@ -73,6 +73,16 @@ class Commands(cmd.Cmd): def do_show_layers(self, args): logger.info(str(self.config_data.getVar('BBLAYERS', True))) + def do_show_overlayed(self, args): + if self.cooker.overlayedlist: + logger.info('Overlayed recipes:') + for f in self.cooker.overlayedlist.iterkeys(): + logger.info('%s' % f) + for of in self.cooker.overlayedlist[f]: + logger.info(' %s' % of) + else: + logger.info('No overlayed recipes found') + def do_show_appends(self, args): if not self.cooker_data.appends: logger.info('No append files found')
Signature · 1 line
1.7.4.1
RFC PATCH 3/3 bitbake/cooker: implement layer dependencies, make priority optional
In reply to Paul EggletonImplement (optionally versioned) dependencies between layers, and if layer priorities are not specified using BBFILE_PRIORITY_layername (now optional) then work out the layer priority based on dependencies.
Define LAYERDEPENDS_layername in layer.conf to specify the dependencies of a layer (list of layer names, split with spaces in the usual way); LAYERVERSION_layername can be defined for each layer allowing specific version dependencies to be specified via depname:version in the list of dependencies. An error will be produced if any dependency is missing or the version numbers do not match exactly (if specified).
Note: default priority if unspecified for a layer with no dependencies is lowest defined priority + 1 (or 1 if no priorities are defined).
Addresses [YOCTO #790].
--- bitbake/lib/bb/cooker.py | 79 ++++++++++++++++++++++++++++++++++++++++----- 1 files changed, 70 insertions(+), 9 deletions(-)
bitbake/lib/bb/cooker.py +70 -9
@@ -702,26 +702,87 @@ class BBCooker: """Handle collections""" self.status.bbfile_config_priorities = [] if collections: + collection_priorities = {} + collection_depends = {} collection_list = collections.split() + min_prio = 0 for c in collection_list: + # Get collection priority if defined explicitly + priority = bb.data.getVar("BBFILE_PRIORITY_%s" % c, self.configuration.data, 1) + if priority: + try: + prio = int(priority) + except ValueError: + parselog.error("invalid value for BBFILE_PRIORITY_%s: \"%s\"", c, priority) + if min_prio == 0 or prio < min_prio: + min_prio = prio + collection_priorities[c] = prio + else: + collection_priorities[c] = None + + # Check dependencies and store information for priority calculation + deps = bb.data.getVar("LAYERDEPENDS_%s" % c, self.configuration.data, 1) + if deps: + depnamelist = [] + deplist = deps.split() + for dep in deplist: + depsplit = dep.split(':') + if len(depsplit) > 1: + try: + depver = int(depsplit[1]) + except ValueError: + parselog.error("invalid version value in LAYERDEPENDS_%s: \"%s\"", c, dep) + continue + else: + depver = None + dep = depsplit[0] + depnamelist.append(dep) + + if dep in collection_list: + if depver: + layerver = bb.data.getVar("LAYERVERSION_%s" % dep, self.configuration.data, 1) + if layerver: + try: + lver = int(layerver) + except ValueError: + parselog.error("invalid value for LAYERVERSION_%s: \"%s\"", c, layerver) + continue + if lver <> depver: + parselog.error("Layer dependency %s of layer %s is at version %d, expected %d", dep, c, lver, depver) + else: + parselog.error("Layer dependency %s of layer %s has no version, expected %d", dep, c, depver) + else: + parselog.error("Layer dependency %s of layer %s not found", dep, c) + collection_depends[c] = depnamelist + else: + collection_depends[c] = [] + + # Recursively work out collection priorities based on dependencies + def calc_layer_priority(collection): + if not collection_priorities[collection]: + max_depprio = min_prio + for dep in collection_depends[collection]: + calc_layer_priority(dep) + depprio = collection_priorities[dep] + if depprio > max_depprio: + max_depprio = depprio + max_depprio += 1 + parselog.debug(1, "Calculated priority of layer %s as %d", collection, max_depprio) + collection_priorities[collection] = max_depprio + + # Calculate all layer priorities using calc_layer_priority and store in bbfile_config_priorities + for c in collection_list: + calc_layer_priority(c) regex = bb.data.getVar("BBFILE_PATTERN_%s" % c, self.configuration.data, 1) if regex == None: parselog.error("BBFILE_PATTERN_%s not defined" % c) continue - priority = bb.data.getVar("BBFILE_PRIORITY_%s" % c, self.configuration.data, 1) - if priority == None: - parselog.error("BBFILE_PRIORITY_%s not defined" % c) - continue try: cre = re.compile(regex) except re.error: parselog.error("BBFILE_PATTERN_%s \"%s\" is not a valid regular expression", c, regex) continue - try: - pri = int(priority) - self.status.bbfile_config_priorities.append((c, regex, cre, pri)) - except ValueError: - parselog.error("invalid value for BBFILE_PRIORITY_%s: \"%s\"", c, priority) + self.status.bbfile_config_priorities.append((c, regex, cre, collection_priorities[c])) def buildSetVars(self): """
Signature · 1 line
1.7.4.1
Re: [RFC PATCH 1/3] bitbake: track 'overlayed' recipes
In reply to Paul EggletonQuoted 6 lines · Paul Eggleton
Recipes that have been 'overlayed' (where there is a recipe in another layer where that layer has a higher priority) are now listed within cooker.overlayedlist for use in bitbake-layers. This is a dict with keys of the topmost (highest priority) recipe file.
Signed-off-by Paul Eggleton <[email protected]>
Seems sane conceptually, but can we name this something else? overlayedlist is pretty weak, particularly being it's not a list at all :) I'd say just overlayed, or overlayed_recipes, or similar.
Signature · 5 lines
Christopher Larson clarson at kergoth dot com Founder - BitBake, OpenEmbedded, OpenZaurus Maintainer - Tslib Senior Software Engineer, Mentor Graphics
Re: [RFC PATCH 1/3] bitbake: track 'overlayed' recipes
Quoted 3 lines · On Friday 24 June 2011 19:15:21 Chris Larson wrote:
Seems sane conceptually, but can we name this something else? overlayedlist is pretty weak, particularly being it's not a list at all :) I'd say just overlayed, or
overlayed_recipes, or similar.
I've changed it to "overlayed" in the poky contrib branch. I can post v2 patches if desired.
Cheers,
Paul
Signature · 2 lines
Paul Eggleton Intel Open Source Technology Centre