devtool related changes, part 2
cover devtool related changes, part 2
The following changes since commit 18cd0ce6a55c9065c3f1bf223b47d817b5efcd8f:
data: escape '$' in shell variable assignment (2014-12-18 10:24:45 +0000)
are available in the git repository at:
git://git.yoctoproject.org/poky-contrib paule/devtool-bb2
http://git.yoctoproject.org/cgit.cgi/poky-contrib/log/?h=paule/devtool-bb2
Paul Eggleton (4):
siggen: ensure nostamp tasks force dependent tasks to re-execute
utils: add basic metadata manipulation functions
bitbake-layers: add commands for adding and removing layers
cooker: add ability to ignore unmatched regexes in BBFILE_PATTERN
bin/bitbake-layers | 54 +++++++++++++++++++++++ lib/bb/cooker.py | 7 +-- lib/bb/siggen.py | 6 +++ lib/bb/utils.py | 127 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 191 insertions(+), 3 deletions(-)
Signature · 1 line
1.9.3
PATCH 1/4 siggen: ensure nostamp tasks force dependent tasks to re-execute
If a nostamp task is depended on by a non-nostamp task, then we want the signature of that task to change such that it re-executes afterwards. This is an unusual situation, but we want this to work in OE in externalsrc.bbclass so that compilation happens every time it is requested.
--- lib/bb/siggen.py | 6 ++++++ 1 file changed, 6 insertions(+)
lib/bb/siggen.py +6 -0
@@ -187,6 +187,12 @@ class SignatureGeneratorBasic(SignatureGenerator): self.file_checksum_values[k][f] = cs data = data + cs + taskdep = dataCache.task_deps[fn] + if 'nostamp' in taskdep and task in taskdep['nostamp']: + # Nostamp tasks need an implicit taint so that they force any dependent tasks to run + import uuid + data = data + str(uuid.uuid4()) + taint = self.read_taint(fn, task, dataCache.stamp[fn]) if taint: data = data + taint
Signature · 1 line
1.9.3
PATCH 2/4 utils: add basic metadata manipulation functions
In reply to Paul Eggleton* Add a generic edit_metadata_file() function to modify variable assignments in any metadata file (conf, bb, bbappend) using a callback for flexibility
* Add a specific edit_bblayers_conf() function to modify conf/bblayers.conf and add and/or remove layers from the BBLAYERS value within it.
--- lib/bb/utils.py | 127 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 127 insertions(+)
lib/bb/utils.py +127 -0
@@ -915,3 +915,130 @@ def exec_flat_python_func(func, *args, **kwargs): comp = bb.utils.better_compile(code, '<string>', '<string>') bb.utils.better_exec(comp, context, code, '<string>') return context['retval'] + +def edit_metadata_file(meta_file, variables, func): + """Edit a recipe or config file and modify one or more specified + variable values set in the file using a specified callback function. + The file is only written to if the value(s) actually change. + """ + var_res = {} + for var in variables: + var_res[var] = re.compile(r'^%s[ \t]*[?=]+' % var) + + updated = False + varset_start = '' + newlines = [] + in_var = None + full_value = '' + + def handle_var_end(): + (newvalue, indent, minbreak) = func(in_var, full_value) + if newvalue != full_value: + if isinstance(newvalue, list): + intentspc = ' ' * indent + if minbreak: + # First item on first line + if len(newvalue) == 1: + newlines.append('%s "%s"\n' % (varset_start, newvalue[0])) + else: + newlines.append('%s "%s\\\n' % (varset_start, newvalue[0])) + for item in newvalue[1:]: + newlines.append('%s%s \\\n' % (intentspc, item)) + newlines.append('%s"\n' % indentspc) + else: + # No item on first line + newlines.append('%s " \\\n' % varset_start) + for item in newvalue: + newlines.append('%s%s \\\n' % (intentspc, item)) + newlines.append('%s"\n' % intentspc) + else: + newlines.append('%s "%s"\n' % (varset_start, newvalue)) + return True + return False + + with open(meta_file, 'r') as f: + for line in f: + if in_var: + value = line.rstrip() + full_value += value[:-1] + if value.endswith('"') or value.endswith("'"): + if handle_var_end(): + updated = True + in_var = None + else: + matched = False + for (varname, var_re) in var_res.iteritems(): + if var_re.match(line): + splitvalue = line.split('"', 1) + varset_start = splitvalue[0].rstrip() + value = splitvalue[1].rstrip() + if value.endswith('\\'): + value = value[:-1] + full_value = value + if value.endswith('"') or value.endswith("'"): + if handle_var_end(): + updated = True + else: + in_var = varname + matched = True + break + if not matched: + newlines.append(line) + if updated: + with open(meta_file, 'w') as f: + f.writelines(newlines) + +def edit_bblayers_conf(bblayers_conf, add, remove): + """Edit bblayers.conf, adding and/or removing layers""" + + import fnmatch + + def remove_trailing_sep(pth): + if pth and pth[-1] == os.sep: + pth = pth[:-1] + return pth + + def layerlist_param(value): + if not value: + return [] + elif isinstance(value, list): + return [remove_trailing_sep(x) for x in value] + else: + return [remove_trailing_sep(value)] + + notadded = [] + notremoved = [] + + addlayers = layerlist_param(add) + removelayers = layerlist_param(remove) + + def handle_bblayers(varname, origvalue): + updated = False + bblayers = [remove_trailing_sep(x) for x in origvalue.split()] + if removelayers: + for removelayer in removelayers: + matched = False + for layer in bblayers: + if fnmatch.fnmatch(layer, removelayer): + updated = True + matched = True + bblayers.remove(layer) + break + if not matched: + notremoved.append(removelayer) + if addlayers: + for addlayer in addlayers: + if addlayer not in bblayers: + updated = True + bblayers.append(addlayer) + else: + notadded.append(addlayer) + + if updated: + return (bblayers, 2, False) + else: + return (origvalue, 2, False) + + edit_metadata_file(bblayers_conf, ['BBLAYERS'], handle_bblayers) + return (notadded, notremoved) +
Signature · 1 line
1.9.3
PATCH 3/4 bitbake-layers: add commands for adding and removing layers
In reply to Paul EggletonAdd add-layer and remove-layer commands for easily adding and removing layers to/from bblayers.conf.
--- bin/bitbake-layers | 54 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+)
bin/bitbake-layers +54 -0
@@ -103,6 +103,60 @@ class Commands(cmd.Cmd): logger.plain("%s %s %d" % (layername.ljust(20), layerdir.ljust(40), layerpri)) + def do_add_layer(self, dirname): + """Add a layer to bblayers.conf + +usage: add-layer <layerdir> +""" + if not dirname: + sys.stderr.write("Please specify the layer directory to add\n") + return + + layerdir = os.path.abspath(dirname) + if not os.path.exists(layerdir): + sys.stderr.write("Specified layer directory doesn't exist\n") + return + + layer_conf = os.path.join(layerdir, 'conf', 'layer.conf') + if not os.path.exists(layer_conf): + sys.stderr.write("Specified layer directory doesn't contain a conf/layer.conf file\n") + return + + bblayers_conf = os.path.join('conf', 'bblayers.conf') + if not os.path.exists(bblayers_conf): + sys.stderr.write("Unable to find bblayers.conf\n") + return + + (notadded, _) = bb.utils.edit_bblayers_conf(bblayers_conf, layerdir, None) + if notadded: + for item in notadded: + sys.stderr.write("Specified layer %s not in BBLAYERS\n" % item) + + + def do_remove_layer(self, dirname): + """Remove a layer from bblayers.conf + +usage: remove-layer <layerdir> +""" + if not dirname: + sys.stderr.write("Please specify the layer directory to remove\n") + return + + bblayers_conf = os.path.join('conf', 'bblayers.conf') + if not os.path.exists(bblayers_conf): + sys.stderr.write("Unable to find bblayers.conf\n") + return + + if dirname.startswith('*'): + layerdir = dirname + else: + layerdir = os.path.abspath(dirname) + (_, notremoved) = bb.utils.edit_bblayers_conf(bblayers_conf, None, layerdir) + if notremoved: + for item in notremoved: + sys.stderr.write("No layers matching %s found in BBLAYERS\n" % item) + + def version_str(self, pe, pv, pr = None): verstr = "%s" % pv if pr:
Signature · 1 line
1.9.3
PATCH 4/4 cooker: add ability to ignore unmatched regexes in BBFILE_PATTERN
In reply to Paul EggletonAdd a BBFILE_PATTERN_IGNORE_EMPTY variable to allow ignoring the fact that a regex specified in BBFILE_PATTERN for a particular collection doesn't match any recipes. This will be used in OpenEmbedded in the workspace layers created by "devtool" which may not always contain any recipes (which is not cause for warning the user).
--- lib/bb/cooker.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-)
lib/bb/cooker.py +4 -3
@@ -1385,7 +1385,7 @@ class BBCooker: raise bb.BBHandledException() self.show_appends_with_no_recipes() self.handlePrefProviders() - self.recipecache.bbfile_priority = self.collection.collection_priorities(self.recipecache.pkg_fn) + self.recipecache.bbfile_priority = self.collection.collection_priorities(self.recipecache.pkg_fn, self.data) self.state = state.running return None
@@ -1610,7 +1610,7 @@ class CookerCollectFiles(object): filelist.append(filename) return filelist - def collection_priorities(self, pkgfns): + def collection_priorities(self, pkgfns, d): priorities = {}
@@ -1639,7 +1639,8 @@ class CookerCollectFiles(object): for collection, pattern, regex, _ in self.bbfile_config_priorities: if regex in unmatched: - collectlog.warn("No bb files matched BBFILE_PATTERN_%s '%s'" % (collection, pattern)) + if d.getVar('BBFILE_PATTERN_IGNORE_EMPTY_%s' % collection, True) != '1': + collectlog.warn("No bb files matched BBFILE_PATTERN_%s '%s'" % (collection, pattern)) return priorities
Signature · 1 line
1.9.3