Skip to contentSkip to conversation

devtool related changes, part 2

PATCH New Paul Eggleton 1 participant 5 messages Dec 19, 2014

lore
Share & tools
lore
#0 lore raw

cover devtool related changes, part 2

The following changes since commit 18cd0ce6a55c9065c3f1bf223b47d817b5efcd8f:

data: escape '$' in shell variable assignment (2014-12-18 10:24:45 +0000)

are available in the git repository at:

git://git.yoctoproject.org/poky-contrib paule/devtool-bb2
http://git.yoctoproject.org/cgit.cgi/poky-contrib/log/?h=paule/devtool-bb2

Paul Eggleton (4):
siggen: ensure nostamp tasks force dependent tasks to re-execute
utils: add basic metadata manipulation functions
bitbake-layers: add commands for adding and removing layers
cooker: add ability to ignore unmatched regexes in BBFILE_PATTERN

 bin/bitbake-layers |  54 +++++++++++++++++++++++
 lib/bb/cooker.py   |   7 +--
 lib/bb/siggen.py   |   6 +++
 lib/bb/utils.py    | 127 +++++++++++++++++++++++++++++++++++++++++++++++++++++
 4 files changed, 191 insertions(+), 3 deletions(-)
Signature · 1 line
1.9.3
#1 lore raw

PATCH 1/4 siggen: ensure nostamp tasks force dependent tasks to re-execute

If a nostamp task is depended on by a non-nostamp task, then we want the signature of that task to change such that it re-executes afterwards. This is an unusual situation, but we want this to work in OE in externalsrc.bbclass so that compilation happens every time it is requested.

---
 lib/bb/siggen.py | 6 ++++++
 1 file changed, 6 insertions(+)
lib/bb/siggen.py +6 -0
index 5103073..0c77d72 100644
@@ -187,6 +187,12 @@ class SignatureGeneratorBasic(SignatureGenerator):
                 self.file_checksum_values[k][f] = cs
                 data = data + cs
 
+        taskdep = dataCache.task_deps[fn]
+        if 'nostamp' in taskdep and task in taskdep['nostamp']:
+            # Nostamp tasks need an implicit taint so that they force any dependent tasks to run
+            import uuid
+            data = data + str(uuid.uuid4())
+
         taint = self.read_taint(fn, task, dataCache.stamp[fn])
         if taint:
             data = data + taint
Signature · 1 line
1.9.3
#2 lore raw

PATCH 2/4 utils: add basic metadata manipulation functions

In reply to Paul Eggleton

* Add a generic edit_metadata_file() function to modify variable assignments in any metadata file (conf, bb, bbappend) using a callback for flexibility
* Add a specific edit_bblayers_conf() function to modify conf/bblayers.conf and add and/or remove layers from the BBLAYERS value within it.

---
 lib/bb/utils.py | 127 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 127 insertions(+)
lib/bb/utils.py +127 -0
index d7c5067..f26349f 100644
@@ -915,3 +915,130 @@ def exec_flat_python_func(func, *args, **kwargs):
     comp = bb.utils.better_compile(code, '<string>', '<string>')
     bb.utils.better_exec(comp, context, code, '<string>')
     return context['retval']
+
+def edit_metadata_file(meta_file, variables, func):
+    """Edit a recipe or config file and modify one or more specified
+    variable values set in the file using a specified callback function.
+    The file is only written to if the value(s) actually change.
+    """
+    var_res = {}
+    for var in variables:
+        var_res[var] = re.compile(r'^%s[ \t]*[?=]+' % var)
+
+    updated = False
+    varset_start = ''
+    newlines = []
+    in_var = None
+    full_value = ''
+
+    def handle_var_end():
+        (newvalue, indent, minbreak) = func(in_var, full_value)
+        if newvalue != full_value:
+            if isinstance(newvalue, list):
+                intentspc = ' ' * indent
+                if minbreak:
+                    # First item on first line
+                    if len(newvalue) == 1:
+                        newlines.append('%s "%s"\n' % (varset_start, newvalue[0]))
+                    else:
+                        newlines.append('%s "%s\\\n' % (varset_start, newvalue[0]))
+                        for item in newvalue[1:]:
+                            newlines.append('%s%s \\\n' % (intentspc, item))
+                        newlines.append('%s"\n' % indentspc)
+                else:
+                    # No item on first line
+                    newlines.append('%s " \\\n' % varset_start)
+                    for item in newvalue:
+                        newlines.append('%s%s \\\n' % (intentspc, item))
+                    newlines.append('%s"\n' % intentspc)
+            else:
+                newlines.append('%s "%s"\n' % (varset_start, newvalue))
+            return True
+        return False
+
+    with open(meta_file, 'r') as f:
+        for line in f:
+            if in_var:
+                value = line.rstrip()
+                full_value += value[:-1]
+                if value.endswith('"') or value.endswith("'"):
+                    if handle_var_end():
+                        updated = True
+                    in_var = None
+            else:
+                matched = False
+                for (varname, var_re) in var_res.iteritems():
+                    if var_re.match(line):
+                        splitvalue = line.split('"', 1)
+                        varset_start = splitvalue[0].rstrip()
+                        value = splitvalue[1].rstrip()
+                        if value.endswith('\\'):
+                            value = value[:-1]
+                        full_value = value
+                        if value.endswith('"') or value.endswith("'"):
+                            if handle_var_end():
+                                updated = True
+                        else:
+                            in_var = varname
+                        matched = True
+                        break
+                if not matched:
+                    newlines.append(line)
+    if updated:
+        with open(meta_file, 'w') as f:
+            f.writelines(newlines)
+
+def edit_bblayers_conf(bblayers_conf, add, remove):
+    """Edit bblayers.conf, adding and/or removing layers"""
+
+    import fnmatch
+
+    def remove_trailing_sep(pth):
+        if pth and pth[-1] == os.sep:
+            pth = pth[:-1]
+        return pth
+
+    def layerlist_param(value):
+        if not value:
+            return []
+        elif isinstance(value, list):
+            return [remove_trailing_sep(x) for x in value]
+        else:
+            return [remove_trailing_sep(value)]
+
+    notadded = []
+    notremoved = []
+
+    addlayers = layerlist_param(add)
+    removelayers = layerlist_param(remove)
+
+    def handle_bblayers(varname, origvalue):
+        updated = False
+        bblayers = [remove_trailing_sep(x) for x in origvalue.split()]
+        if removelayers:
+            for removelayer in removelayers:
+                matched = False
+                for layer in bblayers:
+                    if fnmatch.fnmatch(layer, removelayer):
+                        updated = True
+                        matched = True
+                        bblayers.remove(layer)
+                        break
+                if not matched:
+                    notremoved.append(removelayer)
+        if addlayers:
+            for addlayer in addlayers:
+                if addlayer not in bblayers:
+                    updated = True
+                    bblayers.append(addlayer)
+                else:
+                    notadded.append(addlayer)
+
+        if updated:
+            return (bblayers, 2, False)
+        else:
+            return (origvalue, 2, False)
+
+    edit_metadata_file(bblayers_conf, ['BBLAYERS'], handle_bblayers)
+    return (notadded, notremoved)
+
Signature · 1 line
1.9.3
#3 lore raw

PATCH 3/4 bitbake-layers: add commands for adding and removing layers

In reply to Paul Eggleton

Add add-layer and remove-layer commands for easily adding and removing layers to/from bblayers.conf.

---
 bin/bitbake-layers | 54 ++++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 54 insertions(+)
bin/bitbake-layers +54 -0
index 9964040..edb4d2f 100755
@@ -103,6 +103,60 @@ class Commands(cmd.Cmd):
             logger.plain("%s  %s  %d" % (layername.ljust(20), layerdir.ljust(40), layerpri))
 
 
+    def do_add_layer(self, dirname):
+        """Add a layer to bblayers.conf
+
+usage: add-layer <layerdir>
+"""
+        if not dirname:
+            sys.stderr.write("Please specify the layer directory to add\n")
+            return
+
+        layerdir = os.path.abspath(dirname)
+        if not os.path.exists(layerdir):
+            sys.stderr.write("Specified layer directory doesn't exist\n")
+            return
+
+        layer_conf = os.path.join(layerdir, 'conf', 'layer.conf')
+        if not os.path.exists(layer_conf):
+            sys.stderr.write("Specified layer directory doesn't contain a conf/layer.conf file\n")
+            return
+
+        bblayers_conf = os.path.join('conf', 'bblayers.conf')
+        if not os.path.exists(bblayers_conf):
+            sys.stderr.write("Unable to find bblayers.conf\n")
+            return
+
+        (notadded, _) = bb.utils.edit_bblayers_conf(bblayers_conf, layerdir, None)
+        if notadded:
+            for item in notadded:
+                sys.stderr.write("Specified layer %s not in BBLAYERS\n" % item)
+
+
+    def do_remove_layer(self, dirname):
+        """Remove a layer from bblayers.conf
+
+usage: remove-layer <layerdir>
+"""
+        if not dirname:
+            sys.stderr.write("Please specify the layer directory to remove\n")
+            return
+
+        bblayers_conf = os.path.join('conf', 'bblayers.conf')
+        if not os.path.exists(bblayers_conf):
+            sys.stderr.write("Unable to find bblayers.conf\n")
+            return
+
+        if dirname.startswith('*'):
+            layerdir = dirname
+        else:
+            layerdir = os.path.abspath(dirname)
+        (_, notremoved) = bb.utils.edit_bblayers_conf(bblayers_conf, None, layerdir)
+        if notremoved:
+            for item in notremoved:
+                sys.stderr.write("No layers matching %s found in BBLAYERS\n" % item)
+
+
     def version_str(self, pe, pv, pr = None):
         verstr = "%s" % pv
         if pr:
Signature · 1 line
1.9.3
#4 lore raw

PATCH 4/4 cooker: add ability to ignore unmatched regexes in BBFILE_PATTERN

In reply to Paul Eggleton

Add a BBFILE_PATTERN_IGNORE_EMPTY variable to allow ignoring the fact that a regex specified in BBFILE_PATTERN for a particular collection doesn't match any recipes. This will be used in OpenEmbedded in the workspace layers created by "devtool" which may not always contain any recipes (which is not cause for warning the user).

---
 lib/bb/cooker.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)
lib/bb/cooker.py +4 -3
index 16fd4ad..0d9b85e 100644
@@ -1385,7 +1385,7 @@ class BBCooker:
                 raise bb.BBHandledException()
             self.show_appends_with_no_recipes()
             self.handlePrefProviders()
-            self.recipecache.bbfile_priority = self.collection.collection_priorities(self.recipecache.pkg_fn)
+            self.recipecache.bbfile_priority = self.collection.collection_priorities(self.recipecache.pkg_fn, self.data)
             self.state = state.running
             return None
 
@@ -1610,7 +1610,7 @@ class CookerCollectFiles(object):
                     filelist.append(filename)
         return filelist
 
-    def collection_priorities(self, pkgfns):
+    def collection_priorities(self, pkgfns, d):
 
         priorities = {}
 
@@ -1639,7 +1639,8 @@ class CookerCollectFiles(object):
 
         for collection, pattern, regex, _ in self.bbfile_config_priorities:
             if regex in unmatched:
-                collectlog.warn("No bb files matched BBFILE_PATTERN_%s '%s'" % (collection, pattern))
+                if d.getVar('BBFILE_PATTERN_IGNORE_EMPTY_%s' % collection, True) != '1':
+                    collectlog.warn("No bb files matched BBFILE_PATTERN_%s '%s'" % (collection, pattern))
 
         return priorities
 
Signature · 1 line
1.9.3