Skip to contentSkip to conversation

Fixes for variable dependency handling with contains functions

PATCH New Paul Eggleton 1 participant 5 messages Apr 2, 2017

lore
Share & tools
lore
#0 lore raw

cover Fixes for variable dependency handling with contains functions

Fixes for a few issues I discovered while working on bitbake-diffsigs.

The following changes since commit 58a03531c8183b165bb7dcad86d8559c92bc150d:

fetch2: Do not fail to create symbolic links if they already exist (2017-04-01 08:23:29 +0100)

are available in the git repository at:

git://git.yoctoproject.org/poky-contrib paule/bb-contains
http://git.yoctoproject.org/cgit.cgi/poky-contrib/log/?h=paule/bb-contains

Paul Eggleton (4):
lib/bb/data: fix dependency handling for contains and multiple values
codeparser: improve handling of contains_any() and filter()
bitbake-selftest: add contains tests
codeparser: add some comments

 lib/bb/cache.py            |  2 +-
 lib/bb/codeparser.py       | 34 +++++++++++++++++++++++++++++---
 lib/bb/data.py             | 10 ++++++----
 lib/bb/tests/codeparser.py | 48 ++++++++++++++++++++++++++++++++++++++++++++++
 4 files changed, 86 insertions(+), 8 deletions(-)
Signature · 1 line
2.9.3
#1 lore raw

PATCH 1/4 lib/bb/data: fix dependency handling for contains and multiple values

The code that determines variable dependencies uses the codeparser to find references to "contains" type operations e.g. bb.utils.contains(). That function can take multiple items to check, and all specified items have to be present. However this code didn't handle that - it assumed that only one item would be specified and thus it was treating the multiple items as a single item with spaces in between. Split the value and check if all words are present in order to determine whether the check is "set" or "unset".

---
 lib/bb/data.py | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)
lib/bb/data.py +6 -4
index a85cb3a..0403754 100644
@@ -296,11 +296,13 @@ def build_dependencies(key, keys, shelldeps, varflagsexcl, d):
             newvalue = ""
             for k in sorted(contains):
                 l = (d.getVar(k) or "").split()
-                for word in sorted(contains[k]):
-                    if word in l:
-                        newvalue += "\n%s{%s} = Set" %  (k, word)
+                for item in sorted(contains[k]):
+                    for word in item.split():
+                        if not word in l:
+                            newvalue += "\n%s{%s} = Unset" % (k, item)
+                            break
                     else:
-                        newvalue += "\n%s{%s} = Unset" %  (k, word)
+                        newvalue += "\n%s{%s} = Set" % (k, item)
             if not newvalue:
                 return value
             if not value:
Signature · 1 line
2.9.3
#2 lore raw

PATCH 2/4 codeparser: improve handling of contains_any() and filter()

In reply to Paul Eggleton

Ensure we handle bb.utils.contains_any() as separate items, rather than how we handle contains() where every item must be in the list. Additionally, enable handling bb.utils.filter() which for the purposes of looking at dependencies is the same as contains_any().

Additionally bump the codeparser cache and recipe cache versions to invalidate the user's existing caches (ensuring that the changes take effect and avoiding "taskhash mismatch" errors respectively).

---
 lib/bb/cache.py      |  2 +-
 lib/bb/codeparser.py | 11 ++++++++---
 2 files changed, 9 insertions(+), 4 deletions(-)
lib/bb/cache.py +1 -1
index c04ac13..28e8a87 100644
@@ -37,7 +37,7 @@ import bb.utils
 
 logger = logging.getLogger("BitBake.Cache")
 
-__cache_version__ = "150"
+__cache_version__ = "151"
 
 def getCacheFile(path, filename, data_hash):
     return os.path.join(path, filename + "." + data_hash)
lib/bb/codeparser.py +8 -3
index 89d24ab..f76b478 100644
@@ -117,7 +117,7 @@ class shellCacheLine(object):
 
 class CodeParserCache(MultiProcessCache):
     cache_file_name = "bb_codeparser.dat"
-    CACHE_VERSION = 8
+    CACHE_VERSION = 9
 
     def __init__(self):
         MultiProcessCache.__init__(self)
@@ -193,7 +193,8 @@ class BufferedLogger(Logger):
 class PythonParser():
     getvars = (".getVar", ".appendVar", ".prependVar")
     getvarflags = (".getVarFlag", ".appendVarFlag", ".prependVarFlag")
-    containsfuncs = ("bb.utils.contains", "base_contains", "bb.utils.contains_any")
+    containsfuncs = ("bb.utils.contains", "base_contains")
+    containsanyfuncs = ("bb.utils.contains_any",  "bb.utils.filter")
     execfuncs = ("bb.build.exec_func", "bb.build.exec_task")
 
     def warn(self, func, arg):
@@ -212,13 +213,17 @@ class PythonParser():
 
     def visit_Call(self, node):
         name = self.called_node_name(node.func)
-        if name and (name.endswith(self.getvars) or name.endswith(self.getvarflags) or name in self.containsfuncs):
+        if name and (name.endswith(self.getvars) or name.endswith(self.getvarflags) or name in self.containsfuncs or name in self.containsanyfuncs):
             if isinstance(node.args[0], ast.Str):
                 varname = node.args[0].s
                 if name in self.containsfuncs and isinstance(node.args[1], ast.Str):
                     if varname not in self.contains:
                         self.contains[varname] = set()
                     self.contains[varname].add(node.args[1].s)
+                elif name in self.containsanyfuncs and isinstance(node.args[1], ast.Str):
+                    if varname not in self.contains:
+                        self.contains[varname] = set()
+                    self.contains[varname].update(node.args[1].s.split())
                 elif name.endswith(self.getvarflags):
                     if isinstance(node.args[1], ast.Str):
                         self.references.add('%s[%s]' % (varname, node.args[1].s))
Signature · 1 line
2.9.3
#3 lore raw

PATCH 3/4 bitbake-selftest: add contains tests

In reply to Paul Eggleton

Add some tests to verify that we are extracting "contains" information from python expressions in the code in the bb.data and bb.codeparser modules.

---
 lib/bb/tests/codeparser.py | 48 ++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 48 insertions(+)
lib/bb/tests/codeparser.py +48 -0
index a45a582..e30e78c 100644
@@ -49,6 +49,9 @@ class ReferenceTest(unittest.TestCase):
     def assertExecs(self, execs):
         self.assertEqual(self.execs, execs)
 
+    def assertContains(self, contains):
+        self.assertEqual(self.contains, contains)
+
 class VariableReferenceTest(ReferenceTest):
 
     def parseExpression(self, exp):
@@ -201,6 +204,7 @@ class PythonReferenceTest(ReferenceTest):
 
         self.references = parsedvar.references | parser.references
         self.execs = parser.execs
+        self.contains = parser.contains
 
     @staticmethod
     def indent(value):
@@ -265,6 +269,26 @@ be. These unit tests are testing snippets."""
         self.assertExecs(set(["testget"]))
         del self.context["testget"]
 
+    def test_contains(self):
+        self.parseExpression('bb.utils.contains("TESTVAR", "one", "true", "false", d)')
+        self.assertContains({'TESTVAR': {'one'}})
+
+    def test_contains_multi(self):
+        self.parseExpression('bb.utils.contains("TESTVAR", "one two", "true", "false", d)')
+        self.assertContains({'TESTVAR': {'one two'}})
+
+    def test_contains_any(self):
+        self.parseExpression('bb.utils.contains_any("TESTVAR", "hello", "true", "false", d)')
+        self.assertContains({'TESTVAR': {'hello'}})
+
+    def test_contains_any_multi(self):
+        self.parseExpression('bb.utils.contains_any("TESTVAR", "one two three", "true", "false", d)')
+        self.assertContains({'TESTVAR': {'one', 'two', 'three'}})
+
+    def test_contains_filter(self):
+        self.parseExpression('bb.utils.filter("TESTVAR", "hello there world", d)')
+        self.assertContains({'TESTVAR': {'hello', 'there', 'world'}})
+
 
 class DependencyReferenceTest(ReferenceTest):
 
@@ -370,6 +394,30 @@ esac
 
         self.assertEqual(deps, set(["oe_libinstall"]))
 
+    def test_contains_vardeps(self):
+        expr = '${@bb.utils.filter("TESTVAR", "somevalue anothervalue", d)} \
+                ${@bb.utils.contains("TESTVAR", "testval testval2", "yetanothervalue", "", d)} \
+                ${@bb.utils.contains("TESTVAR", "testval2 testval3", "blah", "", d)} \
+                ${@bb.utils.contains_any("TESTVAR", "testval2 testval3", "lastone", "", d)}'
+        parsedvar = self.d.expandWithRefs(expr, None)
+        # Check contains
+        self.assertEqual(parsedvar.contains, {'TESTVAR': {'testval2 testval3', 'anothervalue', 'somevalue', 'testval testval2', 'testval2', 'testval3'}})
+        # Check dependencies
+        self.d.setVar('ANOTHERVAR', expr)
+        self.d.setVar('TESTVAR', 'anothervalue testval testval2')
+        deps, values = bb.data.build_dependencies("ANOTHERVAR", set(self.d.keys()), set(), set(), self.d)
+        self.assertEqual(sorted(values.splitlines()),
+                         sorted([expr,
+                          'TESTVAR{anothervalue} = Set',
+                          'TESTVAR{somevalue} = Unset',
+                          'TESTVAR{testval testval2} = Set',
+                          'TESTVAR{testval2 testval3} = Unset',
+                          'TESTVAR{testval2} = Set',
+                          'TESTVAR{testval3} = Unset'
+                          ]))
+        # Check final value
+        self.assertEqual(self.d.getVar('ANOTHERVAR').split(), ['anothervalue', 'yetanothervalue', 'lastone'])
+
     #Currently no wildcard support
     #def test_vardeps_wildcards(self):
     #    self.d.setVar("oe_libinstall", "echo test")
Signature · 1 line
2.9.3
#4 lore raw

PATCH 4/4 codeparser: add some comments

In reply to Paul Eggleton

Add a few comments at the top of the file explaining what it's for, and a comment pointing out that you need to increment the cache version when changing any code that changes the output.

---
 lib/bb/codeparser.py | 23 +++++++++++++++++++++++
 1 file changed, 23 insertions(+)
lib/bb/codeparser.py +23 -0
index f76b478..530f44e 100644
@@ -1,3 +1,22 @@
+"""
+BitBake code parser
+
+Parses actual code (i.e. python and shell) for functions and in-line
+expressions. Used mainly to determine dependencies on other functions
+and variables within the BitBake metadata. Also provides a cache for
+this information in order to speed up processing.
+
+(Not to be confused with the code that parses the metadata itself,
+see lib/bb/parse/ for that).
+
+NOTE: if you change how the parsers gather information you will almost
+certainly need to increment CodeParserCache.CACHE_VERSION below so that
+any existing codeparser cache gets invalidated. Additionally you'll need
+to increment __cache_version__ in cache.py in order to ensure that old
+recipe caches don't trigger "Taskhash mismatch" errors.
+
+"""
+
 import ast
 import sys
 import codegen
@@ -117,6 +136,10 @@ class shellCacheLine(object):
 
 class CodeParserCache(MultiProcessCache):
     cache_file_name = "bb_codeparser.dat"
+    # NOTE: you must increment this if you change how the parsers gather information,
+    # so that an existing cache gets invalidated. Additionally you'll need
+    # to increment __cache_version__ in cache.py in order to ensure that old
+    # recipe caches don't trigger "Taskhash mismatch" errors.
     CACHE_VERSION = 9
 
     def __init__(self):
Signature · 1 line
2.9.3