Fixes for variable dependency handling with contains functions
cover Fixes for variable dependency handling with contains functions
Fixes for a few issues I discovered while working on bitbake-diffsigs.
The following changes since commit 58a03531c8183b165bb7dcad86d8559c92bc150d:
fetch2: Do not fail to create symbolic links if they already exist (2017-04-01 08:23:29 +0100)
are available in the git repository at:
git://git.yoctoproject.org/poky-contrib paule/bb-contains
http://git.yoctoproject.org/cgit.cgi/poky-contrib/log/?h=paule/bb-contains
Paul Eggleton (4):
lib/bb/data: fix dependency handling for contains and multiple values
codeparser: improve handling of contains_any() and filter()
bitbake-selftest: add contains tests
codeparser: add some comments
lib/bb/cache.py | 2 +- lib/bb/codeparser.py | 34 +++++++++++++++++++++++++++++--- lib/bb/data.py | 10 ++++++---- lib/bb/tests/codeparser.py | 48 ++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 86 insertions(+), 8 deletions(-)
Signature · 1 line
2.9.3
PATCH 1/4 lib/bb/data: fix dependency handling for contains and multiple values
The code that determines variable dependencies uses the codeparser to find references to "contains" type operations e.g. bb.utils.contains(). That function can take multiple items to check, and all specified items have to be present. However this code didn't handle that - it assumed that only one item would be specified and thus it was treating the multiple items as a single item with spaces in between. Split the value and check if all words are present in order to determine whether the check is "set" or "unset".
--- lib/bb/data.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-)
lib/bb/data.py +6 -4
@@ -296,11 +296,13 @@ def build_dependencies(key, keys, shelldeps, varflagsexcl, d): newvalue = "" for k in sorted(contains): l = (d.getVar(k) or "").split() - for word in sorted(contains[k]): - if word in l: - newvalue += "\n%s{%s} = Set" % (k, word) + for item in sorted(contains[k]): + for word in item.split(): + if not word in l: + newvalue += "\n%s{%s} = Unset" % (k, item) + break else: - newvalue += "\n%s{%s} = Unset" % (k, word) + newvalue += "\n%s{%s} = Set" % (k, item) if not newvalue: return value if not value:
Signature · 1 line
2.9.3
PATCH 2/4 codeparser: improve handling of contains_any() and filter()
In reply to Paul EggletonEnsure we handle bb.utils.contains_any() as separate items, rather than how we handle contains() where every item must be in the list. Additionally, enable handling bb.utils.filter() which for the purposes of looking at dependencies is the same as contains_any().
Additionally bump the codeparser cache and recipe cache versions to invalidate the user's existing caches (ensuring that the changes take effect and avoiding "taskhash mismatch" errors respectively).
--- lib/bb/cache.py | 2 +- lib/bb/codeparser.py | 11 ++++++++--- 2 files changed, 9 insertions(+), 4 deletions(-)
lib/bb/cache.py +1 -1
@@ -37,7 +37,7 @@ import bb.utils logger = logging.getLogger("BitBake.Cache") -__cache_version__ = "150" +__cache_version__ = "151" def getCacheFile(path, filename, data_hash): return os.path.join(path, filename + "." + data_hash)
lib/bb/codeparser.py +8 -3
@@ -117,7 +117,7 @@ class shellCacheLine(object): class CodeParserCache(MultiProcessCache): cache_file_name = "bb_codeparser.dat" - CACHE_VERSION = 8 + CACHE_VERSION = 9 def __init__(self): MultiProcessCache.__init__(self)
@@ -193,7 +193,8 @@ class BufferedLogger(Logger): class PythonParser(): getvars = (".getVar", ".appendVar", ".prependVar") getvarflags = (".getVarFlag", ".appendVarFlag", ".prependVarFlag") - containsfuncs = ("bb.utils.contains", "base_contains", "bb.utils.contains_any") + containsfuncs = ("bb.utils.contains", "base_contains") + containsanyfuncs = ("bb.utils.contains_any", "bb.utils.filter") execfuncs = ("bb.build.exec_func", "bb.build.exec_task") def warn(self, func, arg):
@@ -212,13 +213,17 @@ class PythonParser(): def visit_Call(self, node): name = self.called_node_name(node.func) - if name and (name.endswith(self.getvars) or name.endswith(self.getvarflags) or name in self.containsfuncs): + if name and (name.endswith(self.getvars) or name.endswith(self.getvarflags) or name in self.containsfuncs or name in self.containsanyfuncs): if isinstance(node.args[0], ast.Str): varname = node.args[0].s if name in self.containsfuncs and isinstance(node.args[1], ast.Str): if varname not in self.contains: self.contains[varname] = set() self.contains[varname].add(node.args[1].s) + elif name in self.containsanyfuncs and isinstance(node.args[1], ast.Str): + if varname not in self.contains: + self.contains[varname] = set() + self.contains[varname].update(node.args[1].s.split()) elif name.endswith(self.getvarflags): if isinstance(node.args[1], ast.Str): self.references.add('%s[%s]' % (varname, node.args[1].s))
Signature · 1 line
2.9.3
PATCH 3/4 bitbake-selftest: add contains tests
In reply to Paul EggletonAdd some tests to verify that we are extracting "contains" information from python expressions in the code in the bb.data and bb.codeparser modules.
--- lib/bb/tests/codeparser.py | 48 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+)
lib/bb/tests/codeparser.py +48 -0
@@ -49,6 +49,9 @@ class ReferenceTest(unittest.TestCase): def assertExecs(self, execs): self.assertEqual(self.execs, execs) + def assertContains(self, contains): + self.assertEqual(self.contains, contains) + class VariableReferenceTest(ReferenceTest): def parseExpression(self, exp):
@@ -201,6 +204,7 @@ class PythonReferenceTest(ReferenceTest): self.references = parsedvar.references | parser.references self.execs = parser.execs + self.contains = parser.contains @staticmethod def indent(value):
@@ -265,6 +269,26 @@ be. These unit tests are testing snippets.""" self.assertExecs(set(["testget"])) del self.context["testget"] + def test_contains(self): + self.parseExpression('bb.utils.contains("TESTVAR", "one", "true", "false", d)') + self.assertContains({'TESTVAR': {'one'}}) + + def test_contains_multi(self): + self.parseExpression('bb.utils.contains("TESTVAR", "one two", "true", "false", d)') + self.assertContains({'TESTVAR': {'one two'}}) + + def test_contains_any(self): + self.parseExpression('bb.utils.contains_any("TESTVAR", "hello", "true", "false", d)') + self.assertContains({'TESTVAR': {'hello'}}) + + def test_contains_any_multi(self): + self.parseExpression('bb.utils.contains_any("TESTVAR", "one two three", "true", "false", d)') + self.assertContains({'TESTVAR': {'one', 'two', 'three'}}) + + def test_contains_filter(self): + self.parseExpression('bb.utils.filter("TESTVAR", "hello there world", d)') + self.assertContains({'TESTVAR': {'hello', 'there', 'world'}}) + class DependencyReferenceTest(ReferenceTest):
@@ -370,6 +394,30 @@ esac self.assertEqual(deps, set(["oe_libinstall"])) + def test_contains_vardeps(self): + expr = '${@bb.utils.filter("TESTVAR", "somevalue anothervalue", d)} \ + ${@bb.utils.contains("TESTVAR", "testval testval2", "yetanothervalue", "", d)} \ + ${@bb.utils.contains("TESTVAR", "testval2 testval3", "blah", "", d)} \ + ${@bb.utils.contains_any("TESTVAR", "testval2 testval3", "lastone", "", d)}' + parsedvar = self.d.expandWithRefs(expr, None) + # Check contains + self.assertEqual(parsedvar.contains, {'TESTVAR': {'testval2 testval3', 'anothervalue', 'somevalue', 'testval testval2', 'testval2', 'testval3'}}) + # Check dependencies + self.d.setVar('ANOTHERVAR', expr) + self.d.setVar('TESTVAR', 'anothervalue testval testval2') + deps, values = bb.data.build_dependencies("ANOTHERVAR", set(self.d.keys()), set(), set(), self.d) + self.assertEqual(sorted(values.splitlines()), + sorted([expr, + 'TESTVAR{anothervalue} = Set', + 'TESTVAR{somevalue} = Unset', + 'TESTVAR{testval testval2} = Set', + 'TESTVAR{testval2 testval3} = Unset', + 'TESTVAR{testval2} = Set', + 'TESTVAR{testval3} = Unset' + ])) + # Check final value + self.assertEqual(self.d.getVar('ANOTHERVAR').split(), ['anothervalue', 'yetanothervalue', 'lastone']) + #Currently no wildcard support #def test_vardeps_wildcards(self): # self.d.setVar("oe_libinstall", "echo test")
Signature · 1 line
2.9.3
PATCH 4/4 codeparser: add some comments
In reply to Paul EggletonAdd a few comments at the top of the file explaining what it's for, and a comment pointing out that you need to increment the cache version when changing any code that changes the output.
--- lib/bb/codeparser.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+)
lib/bb/codeparser.py +23 -0
@@ -1,3 +1,22 @@ +""" +BitBake code parser + +Parses actual code (i.e. python and shell) for functions and in-line +expressions. Used mainly to determine dependencies on other functions +and variables within the BitBake metadata. Also provides a cache for +this information in order to speed up processing. + +(Not to be confused with the code that parses the metadata itself, +see lib/bb/parse/ for that). + +NOTE: if you change how the parsers gather information you will almost +certainly need to increment CodeParserCache.CACHE_VERSION below so that +any existing codeparser cache gets invalidated. Additionally you'll need +to increment __cache_version__ in cache.py in order to ensure that old +recipe caches don't trigger "Taskhash mismatch" errors. + +""" + import ast import sys import codegen
@@ -117,6 +136,10 @@ class shellCacheLine(object): class CodeParserCache(MultiProcessCache): cache_file_name = "bb_codeparser.dat" + # NOTE: you must increment this if you change how the parsers gather information, + # so that an existing cache gets invalidated. Additionally you'll need + # to increment __cache_version__ in cache.py in order to ensure that old + # recipe caches don't trigger "Taskhash mismatch" errors. CACHE_VERSION = 9 def __init__(self):
Signature · 1 line
2.9.3