First draft: Variable assignment logging!

PATCH Peter Seebach 1 participant 2 messages May 2, 2012

Share & tools

Peter Seebach

author

2012-05-02 00:42 UTC #0 lore raw

This isn't even close to ready for a real review, so it's not a pull request, more just a thing I'm sending out in case people want to mess with it or provide feedback.

This needs more cleanup to track down a few cases where the defaults are inconsistent, etc.

Sample output from bitbake -e:

---
#
# CVSDIR [2]
# set conf/bitbake.conf:315:
# <${CO_DIR}/cvs>
# set flag doc /home/seebs/poky/meta/conf/documentation.conf:92:
# <The directory where cvs checkouts will be stored in. @group fetcher>
#
# CVSDIR=${CO_DIR}/cvs
CVSDIR="/home/seebs/poky/build/downloads/cvs"
#
# BBFILE_COLLECTIONS [2]
# append /home/seebs/poky/meta/conf/layer.conf:7:
# < normal>
# append /home/seebs/poky/meta-yocto/conf/layer.conf:8:
# < normal yocto>
#
# BBFILE_COLLECTIONS= normal yocto
BBFILE_COLLECTIONS="normal yocto"
---

commit fb41e13523dc27c9d4bb9f0232100dd31879c2d5

Author: Peter Seebach <[email protected]>
Date:   Tue May 1 19:13:25 2012 -0500

data_smart.py: Provide (optional) logging of variable modifications

This is very preliminary, and probably not very good, but it provides a much-wanted feature: An explanation of how variables got their current values.

How it works:

1. We create a history dict in the data_smart object, the members of which are lists of tuples. It's three, count them, THREE unrelated data types in a single object!
2. There is an eventLog(...) function which allows recording things which happen to variables.
3. All over the place, tons of setVar and related functions are updated with two optional arguments, filename and lineno.
4. We log pretty much everything if logging is enabled.
5. You can query the history of a thing with data.getHistory(var)

Tracking must be specifically turned on. If it's not, the history object is a single empty table (cheap) and the calls to eventLog() and extra argument passing are also probably quite cheap. As a proof of concept, the showEnvironment function used by bitbake -e now uses this if available.

Signed-off-by Peter Seebach ([email protected])

bitbake/lib/bb/cooker.py +3 -1

index dea0aad..76104ef 100644
@@ -825,6 +825,8 @@ class BBCooker:
 
     def parseConfigurationFiles(self, prefiles, postfiles):
         data = self.configuration.data
+        if self.configuration.show_environment:
+            data.enableTracking()
         bb.parse.init_parser(data)
 
         # Parse files for loading *before* bitbake.conf and any includes

@@ -871,7 +873,7 @@ class BBCooker:
         bb.codeparser.parser_cache_init(data)
         bb.event.fire(bb.event.ConfigParsed(), data)
         bb.parse.init_parser(data)
-        data.setVar('BBINCLUDED',bb.parse.get_file_depends(data))
+        data.setVar('BBINCLUDED',bb.parse.get_file_depends(data), 'cooker.py', 'N/A')
         self.configuration.data = data
         self.configuration.data_hash = data.get_hash()

bitbake/lib/bb/data.py +23 -8

index c0636e1..13a5f39 100644
@@ -74,14 +74,21 @@ def createCopy(source):
     """
     return source.createCopy()
 
+# These are used in dataSmart, here as protection against KeyErrors.
+def enableTracking():
+    pass
+
+def disableTracking():
+    pass
+
 def initVar(var, d):
     """Non-destructive var init for data structure"""
     d.initVar(var)
 
 
-def setVar(var, value, d):
+def setVar(var, value, d, filename = 'unknown', lineno = 'N/A'):
     """Set a variable to a given value"""
-    d.setVar(var, value)
+    d.setVar(var, value, filename, lineno)
 
 
 def getVar(var, d, exp = 0):

@@ -89,17 +96,17 @@ def getVar(var, d, exp = 0):
     return d.getVar(var, exp)
 
 
-def renameVar(key, newkey, d):
+def renameVar(key, newkey, d, filename = 'unknown', lineno = 'N/A'):
     """Renames a variable from key to newkey"""
-    d.renameVar(key, newkey)
+    d.renameVar(key, newkey, filename, lineno)
 
-def delVar(var, d):
+def delVar(var, d, filename = 'unknown', lineno = 'N/A'):
     """Removes a variable from the data set"""
-    d.delVar(var)
+    d.delVar(var, filename, lineno)
 
-def setVarFlag(var, flag, flagvalue, d):
+def setVarFlag(var, flag, flagvalue, d, filename = 'unknown', lineno = 'N/A'):
     """Set a flag for a given variable to a given value"""
-    d.setVarFlag(var, flag, flagvalue)
+    d.setVarFlag(var, flag, flagvalue, filename, lineno)
 
 def getVarFlag(var, flag, d):
     """Gets given flag from given var"""

@@ -195,6 +202,13 @@ def emit_var(var, o=sys.__stdout__, d = init(), all=False):
 
     if all:
         commentVal = re.sub('\n', '\n#', str(oval))
+        history = d.getHistory(var)
+        if history:
+            o.write('#\n# %s [%d]\n' % (var, len(history)))
+            for events in history:
+                events = (events[0], events[1], events[2], re.sub('\n', '\n#     ', str(events[3])))
+                o.write('#   %s %s:%s:\n#     <%s>\n' % events)
+            o.write('#\n')
         o.write('# %s=%s\n' % (var, commentVal))
 
     if (var.find("-") != -1 or var.find(".") != -1 or var.find('{') != -1 or var.find('}') != -1 or var.find('+') != -1) and not all:

@@ -232,6 +246,7 @@ def emit_env(o=sys.__stdout__, d = init(), all=False):
     isfunc = lambda key: bool(d.getVarFlag(key, "func"))
     keys = sorted((key for key in d.keys() if not key.startswith("__")), key=isfunc)
     grouped = groupby(keys, isfunc)
+    print("got here")
     for isfunc, keys in grouped:
         for key in keys:
             emit_var(key, o, d, all and not isfunc) and o.write('\n')

bitbake/lib/bb/data_smart.py +62 ~~-29~~

231 lines not on this page · raw message

bitbake/lib/bb/parse/__init__.py +1 -1

index 7b9c47e..3c3feee 100644
@@ -75,7 +75,7 @@ def mark_dependency(d, f):
         f = "%s/%s" % (os.getcwd(), f[2:])
     deps = d.getVar('__depends') or set()
     deps.update([(f, cached_mtime(f))])
-    d.setVar('__depends', deps)
+    d.setVar('__depends', deps, f, 'N/A')
 
 def supports(fn, data):
     """Returns true if we have a handler for this file, false otherwise"""

bitbake/lib/bb/parse/ast.py +48 ~~-41~~

232 lines not on this page · raw message

bitbake/lib/bb/parse/parse_py/BBHandler.py +4 -4

index 815bce1..44d7a82 100644
@@ -78,7 +78,7 @@ def inherit(files, fn, lineno, d):
         if not file in __inherit_cache:
             logger.log(logging.DEBUG -1, "BB %s:%d: inheriting %s", fn, lineno, file)
             __inherit_cache.append( file )
-            data.setVar('__inherit_cache', __inherit_cache, d)
+            data.setVar('__inherit_cache', __inherit_cache, d, fn, lineno)
             include(fn, file, lineno, d, "inherit")
             __inherit_cache = data.getVar('__inherit_cache', d) or []

@@ -129,7 +129,7 @@ def handle(fn, d, include):
         __inherit_cache = data.getVar('__inherit_cache', d) or []
         if not fn in __inherit_cache:
             __inherit_cache.append(fn)
-            data.setVar('__inherit_cache', __inherit_cache, d)
+            data.setVar('__inherit_cache', __inherit_cache, d, fn, 'N/A')
 
     if include != 0:
         oldfile = data.getVar('FILE', d)

@@ -146,7 +146,7 @@ def handle(fn, d, include):
 
     # DONE WITH PARSING... time to evaluate
     if ext != ".bbclass":
-        data.setVar('FILE', abs_fn, d)
+        data.setVar('FILE', abs_fn, d, fn, 'N/A')
 
     statements.eval(d)

@@ -157,7 +157,7 @@ def handle(fn, d, include):
             return ast.multi_finalize(fn, d)
 
     if oldfile:
-        d.setVar("FILE", oldfile)
+        d.setVar("FILE", oldfile, fn, 'N/A')
 
     # we have parsed the bb class now
     if ext == ".bbclass" or ext == ".inc":

bitbake/lib/bb/shell.py +1 -1

index 1dd8d54..529692d 100644
@@ -493,7 +493,7 @@ SRC_URI = ""
     def setVar( self, params ):
         """Set an outer BitBake environment variable"""
         var, value = params
-        data.setVar( var, value, cooker.configuration.data )
+        data.setVar( var, value, cooker.configuration.data, 'SHELL', '???')
         print("OK")
     setVar.usage = "<variable> <value>"

bitbake/lib/bb/siggen.py +1 -1

index 8c79b17..4cc69d2 100644
@@ -137,7 +137,7 @@ class SignatureGeneratorBasic(SignatureGenerator):
         #    self.dump_sigtask(fn, task, d.getVar("STAMP", True), False)
 
         for task in taskdeps:
-            d.setVar("BB_BASEHASH_task-%s" % task, self.basehash[fn + "." + task])
+            d.setVar("BB_BASEHASH_task-%s" % task, self.basehash[fn + "." + task], 'siggen.py', 'N/A')
 
     def rundep_check(self, fn, recipename, task, dep, depname, dataCache):
         # Return True if we should keep the dependency, False to drop

Signature · 1 line

Listen, get this.  Nobody with a good compiler needs to be justified.

Peter Seebach

author

2012-05-02 17:27 UTC #1 lore raw

Okay, I fixed a few bugs in that, and added a second patch.

What's that for? Let's ask bitbake -e:

#
# INCLUDE HISTORY:
#

# /home/seebs/poky/meta/conf/abi_version.conf
# conf/site.conf
# conf/auto.conf
# conf/local.conf
# conf/build/x86_64-linux.conf
# conf/target/INVALID-oe-linux.conf
# /home/seebs/poky/meta/conf/machine/qemux86-64.conf includes:
#   /home/seebs/poky/meta/conf/machine/include/tune-x86_64.inc includes:
#     /home/seebs/poky/meta/conf/machine/include/ia32/arch-ia32.inc
#   /home/seebs/poky/meta/conf/machine/include/qemu.inc
# conf/machine-sdk/${SDKMACHINE}.conf
# /home/seebs/poky/meta-yocto/conf/distro/poky.conf

I am not at all sure that this is done correctly, but it passes obvious sanity checks.

Would appreciate feedback/comments on these, I plan to clean them up as pull requests eventually.

commit fee6c6ad07d420565b50fed4216f56b916219c2f

Author: Peter Seebach <[email protected]>
Date:   Tue May 1 19:13:25 2012 -0500

data_smart.py: Provide (optional) logging of variable modifications

This is very preliminary, and probably not very good, but it provides a much-wanted feature: An explanation of how variables got their current values.

How it works:

Signed-off-by Peter Seebach ([email protected])

bitbake/lib/bb/cooker.py +3 -1

index dea0aad..76104ef 100644
@@ -825,6 +825,8 @@ class BBCooker:
 
     def parseConfigurationFiles(self, prefiles, postfiles):
         data = self.configuration.data
+        if self.configuration.show_environment:
+            data.enableTracking()
         bb.parse.init_parser(data)
 
         # Parse files for loading *before* bitbake.conf and any includes

@@ -871,7 +873,7 @@ class BBCooker:
         bb.codeparser.parser_cache_init(data)
         bb.event.fire(bb.event.ConfigParsed(), data)
         bb.parse.init_parser(data)
-        data.setVar('BBINCLUDED',bb.parse.get_file_depends(data))
+        data.setVar('BBINCLUDED',bb.parse.get_file_depends(data), 'cooker.py', 'N/A')
         self.configuration.data = data
         self.configuration.data_hash = data.get_hash()

bitbake/lib/bb/data.py +22 -8

index c0636e1..74efc5d 100644
@@ -74,14 +74,21 @@ def createCopy(source):
     """
     return source.createCopy()
 
+# These are used in dataSmart, here as protection against KeyErrors.
+def enableTracking():
+    pass
+
+def disableTracking():
+    pass
+
 def initVar(var, d):
     """Non-destructive var init for data structure"""
     d.initVar(var)
 
 
-def setVar(var, value, d):
+def setVar(var, value, d, filename = 'unknown', lineno = 'N/A'):
     """Set a variable to a given value"""
-    d.setVar(var, value)
+    d.setVar(var, value, filename, lineno)
 
 
 def getVar(var, d, exp = 0):

@@ -89,17 +96,17 @@ def getVar(var, d, exp = 0):
     return d.getVar(var, exp)
 
 
-def renameVar(key, newkey, d):
+def renameVar(key, newkey, d, filename = 'unknown', lineno = 'N/A'):
     """Renames a variable from key to newkey"""
-    d.renameVar(key, newkey)
+    d.renameVar(key, newkey, filename, lineno)
 
-def delVar(var, d):
+def delVar(var, d, filename = 'unknown', lineno = 'N/A'):
     """Removes a variable from the data set"""
-    d.delVar(var)
+    d.delVar(var, filename, lineno)
 
-def setVarFlag(var, flag, flagvalue, d):
+def setVarFlag(var, flag, flagvalue, d, filename = 'unknown', lineno = 'N/A'):
     """Set a flag for a given variable to a given value"""
-    d.setVarFlag(var, flag, flagvalue)
+    d.setVarFlag(var, flag, flagvalue, filename, lineno)
 
 def getVarFlag(var, flag, d):
     """Gets given flag from given var"""

@@ -195,6 +202,13 @@ def emit_var(var, o=sys.__stdout__, d = init(), all=False):
 
     if all:
         commentVal = re.sub('\n', '\n#', str(oval))
+        history = d.getHistory(var)
+        if history:
+            o.write('#\n# %s [%d]\n' % (var, len(history)))
+            for events in history:
+                events = (events[0], events[1], events[2], re.sub('\n', '\n#     ', str(events[3])))
+                o.write('#   %s %s:%s:\n#     <%s>\n' % events)
+            o.write('#\n')
         o.write('# %s=%s\n' % (var, commentVal))
 
     if (var.find("-") != -1 or var.find(".") != -1 or var.find('{') != -1 or var.find('}') != -1 or var.find('+') != -1) and not all:

bitbake/lib/bb/data_smart.py +62 ~~-29~~

231 lines not on this page · raw message

bitbake/lib/bb/parse/__init__.py +1 -1

index 7b9c47e..3c3feee 100644
@@ -75,7 +75,7 @@ def mark_dependency(d, f):
         f = "%s/%s" % (os.getcwd(), f[2:])
     deps = d.getVar('__depends') or set()
     deps.update([(f, cached_mtime(f))])
-    d.setVar('__depends', deps)
+    d.setVar('__depends', deps, f, 'N/A')
 
 def supports(fn, data):
     """Returns true if we have a handler for this file, false otherwise"""

bitbake/lib/bb/parse/ast.py +48 ~~-41~~

232 lines not on this page · raw message

bitbake/lib/bb/parse/parse_py/BBHandler.py +4 -4

index 815bce1..44d7a82 100644
@@ -78,7 +78,7 @@ def inherit(files, fn, lineno, d):
         if not file in __inherit_cache:
             logger.log(logging.DEBUG -1, "BB %s:%d: inheriting %s", fn, lineno, file)
             __inherit_cache.append( file )
-            data.setVar('__inherit_cache', __inherit_cache, d)
+            data.setVar('__inherit_cache', __inherit_cache, d, fn, lineno)
             include(fn, file, lineno, d, "inherit")
             __inherit_cache = data.getVar('__inherit_cache', d) or []

@@ -129,7 +129,7 @@ def handle(fn, d, include):
         __inherit_cache = data.getVar('__inherit_cache', d) or []
         if not fn in __inherit_cache:
             __inherit_cache.append(fn)
-            data.setVar('__inherit_cache', __inherit_cache, d)
+            data.setVar('__inherit_cache', __inherit_cache, d, fn, 'N/A')
 
     if include != 0:
         oldfile = data.getVar('FILE', d)

@@ -146,7 +146,7 @@ def handle(fn, d, include):
 
     # DONE WITH PARSING... time to evaluate
     if ext != ".bbclass":
-        data.setVar('FILE', abs_fn, d)
+        data.setVar('FILE', abs_fn, d, fn, 'N/A')
 
     statements.eval(d)

@@ -157,7 +157,7 @@ def handle(fn, d, include):
             return ast.multi_finalize(fn, d)
 
     if oldfile:
-        d.setVar("FILE", oldfile)
+        d.setVar("FILE", oldfile, fn, 'N/A')
 
     # we have parsed the bb class now
     if ext == ".bbclass" or ext == ".inc":

bitbake/lib/bb/shell.py +1 -1

index 1dd8d54..529692d 100644
@@ -493,7 +493,7 @@ SRC_URI = ""
     def setVar( self, params ):
         """Set an outer BitBake environment variable"""
         var, value = params
-        data.setVar( var, value, cooker.configuration.data )
+        data.setVar( var, value, cooker.configuration.data, 'SHELL', '???')
         print("OK")
     setVar.usage = "<variable> <value>"

bitbake/lib/bb/siggen.py +1 -1

index 8c79b17..4cc69d2 100644
@@ -137,7 +137,7 @@ class SignatureGeneratorBasic(SignatureGenerator):
         #    self.dump_sigtask(fn, task, d.getVar("STAMP", True), False)
 
         for task in taskdeps:
-            d.setVar("BB_BASEHASH_task-%s" % task, self.basehash[fn + "." + task])
+            d.setVar("BB_BASEHASH_task-%s" % task, self.basehash[fn + "." + task], 'siggen.py', 'N/A')
 
     def rundep_check(self, fn, recipename, task, dep, depname, dataCache):
         # Return True if we should keep the dependency, False to drop it

commit 1e02fc22a6bc9a0241ebcd492a0c8b0397ec5258

Author: Peter Seebach <[email protected]>
Date:   Wed May 2 12:11:08 2012 -0500

data_smart.py: Track configuration file inclusions

This is a preliminary attempt to create a structured representation of the processing of include files. It ignores line numbers because they're meaningless. It could probably be updated to, say, distinguish between includes and requires. Output is added to bitbake -e.

Signed-off-by Peter Seebach <[email protected]>

bitbake/lib/bb/data.py +16 -0

index 74efc5d..5ad7862 100644
@@ -246,10 +246,26 @@ def emit_env(o=sys.__stdout__, d = init(), all=False):
     isfunc = lambda key: bool(d.getVarFlag(key, "func"))
     keys = sorted((key for key in d.keys() if not key.startswith("__")), key=isfunc)
     grouped = groupby(keys, isfunc)
+    # Include history!
+    if d.tracking():
+        o.write('#\n# INCLUDE HISTORY:\n#\n')
+        emit_history(o, d.getIncludeHistory())
+        
     for isfunc, keys in grouped:
         for key in keys:
             emit_var(key, o, d, all and not isfunc) and o.write('\n')
 
+def emit_history(o, h, depth = 0):
+    if not h:
+        return
+    for event in h:
+        o.write("# %*s%s" % (depth * 2, "", event[0]))
+        if event[1]:
+            o.write(" includes:\n")
+            emit_history(o, event[1], depth + 1)
+        else:
+            o.write("\n")
+
 def exported_keys(d):
     return (key for key in d.keys() if not key.startswith('__') and
                                       d.getVarFlag(key, 'export') and

bitbake/lib/bb/data_smart.py +34 -2

index 5a8917f..e527cee 100644
@@ -112,6 +112,8 @@ class DataSmart(MutableMapping):
     def __init__(self, special = COWDictBase.copy(), seen = COWDictBase.copy() ):
         self.dict = {}
         self.history = {}
+        self.include_history = []
+        self.include_stack = [(-1, self.include_history)]
 
         # cookie monster tribute
         self._special_values = special

@@ -120,6 +122,25 @@ class DataSmart(MutableMapping):
 
         self.expand_cache = {}
 
+    def includeLog(self, filename):
+        """includeLog(included_file) shows that the file was included
+        by the currently-processed file or context."""
+        if self._tracking_enabled:
+            event = (filename, [])
+            position = (len(self.include_stack[-1][1]), event[1])
+            self.include_stack[-1][1].append(event)
+            self.include_stack.append(position)
+
+    def includeLogDone(self, filename):
+        if self._tracking_enabled:
+            if len(self.include_stack) > 1:
+                self.include_stack.pop()
+            else:
+                bb.warn("Uh-oh:  includeLogDone(%s) tried to empty the stack." % filename)
+
+    def getIncludeHistory(self):
+        return self.include_history
+
     def tracking(self):
         return self._tracking_enabled

@@ -434,8 +455,19 @@ class DataSmart(MutableMapping):
         # we really want this to be a DataSmart...
         data = DataSmart(seen=self._seen_overrides.copy(), special=self._special_values.copy())
         data.dict["_data"] = self.dict
-        data.history = self.history.copy()
-        data._tracking_enabled = self._tracking_enabled
+        if self._tracking_enabled:
+            data._tracking_enabled = self._tracking_enabled
+            data.history = self.history.copy()
+            data.include_history = copy.deepcopy(self.include_history)
+            data.include_stack = []
+            oldref = self.include_history
+            newref = data.include_history
+            # Create corresponding references.
+            for item in self.include_stack:
+                if item[0] >= 0:
+                    newref = newref[item[0]]
+                newevent = (item[0], newref)
+                data.include_stack.append(newevent)
 
         return data

bitbake/lib/bb/parse/parse_py/ConfHandler.py +3 -0

index 6f77bd4..e5def63 100644
@@ -63,6 +63,7 @@ def include(oldfn, fn, lineno, data, error_out):
         if abs_fn:
             fn = abs_fn
 
+    data.includeLog(fn)
     from bb.parse import handle
     try:
         ret = handle(fn, data, True)

@@ -70,6 +71,8 @@ def include(oldfn, fn, lineno, data, error_out):
         if error_out:
             raise ParseError("Could not %(error_out)s file %(fn)s" % vars(), oldfn, lineno)
         logger.debug(2, "CONF file '%s' not found", fn)
+    finally:
+        data.includeLogDone(fn)
 
 # We have an issue where a UI might want to enforce particular settings such as
 # an empty DISTRO variable. If configuration files do something like

assigning