build/data: Write out more complete python run files

PATCH In review Richard Purdie 2 participants 3 messages Aug 27, 2014 – Sep 3, 2014

Share & tools

Richard Purdie

author

2014-08-27 13:24 UTC #0 lore raw

PATCH 1/1 build/data: Write out more complete python run files

Currently the output in the python task/function run files is rather incomplete and effectively useless. This enhances the code to take advantage of the bitbake's dependency tracking and extend the output to include dependencies. This makes the files more usable for debugging purposes. Since this only happens at python function execution time, the overhead is minimal in the grand scheme of things.

Signed-off-by Richard Purdie <[email protected]>

bitbake/lib/bb/build.py +1 -1

index f2922f3..65cc851 100644
@@ -227,7 +227,7 @@ def exec_func_python(func, d, runfile, cwd=None):
     code = _functionfmt.format(function=func, body=d.getVar(func, True))
     bb.utils.mkdirhier(os.path.dirname(runfile))
     with open(runfile, 'w') as script:
-        script.write(code)
+        bb.data.emit_func_python(func, script, d)
 
     if cwd:
         try:

bitbake/lib/bb/data.py +35 -0

index 3d776b3..91b1eb1 100644
@@ -281,6 +281,41 @@ def emit_func(func, o=sys.__stdout__, d = init()):
                newdeps |= set((d.getVarFlag(dep, "vardeps", True) or "").split())
         newdeps -= seen
 
+_functionfmt = """
+def {function}(d):
+{body}"""
+
+def emit_func_python(func, o=sys.__stdout__, d = init()):
+    """Emits all items in the data store in a format such that it can be sourced by a shell."""
+
+    def write_func(func, o, call = False):
+        body = d.getVar(func, True)
+        if not body.startswith("def"):
+            body = _functionfmt.format(function=func, body=body)
+
+        o.write(body.strip() + "\n\n")
+        if call:
+            o.write(func + "(d)" + "\n\n")
+
+    write_func(func, o, True)
+    pp = bb.codeparser.PythonParser(func, logger)
+    pp.parse_python(d.getVar(func, True))
+    newdeps = pp.execs
+    newdeps |= set((d.getVarFlag(func, "vardeps", True) or "").split())
+    seen = set()
+    while newdeps:
+        deps = newdeps
+        seen |= deps
+        newdeps = set()
+        for dep in deps:
+            if d.getVarFlag(dep, "func") and d.getVarFlag(dep, "python"):
+               write_func(dep, o)
+               pp = bb.codeparser.PythonParser(dep, logger)
+               pp.parse_python(d.getVar(dep, True))
+               newdeps |= pp.execs
+               newdeps |= set((d.getVarFlag(dep, "vardeps", True) or "").split())
+        newdeps -= seen
+
 def update_data(d):
     """Performs final steps upon the datastore, including application of overrides"""
     d.finalize(parent = True)

Olof Johansson

2014-09-02 08:21 UTC #1 lore raw

Quoted 9 lines · Richard Purdie

Currently the output in the python task/function run files is rather incomplete and effectively useless. This enhances the code to take advantage of the bitbake's dependency tracking and extend the output to include dependencies. This makes the files more usable for debugging purposes. Since this only happens at python function execution time, the overhead is minimal in the grand scheme of things.
Signed-off-by Richard Purdie <[email protected]>

....

Quoted 4 lines

bitbake/lib/bb/data.py +0 -0
index 3d776b3..91b1eb1 100644

....

bitbake/lib/bb/data.py def emit_func(func, o=sys.__stdout__, d = init()): · line 294

Quoted 7 lines

bitbake/lib/bb/data.py +7 -0

+def emit_func_python(func, o=sys.__stdout__, d = init()):
+    """Emits all items in the data store in a format such that it can be sourced by a shell."""
+
+    def write_func(func, o, call = False):
+        body = d.getVar(func, True)
+        if not body.startswith("def"):
+            body = _functionfmt.format(function=func, body=body)

....

With this change, we are seeing problems in certain cases when a python function (def foo():) is being resolved. d.getVar("foo") would return None and then fail from not having a startswith() method:

ERROR: Build of do_unpack failed
ERROR: Traceback (most recent call last):
File "/var/opt/builds/olofjn/oe/master/bitbake/lib/bb/build.py", line 497, in exec_task
return _exec_task(fn, task, d, quieterr)
File "/var/opt/builds/olofjn/oe/master/bitbake/lib/bb/build.py", line 438, in _exec_task
exec_func(task, localdata)
File "/var/opt/builds/olofjn/oe/master/bitbake/lib/bb/build.py", line 212, in exec_func
exec_func_python(func, d, runfile, cwd=adir)
File "/var/opt/builds/olofjn/oe/master/bitbake/lib/bb/build.py", line 230, in exec_func_python
bb.data.emit_func_python(func, script, d)
File "/var/opt/builds/olofjn/oe/master/bitbake/lib/bb/data.py", line 313, in emit_func_python
write_func(dep, o)
File "/var/opt/builds/olofjn/oe/master/bitbake/lib/bb/data.py", line 294, in write_func
if not body.startswith("def"):
AttributeError: 'NoneType' object has no attribute 'startswith'

Our qemu_%.bbappend contains the following:

    def _remove_git_submodule_file(filename):
        import os
    
        if not os.path.isfile(filename):
            return
    
        os.unlink(filename)
    
    do_unpack_append() {
        _remove_git_submodule_file(d.getVar("S", True) + "/dtc/.git")
        _remove_git_submodule_file(d.getVar("S", True) + "/pixman/.git")
    }

Moving the function to within do_unpack_append would solve the problem.

Signature · 1 line

olofjn

Richard Purdie

author

2014-09-03 22:08 UTC #2 lore raw

Quoted 61 lines · Olof Johansson

Quoted 9 lines · Richard Purdie
Currently the output in the python task/function run files is rather incomplete and effectively useless. This enhances the code to take advantage of the bitbake's dependency tracking and extend the output to include dependencies. This makes the files more usable for debugging purposes. Since this only happens at python function execution time, the overhead is minimal in the grand scheme of things.
Signed-off-by Richard Purdie <[email protected]>
....
Quoted 4 lines
bitbake/lib/bb/data.py +0 -0
index 3d776b3..91b1eb1 100644
....
bitbake/lib/bb/data.py def emit_func(func, o=sys.__stdout__, d = init()): · line 294
Quoted 7 lines
bitbake/lib/bb/data.py +7 -0
+def emit_func_python(func, o=sys.__stdout__, d = init()):
+    """Emits all items in the data store in a format such that it can be sourced by a shell."""
+
+    def write_func(func, o, call = False):
+        body = d.getVar(func, True)
+        if not body.startswith("def"):
+            body = _functionfmt.format(function=func, body=body)
....
With this change, we are seeing problems in certain cases when a python function (def foo():) is being resolved. d.getVar("foo") would return None and then fail from not having a startswith() method:
ERROR: Build of do_unpack failed
ERROR: Traceback (most recent call last):
File "/var/opt/builds/olofjn/oe/master/bitbake/lib/bb/build.py", line 497, in exec_task
return _exec_task(fn, task, d, quieterr)
File "/var/opt/builds/olofjn/oe/master/bitbake/lib/bb/build.py", line 438, in _exec_task
exec_func(task, localdata)
File "/var/opt/builds/olofjn/oe/master/bitbake/lib/bb/build.py", line 212, in exec_func
exec_func_python(func, d, runfile, cwd=adir)
File "/var/opt/builds/olofjn/oe/master/bitbake/lib/bb/build.py", line 230, in exec_func_python
bb.data.emit_func_python(func, script, d)
File "/var/opt/builds/olofjn/oe/master/bitbake/lib/bb/data.py", line 313, in emit_func_python
write_func(dep, o)
File "/var/opt/builds/olofjn/oe/master/bitbake/lib/bb/data.py", line 294, in write_func
if not body.startswith("def"):
AttributeError: 'NoneType' object has no attribute 'startswith'
Our qemu_%.bbappend contains the following:
    def _remove_git_submodule_file(filename):
        import os
    
        if not os.path.isfile(filename):
            return
    
        os.unlink(filename)
    
    do_unpack_append() {
        _remove_git_submodule_file(d.getVar("S", True) + "/dtc/.git")
        _remove_git_submodule_file(d.getVar("S", True) + "/pixman/.git")
    }

Quoted 1 line

Moving the function to within do_unpack_append would solve the problem.

We should probably put a safeguard in that code, if the function isn't in the datastore, not to try and write it out, just pass over it. Its either that, or pull it from the methodpool, which thinking about it, may not be a bad idea since it is in there somewhere...

Cheers,

Richard