Tinfoil fixes
v2 cover Tinfoil fixes
Add context management and a parsing function, and ensure tinfoil using utilities within the bitbake repo shut down tinfoil properly at the end. The parsing function in particular is going to be needed to properly fix the layer index update script that has been broken since the multiconfig changes.
(This series borrows a few things from my tinfoil2 branch.)
Changes since v1:
* Ensure we can still pass a custom datastore into parse_recipe_file() since the layer index update script needs to be able to do that. To accomplish this, make some minor structural changes in lib/bb/cache.py to make an internal function into something we can easily call from in tinfoil.
The following changes since commit 0ed8975c42718342a104a9764a58816f964ec4ea:
fetch2: clean up remaining cwd saves/changes (2016-08-24 13:55:36 +0100)
are available in the git repository at:
git://git.yoctoproject.org/poky-contrib paule/tinfoil-fixes-bb
http://git.yoctoproject.org/cgit.cgi/poky-contrib/log/?h=paule/tinfoil-fixes-bb
Paul Eggleton (4):
tinfoil: add context manager functions
Ensure tinfoil is shut down correctly in utilities that use it
cache: allow parsing a recipe with a custom config datastore
tinfoil: add a parse_recipe_file function
bin/bitbake-diffsigs | 6 ++--- bin/bitbake-layers | 53 ++++++++++++++++++++------------------ lib/bb/cache.py | 73 ++++++++++++++++++++++++++++------------------------ lib/bb/tinfoil.py | 42 ++++++++++++++++++++++++++++++ 4 files changed, 112 insertions(+), 62 deletions(-)
Signature · 1 line
2.5.5
PATCH v2 1/4 tinfoil: add context manager functions
Since calling the shutdown() function is highly recommended, make tinfoil objects a little easier to deal with by adding context manager support - so you can do the following:
with bb.tinfoil.Tinfoil() as tinfoil:
tinfoil.prepare(True)
...
--- lib/bb/tinfoil.py | 6 ++++++ 1 file changed, 6 insertions(+)
lib/bb/tinfoil.py +6 -0
@@ -59,6 +59,12 @@ class Tinfoil: def register_idle_function(self, function, data): pass + def __enter__(self): + return self + + def __exit__(self, type, value, traceback): + self.shutdown() + def parseRecipes(self): sys.stderr.write("Parsing recipes..") self.logger.setLevel(logging.WARNING)
Signature · 1 line
2.5.5
PATCH v2 2/4 Ensure tinfoil is shut down correctly in utilities that use it
In reply to Paul EggletonWe should always shut down tinfoil when we're finished with it, either by explicitly calling the shutdown() method or by using it as a context manager ("with ...").
--- bin/bitbake-diffsigs | 6 +++--- bin/bitbake-layers | 53 +++++++++++++++++++++++++++------------------------- 2 files changed, 31 insertions(+), 28 deletions(-)
bin/bitbake-diffsigs +3 -3
@@ -115,9 +115,9 @@ parser.add_option("-t", "--task", options, args = parser.parse_args(sys.argv) if options.taskargs: - tinfoil = bb.tinfoil.Tinfoil() - tinfoil.prepare(config_only = True) - find_compare_task(tinfoil, options.taskargs[0], options.taskargs[1]) + with bb.tinfoil.Tinfoil() as tinfoil: + tinfoil.prepare(config_only=True) + find_compare_task(tinfoil, options.taskargs[0], options.taskargs[1]) else: if len(args) == 1: parser.print_help()
bin/bitbake-layers +28 -25
59 lines not on this page · raw message
Signature · 1 line
2.5.5
PATCH v2 3/4 cache: allow parsing a recipe with a custom config datastore
In reply to Paul EggletonTo accommodate the OpenEmbedded layer index recipe parsing, we have to have the ability to pass in a custom config datastore since it constructs a synthetic one. To make this possible after the multi-config changes, rename the internal _load_bbfile() function to parse_recipe(), make it a function at the module level (since it doesn't actually need to access any members of the class or instance) and move setting __BBMULTICONFIG inside it since other code will expect that to be set.
Part of the fix for [YOCTO #10192].
--- lib/bb/cache.py | 73 ++++++++++++++++++++++++++++++--------------------------- 1 file changed, 39 insertions(+), 34 deletions(-)
lib/bb/cache.py +39 -34
@@ -286,6 +286,42 @@ def variant2virtual(realfn, variant): return "multiconfig:" + elems[1] + ":" + realfn return "virtual:" + variant + ":" + realfn +def parse_recipe(bb_data, bbfile, appends, mc=''): + """ + Parse a recipe + """ + + chdir_back = False + + bb_data.setVar("__BBMULTICONFIG", mc) + + # expand tmpdir to include this topdir + bb_data.setVar('TMPDIR', bb_data.getVar('TMPDIR', True) or "") + bbfile_loc = os.path.abspath(os.path.dirname(bbfile)) + oldpath = os.path.abspath(os.getcwd()) + bb.parse.cached_mtime_noerror(bbfile_loc) + + # The ConfHandler first looks if there is a TOPDIR and if not + # then it would call getcwd(). + # Previously, we chdir()ed to bbfile_loc, called the handler + # and finally chdir()ed back, a couple of thousand times. We now + # just fill in TOPDIR to point to bbfile_loc if there is no TOPDIR yet. + if not bb_data.getVar('TOPDIR', False): + chdir_back = True + bb_data.setVar('TOPDIR', bbfile_loc) + try: + if appends: + bb_data.setVar('__BBAPPEND', " ".join(appends)) + bb_data = bb.parse.handle(bbfile, bb_data) + if chdir_back: + os.chdir(oldpath) + return bb_data + except: + if chdir_back: + os.chdir(oldpath) + raise + + class NoCache(object):
@@ -312,54 +348,23 @@ class NoCache(object): if virtonly: (bbfile, virtual, mc) = virtualfn2realfn(bbfile) bb_data = self.databuilder.mcdata[mc].createCopy() - bb_data.setVar("__BBMULTICONFIG", mc) bb_data.setVar("__ONLYFINALISE", virtual or "default") - datastores = self._load_bbfile(bb_data, bbfile, appends) + datastores = parse_recipe(bb_data, bbfile, appends, mc) return datastores bb_data = self.data.createCopy() - datastores = self._load_bbfile(bb_data, bbfile, appends) + datastores = parse_recipe(bb_data, bbfile, appends, mc) for mc in self.databuilder.mcdata: if not mc: continue bb_data = self.databuilder.mcdata[mc].createCopy() - bb_data.setVar("__BBMULTICONFIG", mc) - newstores = self._load_bbfile(bb_data, bbfile, appends) + newstores = parse_recipe(bb_data, bbfile, appends, mc) for ns in newstores: datastores["multiconfig:%s:%s" % (mc, ns)] = newstores[ns] return datastores - def _load_bbfile(self, bb_data, bbfile, appends): - chdir_back = False - - # expand tmpdir to include this topdir - bb_data.setVar('TMPDIR', bb_data.getVar('TMPDIR', True) or "") - bbfile_loc = os.path.abspath(os.path.dirname(bbfile)) - oldpath = os.path.abspath(os.getcwd()) - bb.parse.cached_mtime_noerror(bbfile_loc) - - # The ConfHandler first looks if there is a TOPDIR and if not - # then it would call getcwd(). - # Previously, we chdir()ed to bbfile_loc, called the handler - # and finally chdir()ed back, a couple of thousand times. We now - # just fill in TOPDIR to point to bbfile_loc if there is no TOPDIR yet. - if not bb_data.getVar('TOPDIR', False): - chdir_back = True - bb_data.setVar('TOPDIR', bbfile_loc) - try: - if appends: - bb_data.setVar('__BBAPPEND', " ".join(appends)) - bb_data = bb.parse.handle(bbfile, bb_data) - if chdir_back: - os.chdir(oldpath) - return bb_data - except: - if chdir_back: - os.chdir(oldpath) - raise - class Cache(NoCache): """ BitBake Cache implementation
Signature · 1 line
2.5.5
Re: [PATCH v2 3/4] cache: allow parsing a recipe with a custom config datastore
Quoted 66 lines · On Wed, 31 Aug 2016 08:48:08 Paul Eggleton wrote:
To accommodate the OpenEmbedded layer index recipe parsing, we have to have the ability to pass in a custom config datastore since it constructs a synthetic one. To make this possible after the multi-config changes, rename the internal
_load_bbfile()function toparse_recipe(), make it a function at the module level (since it doesn't actually need to access any members of the class or instance) and move setting__BBMULTICONFIGinside it since other code will expect that to be set.Part of the fix for [YOCTO #10192].
Signed-off-by Paul Eggleton <[email protected]>--- lib/bb/cache.py | 73++++++++++++++++++++++++++++++--------------------------- 1 file changed, 39 insertions(+), 34 deletions(-)
lib/bb/cache.py +36
-0@@ -286,6 +286,42 @@ def variant2virtual(realfn, variant): return "multiconfig:" + elems[1] + ":" + realfn return "virtual:" + variant + ":" + realfn +def parse_recipe(bb_data, bbfile, appends, mc=''): + """ + Parse a recipe + """ + + chdir_back = False + + bb_data.setVar("__BBMULTICONFIG", mc) + + # expand tmpdir to include this topdir + bb_data.setVar('TMPDIR', bb_data.getVar('TMPDIR', True) or "") + bbfile_loc = os.path.abspath(os.path.dirname(bbfile)) + oldpath = os.path.abspath(os.getcwd()) + bb.parse.cached_mtime_noerror(bbfile_loc) + + # The ConfHandler first looks if there is a TOPDIR and if not + # then it would call getcwd(). + # Previously, we chdir()ed to bbfile_loc, called the handler + # and finally chdir()ed back, a couple of thousand times. We now + # just fill in TOPDIR to point to bbfile_loc if there is no TOPDIR yet. + if not bb_data.getVar('TOPDIR', False): + chdir_back = True + bb_data.setVar('TOPDIR', bbfile_loc) + try: + if appends: + bb_data.setVar('__BBAPPEND', " ".join(appends)) + bb_data = bb.parse.handle(bbfile, bb_data) + if chdir_back: + os.chdir(oldpath) + return bb_data + except: + if chdir_back: + os.chdir(oldpath) + raise + + class NoCache(object):@@ -312,54 +348,23 @@ class NoCache(object): if virtonly: (bbfile, virtual, mc) = virtualfn2realfn(bbfile)
lib/bb/cache.py class NoCache(object): · line 356
Quoted 10 lines
lib/bb/cache.py +2
-3bb_data = self.databuilder.mcdata[mc].createCopy() - bb_data.setVar("__BBMULTICONFIG", mc) bb_data.setVar("__ONLYFINALISE", virtual or "default") - datastores = self._load_bbfile(bb_data, bbfile, appends) + datastores = parse_recipe(bb_data, bbfile, appends, mc) return datastores bb_data = self.data.createCopy() - datastores = self._load_bbfile(bb_data, bbfile, appends) + datastores = parse_recipe(bb_data, bbfile, appends, mc)
So the last line above was definitely wrong - I've dropped the "mc" parameter in this call and pushed a new paule/tinfoil-fixes-bb branch.
Cheers,
Paul
Signature · 2 lines
Paul Eggleton Intel Open Source Technology Centre
PATCH v2 4/4 tinfoil: add a parse_recipe_file function
In reply to Paul EggletonParsing a recipe is such a common task for tinfoil-using scripts, and is a little awkward to do properly, so add an API function to do it. This should also isolate scripts a little from future changes to the internal code. The first user of this will be the OpenEmbedded layer index update script.
Part of the fix for [YOCTO #10192].
--- lib/bb/tinfoil.py | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+)
lib/bb/tinfoil.py +36 -0
@@ -90,6 +90,42 @@ class Tinfoil: else: self.parseRecipes() + def parse_recipe_file(self, fn, appends=True, appendlist=None, config_data=None): + """ + Parse the specified recipe file (with or without bbappends) + and return a datastore object representing the environment + for the recipe. + Parameters: + fn: recipe file to parse - can be a file path or virtual + specification + appends: True to apply bbappends, False otherwise + appendlist: optional list of bbappend files to apply, if you + want to filter them + config_data: custom config datastore to use. NOTE: if you + specify config_data then you cannot use a virtual + specification for fn. + """ + if appends and appendlist == []: + appends = False + if appends: + if appendlist: + appendfiles = appendlist + else: + if not hasattr(self.cooker, 'collection'): + raise Exception('You must call tinfoil.prepare() with config_only=False in order to get bbappends') + appendfiles = self.cooker.collection.get_file_appends(fn) + else: + appendfiles = None + if config_data: + # We have to use a different function here if we're passing in a datastore + localdata = bb.data.createCopy(config_data) + envdata = bb.cache.parse_recipe(localdata, fn, appendfiles)[''] + else: + # Use the standard path + parser = bb.cache.NoCache(self.cooker.databuilder) + envdata = parser.loadDataFull(fn, appendfiles) + return envdata + def shutdown(self): self.cooker.shutdown(force=True) self.cooker.post_serve()
Signature · 1 line
2.5.5