Tinfoil usage fixes

PATCH In review Paul Eggleton 2 participants 5 messages Aug 31, 2016

Share & tools

Paul Eggleton

author

2016-08-31 01:48 UTC #0 lore raw

cover Tinfoil usage fixes

NOTE: the scripts/contrib/* patch requires the bitbake series I just sent. We may wish to consider a bitbake version and required bitbake version bump soon.

The following changes since commit 384cf92ca9c3e66763c2c1ff2776c53d47ae25d6:

init-install: Fixes the install script failing when not finding any mmcblk devices (2016-08-30 07:57:46 +0100)

are available in the git repository at:

git://git.openembedded.org/openembedded-core-contrib paule/tinfoil-fixes-oe
http://cgit.openembedded.org/cgit.cgi/openembedded-core-contrib/log/?h=paule/tinfoil-fixes-oe

Paul Eggleton (2):
scripts: ensure tinfoil is shut down correctly scripts/contrib: update scripts for changes to internal API

 scripts/contrib/devtool-stress.py           |   6 +-
 scripts/contrib/list-packageconfig-flags.py |  35 ++-
 scripts/contrib/verify-homepage.py          |  18 +-
 scripts/devtool                             |  10 +-
 scripts/lib/devtool/build_image.py          | 124 +++++-----
 scripts/lib/devtool/deploy.py               | 141 ++++++------
 scripts/lib/devtool/runqemu.py              |   8 +-
 scripts/lib/devtool/standard.py             | 341 ++++++++++++++--------------
 scripts/lib/devtool/upgrade.py              |  85 +++----
 scripts/oe-pkgdata-util                     |   5 +-
 scripts/recipetool                          |  63 ++---
 11 files changed, 432 insertions(+), 404 deletions(-)

Signature · 1 line

2.5.5

Paul Eggleton

author

2016-08-31 01:48 UTC #1 lore raw

PATCH 1/2 scripts: ensure tinfoil is shut down correctly

We should always shut down tinfoil when we're finished with it, either by explicitly calling the shutdown() method or by using it as a context manager ("with ...").

Signed-off-by Paul Eggleton <[email protected]>

---
 scripts/contrib/list-packageconfig-flags.py |  28 +--
 scripts/contrib/verify-homepage.py          |  12 +-
 scripts/devtool                             |  10 +-
 scripts/lib/devtool/build_image.py          | 124 +++++-----
 scripts/lib/devtool/deploy.py               | 141 ++++++------
 scripts/lib/devtool/runqemu.py              |   8 +-
 scripts/lib/devtool/standard.py             | 341 ++++++++++++++--------------
 scripts/lib/devtool/upgrade.py              |  85 +++----
 scripts/oe-pkgdata-util                     |   5 +-
 scripts/recipetool                          |  63 ++---
 10 files changed, 423 insertions(+), 394 deletions(-)

scripts/contrib/list-packageconfig-flags.py +14 ~~-14~~

index b8327e4..1af7de3 100755
@@ -160,20 +160,20 @@ def main():
 
     options, args = parser.parse_args(sys.argv)
 
-    bbhandler = bb.tinfoil.Tinfoil()
-    bbhandler.prepare()
-    print("Gathering recipe data...")
-    data_dict = get_recipesdata(bbhandler, options.preferred)
-
-    if options.listtype == 'flags':
-        pkg_dict = collect_pkgs(data_dict)
-        flag_dict = collect_flags(pkg_dict)
-        display_flags(flag_dict)
-    elif options.listtype == 'recipes':
-        pkg_dict = collect_pkgs(data_dict)
-        display_pkgs(pkg_dict)
-    elif options.listtype == 'all':
-        display_all(data_dict)
+    with bb.tinfoil.Tinfoil() as bbhandler:
+        bbhandler.prepare()
+        print("Gathering recipe data...")
+        data_dict = get_recipesdata(bbhandler, options.preferred)
+
+        if options.listtype == 'flags':
+            pkg_dict = collect_pkgs(data_dict)
+            flag_dict = collect_flags(pkg_dict)
+            display_flags(flag_dict)
+        elif options.listtype == 'recipes':
+            pkg_dict = collect_pkgs(data_dict)
+            display_pkgs(pkg_dict)
+        elif options.listtype == 'all':
+            display_all(data_dict)
 
 if __name__ == "__main__":
     main()

scripts/contrib/verify-homepage.py +6 -6

index 61a047c4..04917e8 100755
@@ -54,9 +54,9 @@ def verifyHomepage(bbhandler):
     return count
 
 if __name__=='__main__':
-    bbhandler = bb.tinfoil.Tinfoil()
-    bbhandler.prepare()
-    logger.info("Start verifying HOMEPAGE:")
-    failcount = verifyHomepage(bbhandler)
-    logger.info("Finished verifying HOMEPAGE.")
-    logger.info("Summary: %s failed" % failcount)
+    with bb.tinfoil.Tinfoil() as bbhandler:
+        bbhandler.prepare()
+        logger.info("Start verifying HOMEPAGE:")
+        failcount = verifyHomepage(bbhandler)
+        logger.info("Finished verifying HOMEPAGE.")
+        logger.info("Summary: %s failed" % failcount)

scripts/devtool +4 -6

index b1274d6..91e3954 100755
@@ -289,17 +289,15 @@ def main():
 
     if global_args.bbpath is None:
         tinfoil = setup_tinfoil(config_only=True, basepath=basepath)
-        global_args.bbpath = tinfoil.config_data.getVar('BBPATH', True)
-    else:
-        tinfoil = None
+        try:
+            global_args.bbpath = tinfoil.config_data.getVar('BBPATH', True)
+        finally:
+            tinfoil.shutdown()
 
     for path in [scripts_path] + global_args.bbpath.split(':'):
         pluginpath = os.path.join(path, 'lib', 'devtool')
         scriptutils.load_plugins(logger, plugins, pluginpath)
 
-    if tinfoil:
-        tinfoil.shutdown()
-
     subparsers = parser.add_subparsers(dest="subparser_name", title='subcommands', metavar='<subcommand>')
     subparsers.required = True

scripts/lib/devtool/build_image.py +65 ~~-59~~

135 lines not on this page · raw message

scripts/lib/devtool/deploy.py +72 ~~-69~~

155 lines not on this page · raw message

scripts/lib/devtool/runqemu.py +5 -3

index daee7fb..303abca 100644
@@ -30,9 +30,11 @@ def runqemu(args, config, basepath, workspace):
     """Entry point for the devtool 'runqemu' subcommand"""
 
     tinfoil = setup_tinfoil(config_only=True, basepath=basepath)
-    machine = tinfoil.config_data.getVar('MACHINE', True)
-    bindir_native = tinfoil.config_data.getVar('STAGING_BINDIR_NATIVE', True)
-    tinfoil.shutdown()
+    try:
+        machine = tinfoil.config_data.getVar('MACHINE', True)
+        bindir_native = tinfoil.config_data.getVar('STAGING_BINDIR_NATIVE', True)
+    finally:
+        tinfoil.shutdown()
 
     if not glob.glob(os.path.join(bindir_native, 'qemu-system-*')):
         raise DevtoolError('QEMU is not available within this SDK')

scripts/lib/devtool/standard.py +176 ~~-165~~

386 lines not on this page · raw message

scripts/lib/devtool/upgrade.py +44 ~~-41~~

92 lines not on this page · raw message

scripts/oe-pkgdata-util +4 -1

index e4d262d..bb917b4 100755
@@ -569,7 +569,10 @@ def main():
             sys.exit(1)
         logger.debug('Found bitbake path: %s' % bitbakepath)
         tinfoil = tinfoil_init()
-        args.pkgdata_dir = tinfoil.config_data.getVar('PKGDATA_DIR', True)
+        try:
+            args.pkgdata_dir = tinfoil.config_data.getVar('PKGDATA_DIR', True)
+        finally:
+            tinfoil.shutdown()
         logger.debug('Value of PKGDATA_DIR is "%s"' % args.pkgdata_dir)
         if not args.pkgdata_dir:
             logger.error('Unable to determine pkgdata directory from PKGDATA_DIR')

scripts/recipetool +33 ~~-30~~

70 lines not on this page · raw message

Signature · 1 line

2.5.5

Richard Purdie

2016-08-31 10:55 UTC #2 lore raw

Re: [PATCH 1/2] scripts: ensure tinfoil is shut down correctly

Quoted 4 lines · Paul Eggleton

We should always shut down tinfoil when we're finished with it, either
by explicitly calling the shutdown() method or by using it as a context manager ("with ...").

Could I trouble you to rebase this onto master-next please? It currently has conflicts which didn't seem immediately obvious to me to resolve.

Cheers,

Richard

Paul Eggleton

author

2016-08-31 19:52 UTC #3 lore raw

Re: [PATCH 1/2] scripts: ensure tinfoil is shut down correctly

Quoted 9 lines · On Wed, 31 Aug 2016 11:55:44 Richard Purdie wrote:

Quoted 4 lines · Paul Eggleton
We should always shut down tinfoil when we're finished with it, either
by explicitly calling the shutdown() method or by using it as a context manager ("with ...").
Could I trouble you to rebase this onto master-next please? It currently has conflicts which didn't seem immediately obvious to me to resolve.

Done - I've pushed a new paule/tinfoil-fixes-oe branch.

Cheers,
Paul

Signature · 2 lines

Paul Eggleton
Intel Open Source Technology Centre

Paul Eggleton

author

2016-08-31 01:48 UTC #4 lore raw

PATCH 2/2 scripts/contrib: update scripts for changes to internal API

In reply to Paul Eggleton

The multiconfig changes altered some of the functions being called here, so update the calls. Make use of the new Tinfoil.parse_recipe_file() function to make parsing easier.

Signed-off-by Paul Eggleton <[email protected]>

---
 scripts/contrib/devtool-stress.py           | 6 +++---
 scripts/contrib/list-packageconfig-flags.py | 7 +++----
 scripts/contrib/verify-homepage.py          | 6 +++---
 3 files changed, 9 insertions(+), 10 deletions(-)

scripts/contrib/devtool-stress.py +3 -3

index 7ba0984..d555c51 100755
@@ -43,15 +43,15 @@ def select_recipes(args):
     tinfoil = bb.tinfoil.Tinfoil()
     tinfoil.prepare(False)
 
-    pkg_pn = tinfoil.cooker.recipecache.pkg_pn
-    (latest_versions, preferred_versions) = bb.providers.findProviders(tinfoil.config_data, tinfoil.cooker.recipecache, pkg_pn)
+    pkg_pn = tinfoil.cooker.recipecaches[''].pkg_pn
+    (latest_versions, preferred_versions) = bb.providers.findProviders(tinfoil.config_data, tinfoil.cooker.recipecaches[''], pkg_pn)
 
     skip_classes = args.skip_classes.split(',')
 
     recipelist = []
     for pn in sorted(pkg_pn):
         pref = preferred_versions[pn]
-        inherits = [os.path.splitext(os.path.basename(f))[0] for f in tinfoil.cooker.recipecache.inherits[pref[1]]]
+        inherits = [os.path.splitext(os.path.basename(f))[0] for f in tinfoil.cooker.recipecaches[''].inherits[pref[1]]]
         for cls in skip_classes:
             if cls in inherits:
                 break

scripts/contrib/list-packageconfig-flags.py +3 -4

index 1af7de3..389fb97 100755
@@ -37,7 +37,6 @@ if not bitbakepath:
     sys.stderr.write("Unable to find bitbake by searching parent directory of this script or PATH\n")
     sys.exit(1)
 
-import bb.cache
 import bb.cooker
 import bb.providers
 import bb.tinfoil

@@ -45,7 +44,7 @@ import bb.tinfoil
 def get_fnlist(bbhandler, pkg_pn, preferred):
     ''' Get all recipe file names '''
     if preferred:
-        (latest_versions, preferred_versions) = bb.providers.findProviders(bbhandler.config_data, bbhandler.cooker.recipecache, pkg_pn)
+        (latest_versions, preferred_versions) = bb.providers.findProviders(bbhandler.config_data, bbhandler.cooker.recipecaches[''], pkg_pn)
 
     fn_list = []
     for pn in sorted(pkg_pn):

@@ -58,11 +57,11 @@ def get_fnlist(bbhandler, pkg_pn, preferred):
 
 def get_recipesdata(bbhandler, preferred):
     ''' Get data of all available recipes which have PACKAGECONFIG flags '''
-    pkg_pn = bbhandler.cooker.recipecache.pkg_pn
+    pkg_pn = bbhandler.cooker.recipecaches[''].pkg_pn
 
     data_dict = {}
     for fn in get_fnlist(bbhandler, pkg_pn, preferred):
-        data = bb.cache.Cache.loadDataFull(fn, bbhandler.cooker.collection.get_file_appends(fn), bbhandler.config_data)
+        data = bbhandler.parse_recipe_file(fn)
         flags = data.getVarFlags("PACKAGECONFIG")
         flags.pop('doc', None)
         if flags:

scripts/contrib/verify-homepage.py +3 -3

index 04917e8..d39dd1d 100755
@@ -33,17 +33,17 @@ def wgetHomepage(pn, homepage):
         return 0
 
 def verifyHomepage(bbhandler):
-    pkg_pn = bbhandler.cooker.recipecache.pkg_pn
+    pkg_pn = bbhandler.cooker.recipecaches[''].pkg_pn
     pnlist = sorted(pkg_pn)
     count = 0
     checked = []
     for pn in pnlist:
         for fn in pkg_pn[pn]:
             # There's no point checking multiple BBCLASSEXTENDed variants of the same recipe
-            realfn, _ = bb.cache.Cache.virtualfn2realfn(fn)
+            realfn, _, _ = bb.cache.virtualfn2realfn(fn)
             if realfn in checked:
                 continue
-            data = bb.cache.Cache.loadDataFull(realfn, bbhandler.cooker.collection.get_file_appends(realfn), bbhandler.config_data)
+            data = bbhandler.parse_recipe_file(realfn)
             homepage = data.getVar("HOMEPAGE", True)
             if homepage:
                 try:

Signature · 1 line

2.5.5