oe-pkgdata-util improvements
cover oe-pkgdata-util improvements
Some fairly significant improvements to oe-pkgdata-util that make it much easier to use for querying built package information, plus some QA tests to make sure it doesn't break in future.
The following changes since commit eb9d896db2fc67bac8efd258744d06fbbee87f06:
libtool: avoid running automake/autoconf --version (2015-02-05 09:45:49 +0000)
are available in the git repository at:
git://git.openembedded.org/openembedded-core-contrib paule/oe-pkgdata-util
http://cgit.openembedded.org/cgit.cgi/openembedded-core-contrib/log/?h=paule/oe-pkgdata-util
Paul Eggleton (5):
oe-pkgdata-util: improve command-line usage
oe-pkgdata-util: allow reverse package name lookups
oe-pkgdata-util: make find-path show a proper error if no package
found
oe-pkgdata-util: add list-pkgs subcommand
oe-pkgdata-util: add some QA tests
meta/classes/buildhistory.bbclass | 2 +- meta/classes/license.bbclass | 2 +- meta/lib/oe/package_manager.py | 2 +- meta/lib/oeqa/runtime/_ptest.py | 2 +- meta/lib/oeqa/selftest/pkgdata.py | 125 ++++++++++++ scripts/oe-pkgdata-util | 389 ++++++++++++++++++++++++-------------- 6 files changed, 372 insertions(+), 150 deletions(-) create mode 100644 meta/lib/oeqa/selftest/pkgdata.py
Signature · 1 line
1.9.3
PATCH 1/5 oe-pkgdata-util: improve command-line usage
* Use argparse instead of optparse for standardised help output, options and a much cleaner code structure
* Look up pkgdata directory automatically so the user doesn't have to specify it
* Use standard logging
NOTE: this does mean a slight change in syntax - if you do want to specify the pkgdata directory (usually only necessary if you're calling it from within the build process) you need to use the parameter -p (or
--pkgdata-dir) and specify this before the command, not after it.
Examples:
oe-pkgdata-util find-path /sbin/mke2fs
oe-pkgdata-util lookup-recipe libelf1
oe-pkgdata-util read-value PKGSIZE libc6
oe-pkgdata-util -p /home/user/oe/build/tmp/sysroots/qemux86-64/pkgdata read-value PKGSIZE libc6
--- meta/classes/buildhistory.bbclass | 2 +- meta/classes/license.bbclass | 2 +- meta/lib/oe/package_manager.py | 2 +- meta/lib/oeqa/runtime/_ptest.py | 2 +- scripts/oe-pkgdata-util | 264 ++++++++++++++++++-------------------- 5 files changed, 132 insertions(+), 140 deletions(-)
meta/classes/buildhistory.bbclass +1 -1
@@ -374,7 +374,7 @@ buildhistory_get_installed() { printf "" > $1/installed-package-sizes.tmp cat $pkgcache | while read pkg pkgfile pkgarch do - size=`oe-pkgdata-util read-value ${PKGDATA_DIR} "PKGSIZE" ${pkg}_${pkgarch}` + size=`oe-pkgdata-util -p ${PKGDATA_DIR} read-value "PKGSIZE" ${pkg}_${pkgarch}` if [ "$size" != "" ] ; then echo "$size $pkg" >> $1/installed-package-sizes.tmp fi
meta/classes/license.bbclass +1 -1
@@ -80,7 +80,7 @@ license_create_manifest() { if [ "${COPY_LIC_DIRS}" = "1" ]; then for pkg in ${INSTALLED_PKGS}; do mkdir -p ${IMAGE_ROOTFS}/usr/share/common-licenses/${pkg} - pkged_pn="$(oe-pkgdata-util lookup-recipe ${PKGDATA_DIR} ${pkg})" + pkged_pn="$(oe-pkgdata-util -p ${PKGDATA_DIR} lookup-recipe ${pkg})" for lic in `ls ${LICENSE_DIRECTORY}/${pkged_pn}`; do # Really don't need to copy the generics as they're # represented in the manifest and in the actual pkg licenses
meta/lib/oe/package_manager.py +1 -1
@@ -529,7 +529,7 @@ class PackageManager(object): return cmd = [bb.utils.which(os.getenv('PATH'), "oe-pkgdata-util"), - "glob", self.d.getVar('PKGDATA_DIR', True), installed_pkgs_file, + "-p", self.d.getVar('PKGDATA_DIR', True), "glob", installed_pkgs_file, globs] try: bb.note("Installing complementary packages ...")
meta/lib/oeqa/runtime/_ptest.py +1 -1
@@ -86,7 +86,7 @@ class PtestRunnerTest(oeRuntimeTest): installed_pkgs.write(self.pkgs_list.list("arch")) cmd = [bb.utils.which(os.getenv('PATH'), "oe-pkgdata-util"), - "glob", oeRuntimeTest.tc.d.getVar('PKGDATA_DIR', True), installed_pkgs_file, + "-p", oeRuntimeTest.tc.d.getVar('PKGDATA_DIR', True), "glob", installed_pkgs_file, globs] try: bb.note("Installing complementary packages ...")
scripts/oe-pkgdata-util +128 -136
383 lines not on this page · raw message
Signature · 1 line
1.9.3
PATCH 2/5 oe-pkgdata-util: allow reverse package name lookups
In reply to Paul EggletonAdd a -r/--reverse option to the lookup-pkg subcommand to enable looking up the recipe-space package name for one or more runtime package names.
Also make this subcommand into a function that can be reused elsewhere.
--- scripts/oe-pkgdata-util | 54 ++++++++++++++++++++++++++++++++----------------- 1 file changed, 36 insertions(+), 18 deletions(-)
scripts/oe-pkgdata-util +36 -18
@@ -27,7 +27,7 @@ import fnmatch import re import argparse import logging -from collections import defaultdict +from collections import defaultdict, OrderedDict scripts_path = os.path.dirname(os.path.realpath(__file__)) lib_path = scripts_path + '/lib'
@@ -184,30 +184,47 @@ def read_value(args): else: print(readvar(revlink, qvar)) +def lookup_pkglist(pkgs, pkgdata_dir, reverse): + if reverse: + mappings = OrderedDict() + for pkg in pkgs: + revlink = os.path.join(pkgdata_dir, "runtime-reverse", pkg) + logger.debug(revlink) + if os.path.exists(revlink): + mappings[pkg] = os.path.basename(os.readlink(revlink)) + else: + mappings = defaultdict(list) + for pkg in pkgs: + pkgfile = os.path.join(pkgdata_dir, 'runtime', pkg) + if os.path.exists(pkgfile): + with open(pkgfile, 'r') as f: + for line in f: + fields = line.rstrip().split(': ') + if fields[0] == 'PKG_%s' % pkg: + mappings[pkg].append(fields[1]) + break + return mappings + def lookup_pkg(args): # Handle both multiple arguments and multiple values within an arg (old syntax) pkgs = [] - for pkgitem in args.recipepkg: + for pkgitem in args.pkg: pkgs.extend(pkgitem.split()) - mappings = defaultdict(list) - for pkg in pkgs: - pkgfile = os.path.join(args.pkgdata_dir, 'runtime', pkg) - if os.path.exists(pkgfile): - with open(pkgfile, 'r') as f: - for line in f: - fields = line.rstrip().split(': ') - if fields[0] == 'PKG_%s' % pkg: - mappings[pkg].append(fields[1]) - break + mappings = lookup_pkglist(pkgs, args.pkgdata_dir, args.reverse) + if len(mappings) < len(pkgs): missing = list(set(pkgs) - set(mappings.keys())) logger.error("The following packages could not be found: %s" % ', '.join(missing)) sys.exit(1) - items = [] - for pkg in pkgs: - items.extend(mappings.get(pkg, [])) + if args.reverse: + items = mappings.values() + else: + items = [] + for pkg in pkgs: + items.extend(mappings.get(pkg, [])) + print('\n'.join(items)) def lookup_recipe(args):
@@ -260,9 +277,10 @@ def main(): subparsers = parser.add_subparsers(title='subcommands', metavar='<subcommand>') parser_lookup_pkg = subparsers.add_parser('lookup-pkg', - help='Translate recipe-space package names to runtime package names', - description='Looks up the specified recipe-space package name(s) to see what the final runtime package name is (e.g. glibc becomes libc6)') - parser_lookup_pkg.add_argument('recipepkg', nargs='+', help='Recipe-space package name to look up') + help='Translate between recipe-space package names and runtime package names', + description='Looks up the specified recipe-space package name(s) to see what the final runtime package name is (e.g. glibc becomes libc6), or with -r/--reverse looks up the other way.') + parser_lookup_pkg.add_argument('pkg', nargs='+', help='Package name to look up') + parser_lookup_pkg.add_argument('-r', '--reverse', help='Switch to looking up recipe-space package names from runtime package names', action='store_true') parser_lookup_pkg.set_defaults(func=lookup_pkg) parser_lookup_recipe = subparsers.add_parser('lookup-recipe',
Signature · 1 line
1.9.3
PATCH 3/5 oe-pkgdata-util: make find-path show a proper error if no package found
In reply to Paul Eggleton--- scripts/oe-pkgdata-util | 5 +++++ 1 file changed, 5 insertions(+)
scripts/oe-pkgdata-util +5 -0
@@ -256,6 +256,7 @@ def lookup_recipe(args): def find_path(args): import json + found = False for root, dirs, files in os.walk(os.path.join(args.pkgdata_dir, 'runtime')): for fn in files: with open(os.path.join(root,fn)) as f:
@@ -265,8 +266,12 @@ def find_path(args): dictval = json.loads(val) for fullpth in dictval.keys(): if fnmatch.fnmatchcase(fullpth, args.targetpath): + found = True print("%s: %s" % (fn, fullpth)) break + if not found: + logger.error("Unable to find any package producing path %s" % args.targetpath) + sys.exit(1) def main():
Signature · 1 line
1.9.3
PATCH 4/5 oe-pkgdata-util: add list-pkgs subcommand
In reply to Paul EggletonAdd a subcommand to list packages, with options to list packages matching a specification, and packages produced by a particular recipe.
--- scripts/oe-pkgdata-util | 82 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 82 insertions(+)
scripts/oe-pkgdata-util +82 -0
@@ -253,6 +253,79 @@ def lookup_recipe(args): items.extend(mappings.get(pkg, [])) print('\n'.join(items)) +def list_pkgs(args): + found = False + + def matchpkg(pkg): + if args.pkgspec: + matched = False + for pkgspec in args.pkgspec: + if fnmatch.fnmatchcase(pkg, pkgspec): + matched = True + break + if not matched: + return False + if not args.unpackaged: + if args.runtime: + revlink = os.path.join(args.pkgdata_dir, "runtime-reverse", pkg) + if os.path.exists(revlink): + # We're unlikely to get here if the package was not packaged, but just in case + # we add the symlinks for unpackaged files in the future + mappedpkg = os.path.basename(os.readlink(revlink)) + if not os.path.exists(os.path.join(args.pkgdata_dir, 'runtime', '%s.packaged' % mappedpkg)): + return False + else: + return False + else: + if not os.path.exists(os.path.join(args.pkgdata_dir, 'runtime', '%s.packaged' % pkg)): + return False + return True + + if args.recipe: + recipedatafile = os.path.join(args.pkgdata_dir, args.recipe) + if not os.path.exists(recipedatafile): + logger.error("Unable to find packaged recipe with name %s" % args.recipe) + sys.exit(1) + packages = [] + with open(recipedatafile, 'r') as f: + for line in f: + fields = line.rstrip().split(': ') + if fields[0] == 'PACKAGES': + packages = fields[1].split() + break + + if args.runtime: + pkglist = [] + runtime_pkgs = lookup_pkglist(packages, args.pkgdata_dir, False) + for rtpkgs in runtime_pkgs.values(): + pkglist.extend(rtpkgs) + else: + pkglist = packages + + for pkg in pkglist: + if matchpkg(pkg): + found = True + print("%s" % pkg) + else: + if args.runtime: + searchdir = 'runtime-reverse' + else: + searchdir = 'runtime' + + for root, dirs, files in os.walk(os.path.join(args.pkgdata_dir, searchdir)): + for fn in files: + if fn.endswith('.packaged'): + continue + if matchpkg(fn): + found = True + print("%s" % fn) + if not found: + if args.pkgspec: + logger.error("Unable to find any package matching %s" % args.pkgspec) + else: + logger.error("No packages found") + sys.exit(1) + def find_path(args): import json
@@ -288,6 +361,15 @@ def main(): parser_lookup_pkg.add_argument('-r', '--reverse', help='Switch to looking up recipe-space package names from runtime package names', action='store_true') parser_lookup_pkg.set_defaults(func=lookup_pkg) + parser_list_pkgs = subparsers.add_parser('list-pkgs', + help='List packages', + description='Lists packages that have been built') + parser_list_pkgs.add_argument('pkgspec', nargs='*', help='Package name to search for (wildcards * ? allowed, use quotes to avoid shell expansion)') + parser_list_pkgs.add_argument('-r', '--runtime', help='Show runtime package names instead of recipe-space package names', action='store_true') + parser_list_pkgs.add_argument('-p', '--recipe', help='Limit to packages produced by the specified recipe') + parser_list_pkgs.add_argument('-u', '--unpackaged', help='Include unpackaged (i.e. empty) packages', action='store_true') + parser_list_pkgs.set_defaults(func=list_pkgs) + parser_lookup_recipe = subparsers.add_parser('lookup-recipe', help='Find recipe producing one or more packages', description='Looks up the specified runtime package(s) to see which recipe they were produced by')
Signature · 1 line
1.9.3
PATCH 5/5 oe-pkgdata-util: add some QA tests
In reply to Paul EggletonTest each of the subcommands that this utility provides.
--- meta/lib/oeqa/selftest/pkgdata.py | 125 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 125 insertions(+) create mode 100644 meta/lib/oeqa/selftest/pkgdata.py
meta/lib/oeqa/selftest/pkgdata.py new file +125 -0
@@ -0,0 +1,125 @@ +import unittest +import os +import tempfile +import logging + +import oeqa.utils.ftools as ftools +from oeqa.selftest.base import oeSelfTest +from oeqa.utils.commands import runCmd, bitbake, get_bb_var +from oeqa.utils.decorators import testcase + +class OePkgdataUtilTests(oeSelfTest): + + @classmethod + def setUpClass(cls): + # Ensure we have the right data in pkgdata + logger = logging.getLogger("selftest") + logger.info('Running bitbake to generate pkgdata') + bitbake('glibc busybox zlib bash') + + def test_lookup_pkg(self): + # Forward tests + result = runCmd('oe-pkgdata-util lookup-pkg "glibc busybox"') + self.assertEqual(result.output, 'libc6\nbusybox') + result = runCmd('oe-pkgdata-util lookup-pkg zlib-dev') + self.assertEqual(result.output, 'libz-dev') + result = runCmd('oe-pkgdata-util lookup-pkg nonexistentpkg', ignore_status=True) + self.assertEqual(result.status, 1) + self.assertEqual(result.output, 'ERROR: The following packages could not be found: nonexistentpkg') + # Reverse tests + result = runCmd('oe-pkgdata-util lookup-pkg -r "libc6 busybox"') + self.assertEqual(result.output, 'glibc\nbusybox') + result = runCmd('oe-pkgdata-util lookup-pkg -r libz-dev') + self.assertEqual(result.output, 'zlib-dev') + result = runCmd('oe-pkgdata-util lookup-pkg -r nonexistentpkg', ignore_status=True) + self.assertEqual(result.status, 1) + self.assertEqual(result.output, 'ERROR: The following packages could not be found: nonexistentpkg') + + def test_read_value(self): + result = runCmd('oe-pkgdata-util read-value PN libz1') + self.assertEqual(result.output, 'zlib') + result = runCmd('oe-pkgdata-util read-value PKGSIZE bash') + pkgsize = int(result.output.strip()) + self.assertGreater(pkgsize, 1) + + def test_find_path(self): + result = runCmd('oe-pkgdata-util find-path /lib/libc.so.6') + self.assertEqual(result.output, 'glibc: /lib/libc.so.6') + result = runCmd('oe-pkgdata-util find-path /bin/bash') + self.assertEqual(result.output, 'bash: /bin/bash') + result = runCmd('oe-pkgdata-util find-path /not/exist', ignore_status=True) + self.assertEqual(result.status, 1) + self.assertEqual(result.output, 'ERROR: Unable to find any package producing path /not/exist') + + def test_lookup_recipe(self): + result = runCmd('oe-pkgdata-util lookup-recipe "libc6-staticdev busybox"') + self.assertEqual(result.output, 'glibc\nbusybox') + result = runCmd('oe-pkgdata-util lookup-recipe libz-dbg') + self.assertEqual(result.output, 'zlib') + result = runCmd('oe-pkgdata-util lookup-recipe nonexistentpkg', ignore_status=True) + self.assertEqual(result.status, 1) + self.assertEqual(result.output, 'ERROR: The following packages could not be found: nonexistentpkg') + + def test_list_pkgs(self): + # No arguments + result = runCmd('oe-pkgdata-util list-pkgs') + pkglist = result.output.split() + self.assertIn('glibc-utils', pkglist) + self.assertIn('zlib-dev', pkglist) + # No pkgspec, runtime + result = runCmd('oe-pkgdata-util list-pkgs -r') + pkglist = result.output.split() + self.assertIn('libc6-utils', pkglist) + self.assertIn('libz-dev', pkglist) + # With recipe specified + result = runCmd('oe-pkgdata-util list-pkgs -p zlib') + pkglist = sorted(result.output.split()) + try: + pkglist.remove('zlib-ptest') # in case ptest is disabled + except ValueError: + pass + self.assertEqual(pkglist, ['zlib', 'zlib-dbg', 'zlib-dev', 'zlib-doc', 'zlib-staticdev']) + # With recipe specified, runtime + result = runCmd('oe-pkgdata-util list-pkgs -p zlib -r') + pkglist = sorted(result.output.split()) + try: + pkglist.remove('libz-ptest') # in case ptest is disabled + except ValueError: + pass + self.assertEqual(pkglist, ['libz-dbg', 'libz-dev', 'libz-doc', 'libz-staticdev', 'libz1']) + # With recipe specified and unpackaged + result = runCmd('oe-pkgdata-util list-pkgs -p zlib -u') + pkglist = sorted(result.output.split()) + self.assertIn('zlib-locale', pkglist) + # With recipe specified and unpackaged, runtime + result = runCmd('oe-pkgdata-util list-pkgs -p zlib -u -r') + pkglist = sorted(result.output.split()) + self.assertIn('libz-locale', pkglist) + # With recipe specified and pkgspec + result = runCmd('oe-pkgdata-util list-pkgs -p zlib "*-d*"') + pkglist = sorted(result.output.split()) + self.assertEqual(pkglist, ['zlib-dbg', 'zlib-dev', 'zlib-doc']) + # With recipe specified and pkgspec, runtime + result = runCmd('oe-pkgdata-util list-pkgs -p zlib -r "*-d*"') + pkglist = sorted(result.output.split()) + self.assertEqual(pkglist, ['libz-dbg', 'libz-dev', 'libz-doc']) + + + def test_glob(self): + tempdir = tempfile.mkdtemp(prefix='pkgdataqa') + self.track_for_cleanup(tempdir) + pkglistfile = os.path.join(tempdir, 'pkglist') + with open(pkglistfile, 'w') as f: + f.write('libc6\n') + f.write('libz1\n') + f.write('busybox\n') + result = runCmd('oe-pkgdata-util glob %s "*-dev"' % pkglistfile) + desiredresult = ['libc6-dev', 'libz-dev', 'busybox-dev'] + self.assertEqual(sorted(result.output.split()), sorted(desiredresult)) + # The following should not error (because when we use this during rootfs construction, sometimes the complementary package won't exist) + result = runCmd('oe-pkgdata-util glob %s "*-nonexistent"' % pkglistfile) + self.assertEqual(result.output, '') + + def test_specify_pkgdatadir(self): + result = runCmd('oe-pkgdata-util -p %s lookup-pkg glibc' % get_bb_var('PKGDATA_DIR')) + self.assertEqual(result.output, 'libc6')
Signature · 1 line
1.9.3