devtool / recipetool fixes
cover devtool / recipetool fixes
Fix some issues I found when trying to use devtool upgrade on a git recipe; improve extraction of patch header from original commit in devtool; recipetool setvar; and other fixes and cleanups I have been working on for a while.
All devtool/recipetool tests pass with these patches (plus the two bitbake bb.utils patches I sent the other day, though this patchset should not depend on those.)
The following changes since commit 6f98c39418c60b7c0b25b30983d2e5257158a6a4:
gcc: Drop 4.8 (2015-11-16 14:59:07 +0000)
are available in the git repository at:
git://git.openembedded.org/openembedded-core-contrib paule/devtool10
http://cgit.openembedded.org/cgit.cgi/openembedded-core-contrib/log/?h=paule/devtool10
Paul Eggleton (18):
devtool: upgrade: fix removing other recipes from workspace on reset
devtool: upgrade: fix updating PV and SRCREV
devtool: upgrade: remove erroneous error when not renaming recipe
devtool: upgrade: fetch remote repository before checking out new revision
devtool: upgrade: provide a means to update the source branch lib/oe/patch: improve extraction of patch header
devtool: disable creating workspace for extract and search subcommands
devtool: package: use DEPLOY_DIR_<pkgtype> to get deploy directory
devtool: drop unused plugin_init() functions
devtool: search: print SUMMARY value
devtool: ensure we change back to the original dir on error lib/oe/recipeutils: check in validate_pn() for names instead of filenames
devtool: build: use bbappend to set PARALLEL_MAKE
devtool: build: enable showing default task in help
devtool: clarify help text
lib/oe/recipeutils: refactor patch_recipe_file() to use edit_metadata()
recipetool: add setvar subcommand
recipetool: make plugin registration function name consistent with devtool
meta/lib/oe/patch.py | 58 +++++++++---- meta/lib/oe/recipeutils.py | 162 +++++++++++++++++++++--------------- meta/lib/oeqa/selftest/devtool.py | 8 +- scripts/devtool | 7 +- scripts/lib/devtool/__init__.py | 26 +++--- scripts/lib/devtool/build.py | 54 ++++++------ scripts/lib/devtool/package.py | 24 +++--- scripts/lib/devtool/sdk.py | 4 - scripts/lib/devtool/search.py | 79 ++++++++++-------- scripts/lib/devtool/standard.py | 32 ++++--- scripts/lib/devtool/upgrade.py | 98 ++++++++++++---------- scripts/lib/recipetool/append.py | 2 +- scripts/lib/recipetool/create.py | 2 +- scripts/lib/recipetool/newappend.py | 7 +- scripts/lib/recipetool/setvar.py | 75 +++++++++++++++++ scripts/recipetool | 10 ++- 16 files changed, 397 insertions(+), 251 deletions(-) create mode 100644 scripts/lib/recipetool/setvar.py
Signature · 1 line
2.1.0
PATCH 1/18 devtool: upgrade: fix removing other recipes from workspace on reset
If you did a "devtool add" followed by "devtool upgrade" and then did a "devtool reset" on the recipe you upgraded, the first recipe would also be deleted from the workspace - this was because we were erroneously adding the entire "recipes" subdirectory and its contents to be tracked for removal on reset. Remove the unnecessary call to os.path.dirname() that caused this.
--- scripts/lib/devtool/standard.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
scripts/lib/devtool/standard.py +1 -1
@@ -456,7 +456,7 @@ def _add_md5(config, recipename, filename): f.write('%s|%s|%s\n' % (recipename, os.path.relpath(fn, config.workspace_path), md5)) if os.path.isdir(filename): - for root, _, files in os.walk(os.path.dirname(filename)): + for root, _, files in os.walk(filename): for f in files: addfile(os.path.join(root, f)) else:
Signature · 1 line
2.1.0
PATCH 2/18 devtool: upgrade: fix updating PV and SRCREV
In reply to Paul EggletonThis code was clearly never tested. Fix the following issues:
* Actually set SRCREV if it's been specified
* Enable history tracking and reparse so that we handle if variables are set in an inc file next to the recipe
* Use a more accurate check for PV being in the recipe which will work if it's in an inc file next to the recipe
--- scripts/lib/devtool/upgrade.py | 34 ++++++++++++++++++++-------------- 1 file changed, 20 insertions(+), 14 deletions(-)
scripts/lib/devtool/upgrade.py +20 -14
@@ -91,15 +91,13 @@ def _remove_patch_dirs(recipefolder): for d in dirs: shutil.rmtree(os.path.join(root,d)) -def _recipe_contains(rf, var): - import re - found = False - with open(rf) as f: - for line in f: - if re.match("^%s.*=.*" % var, line): - found = True - break - return found +def _recipe_contains(rd, var): + rf = rd.getVar('FILE', True) + varfiles = oe.recipeutils.get_var_files(rf, [var], rd) + for var, fn in varfiles.iteritems(): + if fn and fn.startswith(os.path.dirname(rf) + os.sep): + return True + return False def _rename_recipe_dirs(oldpv, newpv, path): for root, dirs, files in os.walk(path):
@@ -257,7 +255,7 @@ def _extract_new_source(newpv, srctree, no_patch, srcrev, branch, keep_temp, tin return (rev, md5, sha256) -def _create_new_recipe(newpv, md5, sha256, workspace, rd): +def _create_new_recipe(newpv, md5, sha256, srcrev, workspace, tinfoil, rd): """Creates the new recipe under workspace""" crd = rd.createCopy()
@@ -271,8 +269,16 @@ def _create_new_recipe(newpv, md5, sha256, workspace, rd): newpv = oldpv fullpath = _rename_recipe_files(bpn, oldpv, newpv, path) - if _recipe_contains(fullpath, 'PV') and newpv != oldpv: - oe.recipeutils.patch_recipe(d, fullpath, {'PV':newpv}) + newvalues = {} + if _recipe_contains(rd, 'PV') and newpv != oldpv: + newvalues['PV'] = newpv + + if srcrev: + newvalues['SRCREV'] = srcrev + + if newvalues: + rd = oe.recipeutils.parse_recipe(fullpath, None, tinfoil.config_data) + oe.recipeutils.patch_recipe(rd, fullpath, newvalues) if md5 and sha256: # Unfortunately, oe.recipeutils.patch_recipe cannot update flags.
@@ -294,7 +300,7 @@ def upgrade(args, config, basepath, workspace): if reason: raise DevtoolError(reason) - tinfoil = setup_tinfoil(basepath=basepath) + tinfoil = setup_tinfoil(basepath=basepath, tracking=True) rd = parse_recipe(config, tinfoil, args.recipename, True) if not rd:
@@ -316,7 +322,7 @@ def upgrade(args, config, basepath, workspace): rev2, md5, sha256 = _extract_new_source(args.version, args.srctree, args.no_patch, args.srcrev, args.branch, args.keep_temp, tinfoil, rd) - rf = _create_new_recipe(args.version, md5, sha256, config.workspace_path, rd) + rf = _create_new_recipe(args.version, md5, sha256, args.srcrev, config.workspace_path, tinfoil, rd) except bb.process.CmdError as e: _upgrade_error(e, rf, args.srctree) except DevtoolError as e:
Signature · 1 line
2.1.0
PATCH 3/18 devtool: upgrade: remove erroneous error when not renaming recipe
In reply to Paul EggletonIf we're upgrading a git recipe the recipe file usually won't need renaming; for some unknown reason we were throwing an error here which isn't correct.
--- scripts/lib/devtool/upgrade.py | 1 - 1 file changed, 1 deletion(-)
scripts/lib/devtool/upgrade.py +0 -1
@@ -117,7 +117,6 @@ def _rename_recipe_file(bpn, oldpv, newpv, path): recipe = "%s_git.bb" % bpn if os.path.isfile(os.path.join(path, recipe)): newrecipe = recipe - raise DevtoolError("Original recipe not found on workspace") return os.path.join(path, newrecipe) def _rename_recipe_files(bpn, oldpv, newpv, path):
Signature · 1 line
2.1.0
PATCH 4/18 devtool: upgrade: fetch remote repository before checking out new revision
In reply to Paul EggletonIf we're upgrading a recipe that fetches from git, and we've simply fetched a tarball of the repo instead of directly from the upstream repo (this can happen if you have PREMIRRORS set up as in poky with a core recipe, e.g. kernelshark) then we won't have any new revisions, and the checkout will fail with "fatal: reference is not a tree: <hash>". To avoid this, do a "git fetch" before checking out the new revision.
--- scripts/lib/devtool/upgrade.py | 1 + 1 file changed, 1 insertion(+)
scripts/lib/devtool/upgrade.py +1 -0
@@ -199,6 +199,7 @@ def _extract_new_source(newpv, srctree, no_patch, srcrev, branch, keep_temp, tin if srcrev: rev = srcrev if uri.startswith('git://'): + __run('git fetch') __run('git checkout %s' % rev) __run('git tag -f devtool-base-new') md5 = None
Signature · 1 line
2.1.0
PATCH 5/18 devtool: upgrade: provide a means to update the source branch
In reply to Paul EggletonIf you're upgrading a git recipe to a revision on a release branch that's different to the branch for the current revision, then you'll need to update the branch parameter in SRC_URI, so add a --srcbranch/-B command-line parameter to let you do that easily. It handles both when the branch is stated verbatim in the recipe, and when a reference to another variable is used (a common convention is to use a SRCBRANCH variable for this, though the code doesn't care what variable is used if any).
--- scripts/lib/devtool/upgrade.py | 32 ++++++++++++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-)
scripts/lib/devtool/upgrade.py +30 -2
@@ -255,7 +255,7 @@ def _extract_new_source(newpv, srctree, no_patch, srcrev, branch, keep_temp, tin return (rev, md5, sha256) -def _create_new_recipe(newpv, md5, sha256, srcrev, workspace, tinfoil, rd): +def _create_new_recipe(newpv, md5, sha256, srcrev, srcbranch, workspace, tinfoil, rd): """Creates the new recipe under workspace""" crd = rd.createCopy()
@@ -276,6 +276,31 @@ def _create_new_recipe(newpv, md5, sha256, srcrev, workspace, tinfoil, rd): if srcrev: newvalues['SRCREV'] = srcrev + if srcbranch: + src_uri = oe.recipeutils.split_var_value(rd.getVar('SRC_URI', False) or '') + changed = False + replacing = True + new_src_uri = [] + for entry in src_uri: + scheme, network, path, user, passwd, params = bb.fetch2.decodeurl(entry) + if replacing and scheme in ['git', 'gitsm']: + branch = params.get('branch', 'master') + if rd.expand(branch) != srcbranch: + # Handle case where branch is set through a variable + res = re.match(r'\$\{([^}@]+)\}', branch) + if res: + newvalues[res.group(1)] = srcbranch + # We know we won't change SRC_URI now, so break out + break + else: + params['branch'] = srcbranch + entry = bb.fetch2.encodeurl((scheme, network, path, user, passwd, params)) + changed = True + replacing = False + new_src_uri.append(entry) + if changed: + newvalues['SRC_URI'] = ' '.join(new_src_uri) + if newvalues: rd = oe.recipeutils.parse_recipe(fullpath, None, tinfoil.config_data) oe.recipeutils.patch_recipe(rd, fullpath, newvalues)
@@ -295,6 +320,8 @@ def upgrade(args, config, basepath, workspace): raise DevtoolError("recipe %s is already in your workspace" % args.recipename) if not args.version and not args.srcrev: raise DevtoolError("You must provide a version using the --version/-V option, or for recipes that fetch from an SCM such as git, the --srcrev/-S option") + if args.srcbranch and not args.srcrev: + raise DevtoolError("If you specify --srcbranch/-B then you must use --srcrev/-S to specify the revision" % args.recipename) reason = oe.recipeutils.validate_pn(args.recipename) if reason:
@@ -322,7 +349,7 @@ def upgrade(args, config, basepath, workspace): rev2, md5, sha256 = _extract_new_source(args.version, args.srctree, args.no_patch, args.srcrev, args.branch, args.keep_temp, tinfoil, rd) - rf = _create_new_recipe(args.version, md5, sha256, args.srcrev, config.workspace_path, tinfoil, rd) + rf = _create_new_recipe(args.version, md5, sha256, args.srcrev, args.srcbranch, config.workspace_path, tinfoil, rd) except bb.process.CmdError as e: _upgrade_error(e, rf, args.srctree) except DevtoolError as e:
@@ -343,6 +370,7 @@ def register_commands(subparsers, context): parser_upgrade.add_argument('srctree', help='Path to where to extract the source tree') parser_upgrade.add_argument('--version', '-V', help='Version to upgrade to (PV)') parser_upgrade.add_argument('--srcrev', '-S', help='Source revision to upgrade to (if fetching from an SCM such as git)') + parser_upgrade.add_argument('--srcbranch', '-B', help='Branch in source repository containing the revision to use (if fetching from an SCM such as git)') parser_upgrade.add_argument('--branch', '-b', default="devtool", help='Name for new development branch to checkout (default "%(default)s")') parser_upgrade.add_argument('--no-patch', action="store_true", help='Do not apply patches from the recipe to the new source code') group = parser_upgrade.add_mutually_exclusive_group()
Signature · 1 line
2.1.0
PATCH 6/18 lib/oe/patch: improve extraction of patch header
In reply to Paul EggletonFor patches that we have to extract the header information by hand (i.e. will not apply with "git am"), make the following improvements:
* If we can't extract author/date/subject, then try to do so from the commit that added the patch in git (assuming the metadata is tracked by git)
* Take only first Signed-off-by line instead of last
* Accept any case for "Signed-off-by" in case author has typed it by hand
* Improve conditional - we can skip the other cases if one matches
Implements [YOCTO #7624].
--- meta/lib/oe/patch.py | 58 ++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 43 insertions(+), 15 deletions(-)
meta/lib/oe/patch.py +43 -15
@@ -282,33 +282,32 @@ class GitApplyTree(PatchTree): return lines @staticmethod - def prepareCommit(patchfile): - """ - Prepare a git commit command line based on the header from a patch file - (typically this is useful for patches that cannot be applied with "git am" due to formatting) - """ - import tempfile + def decodeAuthor(line): + from email.header import decode_header + authorval = line.split(':', 1)[1].strip().replace('"', '') + return decode_header(authorval)[0][0] + + @staticmethod + def interpretPatchHeader(headerlines): import re author_re = re.compile('[\S ]+ <\S+@\S+\.\S+>') - # Process patch header and extract useful information - lines = GitApplyTree.extractPatchHeader(patchfile) outlines = [] author = None date = None - for line in lines: + subject = None + for line in headerlines: if line.startswith('Subject: '): subject = line.split(':', 1)[1] # Remove any [PATCH][oe-core] etc. subject = re.sub(r'\[.+?\]\s*', '', subject) - outlines.insert(0, '%s\n\n' % subject.strip()) continue - if line.startswith('From: ') or line.startswith('Author: '): - authorval = line.split(':', 1)[1].strip().replace('"', '') + elif line.startswith('From: ') or line.startswith('Author: '): + authorval = GitApplyTree.decodeAuthor(line) # git is fussy about author formatting i.e. it must be Name <email@domain> if author_re.match(authorval): author = authorval continue - if line.startswith('Date: '): + elif line.startswith('Date: '): if date is None: dateval = line.split(':', 1)[1].strip() # Very crude check for date format, since git will blow up if it's not in the right
@@ -316,12 +315,41 @@ class GitApplyTree(PatchTree): if len(dateval) > 12: date = dateval continue - if line.startswith('Signed-off-by: '): - authorval = line.split(':', 1)[1].strip().replace('"', '') + elif not author and line.lower().startswith('signed-off-by: '): + authorval = GitApplyTree.decodeAuthor(line) # git is fussy about author formatting i.e. it must be Name <email@domain> if author_re.match(authorval): author = authorval outlines.append(line) + return outlines, author, date, subject + + @staticmethod + def prepareCommit(patchfile): + """ + Prepare a git commit command line based on the header from a patch file + (typically this is useful for patches that cannot be applied with "git am" due to formatting) + """ + import tempfile + # Process patch header and extract useful information + lines = GitApplyTree.extractPatchHeader(patchfile) + outlines, author, date, subject = GitApplyTree.interpretPatchHeader(lines) + if not author or not subject: + try: + shellcmd = ["git", "log", "--format=email", "--diff-filter=A", "--", patchfile] + out = runcmd(["sh", "-c", " ".join(shellcmd)], os.path.dirname(patchfile)) + except CmdError: + out = None + if out: + _, newauthor, newdate, newsubject = GitApplyTree.interpretPatchHeader(out.splitlines()) + if not author or not date: + # These really need to go together + author = newauthor + date = newdate + if not subject: + subject = newsubject + if subject: + outlines.insert(0, '%s\n\n' % subject.strip()) + # Write out commit message to a file with tempfile.NamedTemporaryFile('w', delete=False) as tf: tmpfile = tf.name
Signature · 1 line
2.1.0
PATCH 7/18 devtool: disable creating workspace for extract and search subcommands
In reply to Paul EggletonFor subcommands that don't actually involve the workspace, don't auto-create the workspace.
--- meta/lib/oeqa/selftest/devtool.py | 8 ++++---- scripts/devtool | 4 ++-- scripts/lib/devtool/search.py | 2 +- scripts/lib/devtool/standard.py | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-)
meta/lib/oeqa/selftest/devtool.py +4 -4
@@ -823,10 +823,10 @@ class DevtoolTests(DevtoolBase): tempdir = tempfile.mkdtemp(prefix='devtoolqa') # Try devtool extract self.track_for_cleanup(tempdir) - self.track_for_cleanup(self.workspacedir) - self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') result = runCmd('devtool extract remake %s' % tempdir) self.assertTrue(os.path.exists(os.path.join(tempdir, 'Makefile.am')), 'Extracted source could not be found') + # devtool extract shouldn't create the workspace + self.assertFalse(os.path.exists(self.workspacedir)) self._check_src_repo(tempdir) @testcase(1379)
@@ -834,10 +834,10 @@ class DevtoolTests(DevtoolBase): tempdir = tempfile.mkdtemp(prefix='devtoolqa') # Try devtool extract self.track_for_cleanup(tempdir) - self.track_for_cleanup(self.workspacedir) - self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') result = runCmd('devtool extract virtual/libx11 %s' % tempdir) self.assertTrue(os.path.exists(os.path.join(tempdir, 'Makefile.am')), 'Extracted source could not be found') + # devtool extract shouldn't create the workspace + self.assertFalse(os.path.exists(self.workspacedir)) self._check_src_repo(tempdir) @testcase(1168)
scripts/devtool +2 -2
@@ -261,7 +261,7 @@ def main(): description='Sets up a new workspace. NOTE: other devtool subcommands will create a workspace automatically as needed, so you only need to use %(prog)s if you want to specify where the workspace should be located.') parser_create_workspace.add_argument('layerpath', nargs='?', help='Path in which the workspace layer should be created') parser_create_workspace.add_argument('--create-only', action="store_true", help='Only create the workspace layer, do not alter configuration') - parser_create_workspace.set_defaults(func=create_workspace) + parser_create_workspace.set_defaults(func=create_workspace, no_workspace=True) for plugin in plugins: if hasattr(plugin, 'register_commands'):
@@ -269,7 +269,7 @@ def main(): args = parser.parse_args(unparsed_args, namespace=global_args) - if args.subparser_name != 'create-workspace': + if not getattr(args, 'no_workspace', False): read_workspace() try:
scripts/lib/devtool/search.py +1 -1
@@ -77,4 +77,4 @@ def register_commands(subparsers, context): parser_search = subparsers.add_parser('search', help='Search available recipes', description='Searches for available target recipes. Matches on recipe name, package name, description and installed files, and prints the recipe name on match.') parser_search.add_argument('keyword', help='Keyword to search for (regular expression syntax allowed)') - parser_search.set_defaults(func=search) + parser_search.set_defaults(func=search, no_workspace=True)
scripts/lib/devtool/standard.py +1 -1
@@ -1117,7 +1117,7 @@ def register_commands(subparsers, context): parser_extract.add_argument('srctree', help='Path to where to extract the source tree') parser_extract.add_argument('--branch', '-b', default="devtool", help='Name for development branch to checkout') parser_extract.add_argument('--keep-temp', action="store_true", help='Keep temporary directory (for debugging)') - parser_extract.set_defaults(func=extract) + parser_extract.set_defaults(func=extract, no_workspace=True) parser_update_recipe = subparsers.add_parser('update-recipe', help='Apply changes from external source tree to recipe', description='Applies changes from external source tree to a recipe (updating/adding/removing patches as necessary, or by updating SRCREV)')
Signature · 1 line
2.1.0
PATCH 8/18 devtool: package: use DEPLOY_DIR_<pkgtype> to get deploy directory
In reply to Paul EggletonRather than reconstructing the output path for packages, use the proper variable.
--- scripts/lib/devtool/package.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-)
scripts/lib/devtool/package.py +12 -8
@@ -32,14 +32,17 @@ def package(args, config, basepath, workspace): """Entry point for the devtool 'package' subcommand""" check_workspace_recipe(workspace, args.recipename) - image_pkgtype = config.get('Package', 'image_pkgtype', '') - if not image_pkgtype: - tinfoil = setup_tinfoil(basepath=basepath) - try: - tinfoil.prepare(config_only=True) + tinfoil = setup_tinfoil(basepath=basepath) + try: + tinfoil.prepare(config_only=True) + + image_pkgtype = config.get('Package', 'image_pkgtype', '') + if not image_pkgtype: image_pkgtype = tinfoil.config_data.getVar('IMAGE_PKGTYPE', True) - finally: - tinfoil.shutdown() + + deploy_dir_pkg = tinfoil.config_data.getVar('DEPLOY_DIR_%s' % image_pkgtype.upper(), True) + finally: + tinfoil.shutdown() package_task = config.get('Package', 'package_task', 'package_write_%s' % image_pkgtype) try:
@@ -47,7 +50,8 @@ def package(args, config, basepath, workspace): except bb.process.ExecutionError as e: # We've already seen the output since watch=True, so just ensure we return something to the user return e.exitcode - logger.info('Your packages are in %s/tmp/deploy/%s' % (basepath, image_pkgtype)) + + logger.info('Your packages are in %s' % deploy_dir_pkg) return 0
Signature · 1 line
2.1.0
PATCH 9/18 devtool: drop unused plugin_init() functions
In reply to Paul EggletonThis function is no longer required to be defined for a plugin, so drop it where it's a no-op.
--- scripts/lib/devtool/build.py | 4 ---- scripts/lib/devtool/package.py | 4 ---- scripts/lib/devtool/sdk.py | 4 ---- scripts/lib/devtool/upgrade.py | 4 ---- scripts/lib/recipetool/newappend.py | 5 ----- 5 files changed, 21 deletions(-)
scripts/lib/devtool/build.py +0 -4
@@ -25,10 +25,6 @@ from devtool import exec_build_env_command, check_workspace_recipe, DevtoolError logger = logging.getLogger('devtool') -def plugin_init(pluginlist): - """Plugin initialization""" - pass - def _create_conf_file(values, conf_file=None): if not conf_file: fd, conf_file = tempfile.mkstemp(suffix='.conf')
scripts/lib/devtool/package.py +0 -4
@@ -24,10 +24,6 @@ from devtool import exec_build_env_command, setup_tinfoil, check_workspace_recip logger = logging.getLogger('devtool') -def plugin_init(pluginlist): - """Plugin initialization""" - pass - def package(args, config, basepath, workspace): """Entry point for the devtool 'package' subcommand""" check_workspace_recipe(workspace, args.recipename)
scripts/lib/devtool/sdk.py +0 -4
@@ -11,10 +11,6 @@ from devtool import exec_build_env_command, setup_tinfoil, DevtoolError logger = logging.getLogger('devtool') -def plugin_init(pluginlist): - """Plugin initialization""" - pass - def parse_locked_sigs(sigfile_path): """Return <pn:task>:<hash> dictionary""" sig_dict = {}
scripts/lib/devtool/upgrade.py +0 -4
@@ -33,10 +33,6 @@ from devtool import exec_build_env_command, setup_tinfoil, DevtoolError, parse_r logger = logging.getLogger('devtool') -def plugin_init(pluginlist): - """Plugin initialization""" - pass - def _run(cmd, cwd=''): logger.debug("Running command %s> %s" % (cwd,cmd)) return bb.process.run('%s' % cmd, cwd=cwd)
scripts/lib/recipetool/newappend.py +0 -5
@@ -32,11 +32,6 @@ logger = logging.getLogger('recipetool') tinfoil = None -def plugin_init(pluginlist): - # Don't need to do anything here right now, but plugins must have this function defined - pass - - def tinfoil_init(instance): global tinfoil tinfoil = instance
Signature · 1 line
2.1.0
PATCH 10/18 devtool: search: print SUMMARY value
In reply to Paul EggletonPrint the SUMMARY value for each matched item assuming it's not the default.
--- scripts/lib/devtool/search.py | 77 +++++++++++++++++++++++-------------------- 1 file changed, 42 insertions(+), 35 deletions(-)
scripts/lib/devtool/search.py +42 -35
@@ -22,53 +22,60 @@ import bb import logging import argparse import re -from devtool import setup_tinfoil, DevtoolError +from devtool import setup_tinfoil, parse_recipe, DevtoolError logger = logging.getLogger('devtool') def search(args, config, basepath, workspace): """Entry point for the devtool 'search' subcommand""" - tinfoil = setup_tinfoil(config_only=True, basepath=basepath) - pkgdata_dir = tinfoil.config_data.getVar('PKGDATA_DIR', True) - tinfoil.shutdown() + tinfoil = setup_tinfoil(config_only=False, basepath=basepath) + try: + pkgdata_dir = tinfoil.config_data.getVar('PKGDATA_DIR', True) + defsummary = tinfoil.config_data.getVar('SUMMARY', False) or '' - keyword_rc = re.compile(args.keyword) + keyword_rc = re.compile(args.keyword) - for fn in os.listdir(pkgdata_dir): - pfn = os.path.join(pkgdata_dir, fn) - if not os.path.isfile(pfn): - continue + for fn in os.listdir(pkgdata_dir): + pfn = os.path.join(pkgdata_dir, fn) + if not os.path.isfile(pfn): + continue - packages = [] - match = False - if keyword_rc.search(fn): - match = True + packages = [] + match = False + if keyword_rc.search(fn): + match = True - if not match: - with open(pfn, 'r') as f: - for line in f: - if line.startswith('PACKAGES:'): - packages = line.split(':', 1)[1].strip().split() + if not match: + with open(pfn, 'r') as f: + for line in f: + if line.startswith('PACKAGES:'): + packages = line.split(':', 1)[1].strip().split() - for pkg in packages: - if keyword_rc.search(pkg): - match = True - break - if os.path.exists(os.path.join(pkgdata_dir, 'runtime', pkg + '.packaged')): - with open(os.path.join(pkgdata_dir, 'runtime', pkg), 'r') as f: - for line in f: - if ': ' in line: - splitline = line.split(':', 1) - key = splitline[0] - value = splitline[1].strip() - if key in ['PKG_%s' % pkg, 'DESCRIPTION', 'FILES_INFO'] or key.startswith('FILERPROVIDES_'): - if keyword_rc.search(value): - match = True - break + for pkg in packages: + if keyword_rc.search(pkg): + match = True + break + if os.path.exists(os.path.join(pkgdata_dir, 'runtime', pkg + '.packaged')): + with open(os.path.join(pkgdata_dir, 'runtime', pkg), 'r') as f: + for line in f: + if ': ' in line: + splitline = line.split(':', 1) + key = splitline[0] + value = splitline[1].strip() + if key in ['PKG_%s' % pkg, 'DESCRIPTION', 'FILES_INFO'] or key.startswith('FILERPROVIDES_'): + if keyword_rc.search(value): + match = True + break - if match: - print(fn) + if match: + rd = parse_recipe(config, tinfoil, fn, True) + summary = rd.getVar('SUMMARY', True) + if summary == rd.expand(defsummary): + summary = '' + print("%s %s" % (fn.ljust(20), summary)) + finally: + tinfoil.shutdown() return 0
Signature · 1 line
2.1.0
PATCH 11/18 devtool: ensure we change back to the original dir on error
In reply to Paul EggletonThis is just belt-and-braces but we ought to use try..finally in this kind of situation, so just do it.
--- scripts/lib/devtool/__init__.py | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-)
scripts/lib/devtool/__init__.py +14 -12
@@ -100,18 +100,20 @@ def setup_tinfoil(config_only=False, basepath=None, tracking=False): """Initialize tinfoil api from bitbake""" import scriptpath orig_cwd = os.path.abspath(os.curdir) - if basepath: - os.chdir(basepath) - bitbakepath = scriptpath.add_bitbake_lib_path() - if not bitbakepath: - logger.error("Unable to find bitbake by searching parent directory of this script or PATH") - sys.exit(1) - - import bb.tinfoil - tinfoil = bb.tinfoil.Tinfoil(tracking=tracking) - tinfoil.prepare(config_only) - tinfoil.logger.setLevel(logger.getEffectiveLevel()) - os.chdir(orig_cwd) + try: + if basepath: + os.chdir(basepath) + bitbakepath = scriptpath.add_bitbake_lib_path() + if not bitbakepath: + logger.error("Unable to find bitbake by searching parent directory of this script or PATH") + sys.exit(1) + + import bb.tinfoil + tinfoil = bb.tinfoil.Tinfoil(tracking=tracking) + tinfoil.prepare(config_only) + tinfoil.logger.setLevel(logger.getEffectiveLevel()) + finally: + os.chdir(orig_cwd) return tinfoil def get_recipe_file(cooker, pn):
Signature · 1 line
2.1.0
PATCH 12/18 lib/oe/recipeutils: check in validate_pn() for names instead of filenames
In reply to Paul EggletonEnsure that the user specifies just the name portion instead of a file name with extension. (We can't just look for . since there are recipe names such as "glib-2.0" that legitimately contain .).
--- meta/lib/oe/recipeutils.py | 2 ++ 1 file changed, 2 insertions(+)
meta/lib/oe/recipeutils.py +2 -0
@@ -398,6 +398,8 @@ def validate_pn(pn): return 'Recipe name "%s" is invalid: is a reserved keyword' % pn elif pn.startswith('pn-'): return 'Recipe name "%s" is invalid: names starting with "pn-" are reserved' % pn + elif pn.endswith(('.bb', '.bbappend', '.bbclass', '.inc', '.conf')): + return 'Recipe name "%s" is invalid: should be just a name, not a file name' % pn return ''
Signature · 1 line
2.1.0
PATCH 13/18 devtool: build: use bbappend to set PARALLEL_MAKE
In reply to Paul EggletonUse a bbappend file to set PARALLEL_MAKE instead of a postfile; this is a bit neater and only affects the specified recipe.
--- scripts/lib/devtool/build.py | 45 +++++++++++++++++++++++++------------------- 1 file changed, 26 insertions(+), 19 deletions(-)
scripts/lib/devtool/build.py +26 -19
@@ -25,16 +25,27 @@ from devtool import exec_build_env_command, check_workspace_recipe, DevtoolError logger = logging.getLogger('devtool') -def _create_conf_file(values, conf_file=None): - if not conf_file: - fd, conf_file = tempfile.mkstemp(suffix='.conf') - elif not os.path.exists(os.path.dirname(conf_file)): - logger.debug("Creating folder %s" % os.path.dirname(conf_file)) - bb.utils.mkdirhier(os.path.dirname(conf_file)) - with open(conf_file, 'w') as f: - for key, value in values.iteritems(): - f.write('%s = "%s"\n' % (key, value)) - return conf_file + +def _set_file_values(fn, values): + remaining = values.keys() + + def varfunc(varname, origvalue, op, newlines): + newvalue = values.get(varname, origvalue) + remaining.remove(varname) + return (newvalue, '=', 0, True) + + with open(fn, 'r') as f: + (updated, newlines) = bb.utils.edit_metadata(f, values, varfunc) + + for item in remaining: + updated = True + newlines.append('%s = "%s"' % (item, values[item])) + + if updated: + with open(fn, 'w') as f: + f.writelines(newlines) + return updated + def build(args, config, basepath, workspace): """Entry point for the devtool 'build' subcommand"""
@@ -42,22 +53,18 @@ def build(args, config, basepath, workspace): build_task = config.get('Build', 'build_task', 'populate_sysroot') - postfile_param = "" - postfile = "" + bbappend = workspace[args.recipename]['bbappend'] if args.disable_parallel_make: logger.info("Disabling 'make' parallelism") - postfile = os.path.join(basepath, 'conf', 'disable_parallelism.conf') - _create_conf_file({'PARALLEL_MAKE':''}, postfile) - postfile_param = "-R %s" % postfile + _set_file_values(bbappend, {'PARALLEL_MAKE': ''}) try: - exec_build_env_command(config.init_path, basepath, 'bitbake -c %s %s %s' % (build_task, postfile_param, args.recipename), watch=True) + exec_build_env_command(config.init_path, basepath, 'bitbake -c %s %s' % (build_task, args.recipename), watch=True) except bb.process.ExecutionError as e: # We've already seen the output since watch=True, so just ensure we return something to the user return e.exitcode finally: - if postfile: - logger.debug('Removing postfile') - os.remove(postfile) + if args.disable_parallel_make: + _set_file_values(bbappend, {'PARALLEL_MAKE': None}) return 0
Signature · 1 line
2.1.0
PATCH 14/18 devtool: build: enable showing default task in help
In reply to Paul EggletonEnable access to the configuration object in register_commands() so that we can read configuration values there; this allows us to show the task that will be run in the command line help for the build subcommand.
--- scripts/devtool | 1 + scripts/lib/devtool/build.py | 6 ++++-- 2 files changed, 5 insertions(+), 2 deletions(-)
scripts/devtool +1 -0
@@ -220,6 +220,7 @@ def main(): config = ConfigHandler(os.path.join(basepath, 'conf', 'devtool.conf')) if not config.read(): return -1 + context.config = config bitbake_subdir = config.get('General', 'bitbake_subdir', '') if bitbake_subdir:
scripts/lib/devtool/build.py +4 -2
@@ -46,12 +46,14 @@ def _set_file_values(fn, values): f.writelines(newlines) return updated +def _get_build_task(config): + return config.get('Build', 'build_task', 'populate_sysroot') def build(args, config, basepath, workspace): """Entry point for the devtool 'build' subcommand""" check_workspace_recipe(workspace, args.recipename) - build_task = config.get('Build', 'build_task', 'populate_sysroot') + build_task = _get_build_task(config) bbappend = workspace[args.recipename]['bbappend'] if args.disable_parallel_make:
@@ -71,7 +73,7 @@ def build(args, config, basepath, workspace): def register_commands(subparsers, context): """Register devtool subcommands from this plugin""" parser_build = subparsers.add_parser('build', help='Build a recipe', - description='Builds the specified recipe using bitbake', + description='Builds the specified recipe using bitbake (up to and including do_%s)' % _get_build_task(context.config), formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser_build.add_argument('recipename', help='Recipe to build') parser_build.add_argument('-s', '--disable-parallel-make', action="store_true", help='Disable make parallelism')
Signature · 1 line
2.1.0
PATCH 15/18 devtool: clarify help text
In reply to Paul Eggleton* Make some minor clarifications to help text
* Drop ArgumentDefaultsHelpFormatter and just put the defaults in the text itself where needed (because otherwise you get defaults shown for store_true options which is somewhat confusing).
--- scripts/devtool | 2 +- scripts/lib/devtool/build.py | 3 +-- scripts/lib/devtool/standard.py | 28 ++++++++++++---------------- scripts/lib/devtool/upgrade.py | 4 ++-- 4 files changed, 16 insertions(+), 21 deletions(-)
scripts/devtool +1 -1
@@ -258,7 +258,7 @@ def main(): if not context.fixed_setup: parser_create_workspace = subparsers.add_parser('create-workspace', - help='Set up a workspace', + help='Set up workspace in an alternative location', description='Sets up a new workspace. NOTE: other devtool subcommands will create a workspace automatically as needed, so you only need to use %(prog)s if you want to specify where the workspace should be located.') parser_create_workspace.add_argument('layerpath', nargs='?', help='Path in which the workspace layer should be created') parser_create_workspace.add_argument('--create-only', action="store_true", help='Only create the workspace layer, do not alter configuration')
scripts/lib/devtool/build.py +1 -2
@@ -73,8 +73,7 @@ def build(args, config, basepath, workspace): def register_commands(subparsers, context): """Register devtool subcommands from this plugin""" parser_build = subparsers.add_parser('build', help='Build a recipe', - description='Builds the specified recipe using bitbake (up to and including do_%s)' % _get_build_task(context.config), - formatter_class=argparse.ArgumentDefaultsHelpFormatter) + description='Builds the specified recipe using bitbake (up to and including do_%s)' % _get_build_task(context.config)) parser_build.add_argument('recipename', help='Recipe to build') parser_build.add_argument('-s', '--disable-parallel-make', action="store_true", help='Disable make parallelism') parser_build.set_defaults(func=build)
scripts/lib/devtool/standard.py +12 -16
@@ -1085,8 +1085,8 @@ def reset(args, config, basepath, workspace): def register_commands(subparsers, context): """Register devtool subcommands from this plugin""" parser_add = subparsers.add_parser('add', help='Add a new recipe', - description='Adds a new recipe') - parser_add.add_argument('recipename', help='Name for new recipe to add') + description='Adds a new recipe to the workspace to build a specified source tree') + parser_add.add_argument('recipename', help='Name for new recipe to add (just name - no version, path or extension)') parser_add.add_argument('srctree', help='Path to external source tree') group = parser_add.add_mutually_exclusive_group() group.add_argument('--same-dir', '-s', help='Build in same directory as source', action="store_true")
@@ -1098,45 +1098,41 @@ def register_commands(subparsers, context): parser_add.set_defaults(func=add) parser_modify = subparsers.add_parser('modify', help='Modify the source for an existing recipe', - description='Enables modifying the source for an existing recipe', - formatter_class=argparse.ArgumentDefaultsHelpFormatter) - parser_modify.add_argument('recipename', help='Name for recipe to edit') + description='Enables modifying the source for an existing recipe') + parser_modify.add_argument('recipename', help='Name of existing recipe to edit (just name - no version, path or extension)') parser_modify.add_argument('srctree', help='Path to external source tree') parser_modify.add_argument('--wildcard', '-w', action="store_true", help='Use wildcard for unversioned bbappend') parser_modify.add_argument('--extract', '-x', action="store_true", help='Extract source as well') group = parser_modify.add_mutually_exclusive_group() group.add_argument('--same-dir', '-s', help='Build in same directory as source', action="store_true") group.add_argument('--no-same-dir', help='Force build in a separate build directory', action="store_true") - parser_modify.add_argument('--branch', '-b', default="devtool", help='Name for development branch to checkout (only when using -x)') + parser_modify.add_argument('--branch', '-b', default="devtool", help='Name for development branch to checkout (only when using -x) (default "%(default)s")') parser_modify.set_defaults(func=modify) parser_extract = subparsers.add_parser('extract', help='Extract the source for an existing recipe', - description='Extracts the source for an existing recipe', - formatter_class=argparse.ArgumentDefaultsHelpFormatter) - parser_extract.add_argument('recipename', help='Name for recipe to extract the source for') + description='Extracts the source for an existing recipe') + parser_extract.add_argument('recipename', help='Name of recipe to extract the source for') parser_extract.add_argument('srctree', help='Path to where to extract the source tree') - parser_extract.add_argument('--branch', '-b', default="devtool", help='Name for development branch to checkout') + parser_extract.add_argument('--branch', '-b', default="devtool", help='Name for development branch to checkout (default "%(default)s")') parser_extract.add_argument('--keep-temp', action="store_true", help='Keep temporary directory (for debugging)') parser_extract.set_defaults(func=extract, no_workspace=True) parser_update_recipe = subparsers.add_parser('update-recipe', help='Apply changes from external source tree to recipe', - description='Applies changes from external source tree to a recipe (updating/adding/removing patches as necessary, or by updating SRCREV)') + description='Applies changes from external source tree to a recipe (updating/adding/removing patches as necessary, or by updating SRCREV). Note that these changes need to have been committed to the git repository in order to be recognised.') parser_update_recipe.add_argument('recipename', help='Name of recipe to update') parser_update_recipe.add_argument('--mode', '-m', choices=['patch', 'srcrev', 'auto'], default='auto', help='Update mode (where %(metavar)s is %(choices)s; default is %(default)s)', metavar='MODE') - parser_update_recipe.add_argument('--initial-rev', help='Starting revision for patches') + parser_update_recipe.add_argument('--initial-rev', help='Override starting revision for patches') parser_update_recipe.add_argument('--append', '-a', help='Write changes to a bbappend in the specified layer instead of the recipe', metavar='LAYERDIR') parser_update_recipe.add_argument('--wildcard-version', '-w', help='In conjunction with -a/--append, use a wildcard to make the bbappend apply to any recipe version', action='store_true') parser_update_recipe.add_argument('--no-remove', '-n', action="store_true", help='Don\'t remove patches, only add or update') parser_update_recipe.set_defaults(func=update_recipe) parser_status = subparsers.add_parser('status', help='Show workspace status', - description='Lists recipes currently in your workspace and the paths to their respective external source trees', - formatter_class=argparse.ArgumentDefaultsHelpFormatter) + description='Lists recipes currently in your workspace and the paths to their respective external source trees') parser_status.set_defaults(func=status) parser_reset = subparsers.add_parser('reset', help='Remove a recipe from your workspace', - description='Removes the specified recipe from your workspace (resetting its state)', - formatter_class=argparse.ArgumentDefaultsHelpFormatter) + description='Removes the specified recipe from your workspace (resetting its state)') parser_reset.add_argument('recipename', nargs='?', help='Recipe to reset') parser_reset.add_argument('--all', '-a', action="store_true", help='Reset all recipes (clear workspace)') parser_reset.add_argument('--no-clean', '-n', action="store_true", help='Don\'t clean the sysroot to remove recipe output')
scripts/lib/devtool/upgrade.py +2 -2
@@ -361,8 +361,8 @@ def upgrade(args, config, basepath, workspace): def register_commands(subparsers, context): """Register devtool subcommands from this plugin""" parser_upgrade = subparsers.add_parser('upgrade', help='Upgrade an existing recipe', - description='Upgrades an existing recipe to a new upstream version') - parser_upgrade.add_argument('recipename', help='Name for recipe to extract the source for') + description='Upgrades an existing recipe to a new upstream version. Puts the upgraded recipe file into the workspace along with any associated files, and extracts the source tree to a specified location (in case patches need rebasing or adding to as a result of the upgrade).') + parser_upgrade.add_argument('recipename', help='Name of recipe to upgrade (just name - no version, path or extension)') parser_upgrade.add_argument('srctree', help='Path to where to extract the source tree') parser_upgrade.add_argument('--version', '-V', help='Version to upgrade to (PV)') parser_upgrade.add_argument('--srcrev', '-S', help='Source revision to upgrade to (if fetching from an SCM such as git)')
Signature · 1 line
2.1.0
PATCH 16/18 lib/oe/recipeutils: refactor patch_recipe_file() to use edit_metadata()
In reply to Paul EggletonUse bb.utils.edit_metadata() to replace some of the logic in this function; this avoids us effectively having two implementations of the same thing. In the process fix the following issues:
* Insert values before any leading comments for the next variable instead of after them
* Insert overridden variables (e.g. RDEPENDS_${PN}) in the correct place
* Properly handle replacing varflag settings (e.g. SRC_URI[md5sum])
--- meta/lib/oe/recipeutils.py | 160 ++++++++++++++++++++++++----------------- scripts/lib/devtool/upgrade.py | 30 ++------ 2 files changed, 97 insertions(+), 93 deletions(-)
meta/lib/oe/recipeutils.py +93 -67
@@ -19,9 +19,9 @@ from collections import OrderedDict, defaultdict # Help us to find places to insert values -recipe_progression = ['SUMMARY', 'DESCRIPTION', 'HOMEPAGE', 'BUGTRACKER', 'SECTION', 'LICENSE', 'LIC_FILES_CHKSUM', 'PROVIDES', 'DEPENDS', 'PR', 'PV', 'SRCREV', 'SRC_URI', 'S', 'do_fetch', 'do_unpack', 'do_patch', 'EXTRA_OECONF', 'do_configure', 'EXTRA_OEMAKE', 'do_compile', 'do_install', 'do_populate_sysroot', 'INITSCRIPT', 'USERADD', 'GROUPADD', 'PACKAGES', 'FILES', 'RDEPENDS', 'RRECOMMENDS', 'RSUGGESTS', 'RPROVIDES', 'RREPLACES', 'RCONFLICTS', 'ALLOW_EMPTY', 'do_package', 'do_deploy'] +recipe_progression = ['SUMMARY', 'DESCRIPTION', 'HOMEPAGE', 'BUGTRACKER', 'SECTION', 'LICENSE', 'LIC_FILES_CHKSUM', 'PROVIDES', 'DEPENDS', 'PR', 'PV', 'SRCREV', 'SRC_URI', 'S', 'do_fetch()', 'do_unpack()', 'do_patch()', 'EXTRA_OECONF', 'do_configure()', 'EXTRA_OEMAKE', 'do_compile()', 'do_install()', 'do_populate_sysroot()', 'INITSCRIPT', 'USERADD', 'GROUPADD', 'PACKAGES', 'FILES', 'RDEPENDS', 'RRECOMMENDS', 'RSUGGESTS', 'RPROVIDES', 'RREPLACES', 'RCONFLICTS', 'ALLOW_EMPTY', 'do_package()', 'do_deploy()'] # Variables that sometimes are a bit long but shouldn't be wrapped -nowrap_vars = ['SUMMARY', 'HOMEPAGE', 'BUGTRACKER'] +nowrap_vars = ['SUMMARY', 'HOMEPAGE', 'BUGTRACKER', 'SRC_URI[md5sum]', 'SRC_URI[sha256sum]'] list_vars = ['SRC_URI', 'LIC_FILES_CHKSUM'] meta_vars = ['SUMMARY', 'DESCRIPTION', 'HOMEPAGE', 'BUGTRACKER', 'SECTION']
@@ -164,85 +164,111 @@ def patch_recipe_file(fn, values, patch=False, relpath=''): Note that some manual inspection/intervention may be required since this cannot handle all situations. """ + + import bb.utils + + recipe_progression_res = [] + recipe_progression_restrs = [] + for item in recipe_progression: + if item.endswith('()'): + key = item[:-2] + else: + key = item + restr = '%s(_[a-zA-Z0-9-_$(){}]+|\[[^\]]*\])?' % key + if item.endswith('()'): + recipe_progression_restrs.append(restr + '()') + else: + recipe_progression_restrs.append(restr) + recipe_progression_res.append(re.compile('^%s$' % restr)) + + def get_recipe_pos(variable): + for i, p in enumerate(recipe_progression_res): + if p.match(variable): + return i + return -1 + remainingnames = {} for k in values.keys(): - remainingnames[k] = recipe_progression.index(k) if k in recipe_progression else -1 + remainingnames[k] = get_recipe_pos(k) remainingnames = OrderedDict(sorted(remainingnames.iteritems(), key=lambda x: x[1])) - with tempfile.NamedTemporaryFile('w', delete=False) as tf: - def outputvalue(name): - rawtext = '%s = "%s"\n' % (name, values[name]) - if name in nowrap_vars: - tf.write(rawtext) - elif name in list_vars: - splitvalue = split_var_value(values[name], assignment=False) - if len(splitvalue) > 1: - linesplit = ' \\\n' + (' ' * (len(name) + 4)) - tf.write('%s = "%s%s"\n' % (name, linesplit.join(splitvalue), linesplit)) - else: - tf.write(rawtext) + modifying = False + + def outputvalue(name, lines, rewindcomments=False): + if values[name] is None: + return + rawtext = '%s = "%s"\n' % (name, values[name]) + addlines = [] + if name in nowrap_vars: + addlines.append(rawtext) + elif name in list_vars: + splitvalue = split_var_value(values[name], assignment=False) + if len(splitvalue) > 1: + linesplit = ' \\\n' + (' ' * (len(name) + 4)) + addlines.append('%s = "%s%s"\n' % (name, linesplit.join(splitvalue), linesplit)) else: - wrapped = textwrap.wrap(rawtext) - for wrapline in wrapped[:-1]: - tf.write('%s \\\n' % wrapline) - tf.write('%s\n' % wrapped[-1]) - - tfn = tf.name - with open(fn, 'r') as f: - # First runthrough - find existing names (so we know not to insert based on recipe_progression) - # Second runthrough - make the changes - existingnames = [] - for runthrough in [1, 2]: - currname = None - for line in f: - if not currname: - insert = False - for k in remainingnames.keys(): - for p in recipe_progression: - if re.match('^%s(_prepend|_append)*[ ?:=(]' % p, line): - if remainingnames[k] > -1 and recipe_progression.index(p) > remainingnames[k] and runthrough > 1 and not k in existingnames: - outputvalue(k) - del remainingnames[k] - break - for k in remainingnames.keys(): - if re.match('^%s[ ?:=]' % k, line): - currname = k - if runthrough == 1: - existingnames.append(k) - else: - del remainingnames[k] - break - if currname and runthrough > 1: - outputvalue(currname) - - if currname: - sline = line.rstrip() - if not sline.endswith('\\'): - currname = None - continue - if runthrough > 1: - tf.write(line) - f.seek(0) - if remainingnames: - tf.write('\n') - for k in remainingnames.keys(): - outputvalue(k) - - with open(tfn, 'U') as f: - tolines = f.readlines() + addlines.append(rawtext) + else: + wrapped = textwrap.wrap(rawtext) + for wrapline in wrapped[:-1]: + addlines.append('%s \\\n' % wrapline) + addlines.append('%s\n' % wrapped[-1]) + if rewindcomments: + # Ensure we insert the lines before any leading comments + # (that we'd want to ensure remain leading the next value) + for i, ln in reversed(list(enumerate(lines))): + if ln[0] != '#': + lines[i+1:i+1] = addlines + break + else: + lines.extend(addlines) + else: + lines.extend(addlines) + + existingnames = [] + def patch_recipe_varfunc(varname, origvalue, op, newlines): + if modifying: + # Insert anything that should come before this variable + pos = get_recipe_pos(varname) + for k in remainingnames.keys()[:]: + if remainingnames[k] > -1 and pos >= remainingnames[k] and not k in existingnames: + outputvalue(k, newlines, rewindcomments=True) + del remainingnames[k] + # Now change this variable, if it needs to be changed + if varname in existingnames: + outputvalue(varname, newlines) + del remainingnames[varname] + return None, None, 0, True + else: + if varname in values: + existingnames.append(varname) + return origvalue, None, 0, True + + # First run - establish which values we want to set are already in the file + varlist = [re.escape(item) for item in values.keys()] + with open(fn, 'r') as f: + changed, fromlines = bb.utils.edit_metadata(f, varlist, patch_recipe_varfunc) + # Second run - actually set everything + modifying = True + varlist.extend(recipe_progression_restrs) + changed, tolines = bb.utils.edit_metadata(fromlines, varlist, patch_recipe_varfunc, match_overrides=True) + + if remainingnames: + if tolines[-1].strip() != '': + tolines.append('\n') + for k in remainingnames.keys(): + outputvalue(k, tolines) + if patch: - with open(fn, 'U') as f: - fromlines = f.readlines() relfn = os.path.relpath(fn, relpath) diff = difflib.unified_diff(fromlines, tolines, 'a/%s' % relfn, 'b/%s' % relfn) - os.remove(tfn) return diff else: with open(fn, 'w') as f: f.writelines(tolines) - os.remove(tfn) return None + def localise_file_vars(fn, varfiles, varlist): """Given a list of variables and variable history (fetched with get_var_files()) find where each variable should be set/changed. This handles for example where a
scripts/lib/devtool/upgrade.py +4 -26
@@ -62,26 +62,6 @@ def _get_checksums(rf): checksums[cs] = m.group(1) return checksums -def _replace_checksums(rf, md5, sha256): - if not md5 and not sha256: - return - checksums = {'md5sum':md5, 'sha256sum':sha256} - with open(rf + ".tmp", "w+") as tmprf: - with open(rf) as f: - for line in f: - m = None - for cs in checksums.keys(): - m = re.match("^SRC_URI\[%s\].*=.*\"(.*)\"" % cs, line) - if m: - if checksums[cs]: - oldcheck = m.group(1) - newcheck = checksums[cs] - line = line.replace(oldcheck, newcheck) - break - tmprf.write(line) - os.rename(rf + ".tmp", rf) - - def _remove_patch_dirs(recipefolder): for root, dirs, files in os.walk(recipefolder): for d in dirs:
@@ -297,16 +277,14 @@ def _create_new_recipe(newpv, md5, sha256, srcrev, srcbranch, workspace, tinfoil if changed: newvalues['SRC_URI'] = ' '.join(new_src_uri) + if md5 and sha256: + newvalues['SRC_URI[md5sum]'] = md5 + newvalues['SRC_URI[sha256sum]'] = sha256 + if newvalues: rd = oe.recipeutils.parse_recipe(fullpath, None, tinfoil.config_data) oe.recipeutils.patch_recipe(rd, fullpath, newvalues) - if md5 and sha256: - # Unfortunately, oe.recipeutils.patch_recipe cannot update flags. - # once the latter feature is implemented, we should call patch_recipe - # instead of the following function - _replace_checksums(fullpath, md5, sha256) - return fullpath def upgrade(args, config, basepath, workspace):
Signature · 1 line
2.1.0
PATCH 17/18 recipetool: add setvar subcommand
In reply to Paul EggletonAdd a recipetool subcommand "setvar" to set a variable in a file. This uses our existing logic such that it doesn't matter if the variable is already set in the recipe, if it's set in the recipe or some inc file, and if the variable is not currently set that the line setting the variable gets inserted in the right place in the file.
Implements [YOCTO #7676].
--- scripts/lib/recipetool/setvar.py | 75 ++++++++++++++++++++++++++++++++++++++++ scripts/recipetool | 4 ++- 2 files changed, 78 insertions(+), 1 deletion(-) create mode 100644 scripts/lib/recipetool/setvar.py
scripts/lib/recipetool/setvar.py new file +75 -0
@@ -0,0 +1,75 @@ +# Recipe creation tool - set variable plugin +# +# Copyright (C) 2015 Intel Corporation +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import sys +import os +import argparse +import glob +import fnmatch +import re +import logging +import scriptutils + +logger = logging.getLogger('recipetool') + +tinfoil = None +plugins = None + +def tinfoil_init(instance): + global tinfoil + tinfoil = instance + +def setvar(args): + import oe.recipeutils + + if args.delete: + if args.value: + logger.error('-D/--delete and specifying a value are mutually exclusive') + return 1 + value = None + else: + if args.value is None: + logger.error('You must specify a value if not using -D/--delete') + return 1 + value = args.value + varvalues = {args.varname: value} + + if args.recipe_only: + patches = [oe.recipeutils.patch_recipe_file(args.recipefile, varvalues, patch=args.patch)] + else: + rd = oe.recipeutils.parse_recipe(args.recipefile, None, tinfoil.config_data) + if not rd: + return 1 + patches = oe.recipeutils.patch_recipe(rd, args.recipefile, varvalues, patch=args.patch) + if args.patch: + for patch in patches: + for line in patch: + sys.stdout.write(line) + return 0 + + +def register_command(subparsers): + parser_setvar = subparsers.add_parser('setvar', + help='Set a variable within a recipe', + description='Adds/updates the value a variable is set to in a recipe') + parser_setvar.add_argument('recipefile', help='Recipe file to update') + parser_setvar.add_argument('varname', help='Variable name to set') + parser_setvar.add_argument('value', nargs='?', help='New value to set the variable to') + parser_setvar.add_argument('--recipe-only', '-r', help='Do not set variable in any include file if present', action='store_true') + parser_setvar.add_argument('--patch', '-p', help='Create a patch to make the change instead of modifying the recipe', action='store_true') + parser_setvar.add_argument('--delete', '-D', help='Delete the specified value instead of setting it', action='store_true') + parser_setvar.set_defaults(func=setvar)
scripts/recipetool +3 -1
@@ -34,7 +34,7 @@ plugins = [] def tinfoil_init(parserecipes): import bb.tinfoil import logging - tinfoil = bb.tinfoil.Tinfoil() + tinfoil = bb.tinfoil.Tinfoil(tracking=True) tinfoil.prepare(not parserecipes) tinfoil.logger.setLevel(logger.getEffectiveLevel()) return tinfoil
@@ -96,7 +96,9 @@ def main(): try: if getattr(args, 'parserecipes', False): + tinfoil.config_data.disableTracking() tinfoil.parseRecipes() + tinfoil.config_data.enableTracking() ret = args.func(args) except bb.BBHandledException: ret = 1
Signature · 1 line
2.1.0
PATCH 18/18 recipetool: make plugin registration function name consistent with devtool
In reply to Paul EggletonThis should have been register_commands rather than register_command; I used register_commands in devtool so lets change this here to be consistent with that. (Since this is extensible through layers though we need to remain compatible with the old name, so fall back to that if the new function name isn't there.)
--- scripts/lib/recipetool/append.py | 2 +- scripts/lib/recipetool/create.py | 2 +- scripts/lib/recipetool/newappend.py | 2 +- scripts/lib/recipetool/setvar.py | 2 +- scripts/recipetool | 6 +++++- 5 files changed, 9 insertions(+), 5 deletions(-)
scripts/lib/recipetool/append.py +1 -1
@@ -433,7 +433,7 @@ def target_path(targetpath): return targetpath -def register_command(subparsers): +def register_commands(subparsers): common = argparse.ArgumentParser(add_help=False) common.add_argument('-m', '--machine', help='Make bbappend changes specific to a machine only', metavar='MACHINE') common.add_argument('-w', '--wildcard-version', help='Use wildcard to make the bbappend apply to any recipe version', action='store_true')
scripts/lib/recipetool/create.py +1 -1
@@ -436,7 +436,7 @@ def convert_debian(debpath): return values -def register_command(subparsers): +def register_commands(subparsers): parser_create = subparsers.add_parser('create', help='Create a new recipe', description='Creates a new recipe from a source tree')
scripts/lib/recipetool/newappend.py +1 -1
@@ -97,7 +97,7 @@ def newappend(args): print(append_path) -def register_command(subparsers): +def register_commands(subparsers): parser = subparsers.add_parser('newappend', help='Create a bbappend for the specified target in the specified layer') parser.add_argument('-w', '--wildcard-version', help='Use wildcard to make the bbappend apply to any recipe version', action='store_true')
scripts/lib/recipetool/setvar.py +1 -1
@@ -62,7 +62,7 @@ def setvar(args): return 0 -def register_command(subparsers): +def register_commands(subparsers): parser_setvar = subparsers.add_parser('setvar', help='Set a variable within a recipe', description='Adds/updates the value a variable is set to in a recipe')
scripts/recipetool +5 -1
@@ -82,7 +82,11 @@ def main(): registered = False for plugin in plugins: - if hasattr(plugin, 'register_command'): + if hasattr(plugin, 'register_commands'): + registered = True + plugin.register_commands(subparsers) + elif hasattr(plugin, 'register_command'): + # Legacy function name registered = True plugin.register_command(subparsers) if hasattr(plugin, 'tinfoil_init'):
Signature · 1 line
2.1.0