Skip to contentSkip to conversation

devtool / recipetool fixes

PATCH New Paul Eggleton 1 participant 19 messages Nov 23, 2015

lore
Share & tools
lore
#0 lore raw

cover devtool / recipetool fixes

Fix some issues I found when trying to use devtool upgrade on a git recipe; improve extraction of patch header from original commit in devtool; recipetool setvar; and other fixes and cleanups I have been working on for a while.

All devtool/recipetool tests pass with these patches (plus the two bitbake bb.utils patches I sent the other day, though this patchset should not depend on those.)

The following changes since commit 6f98c39418c60b7c0b25b30983d2e5257158a6a4:

gcc: Drop 4.8 (2015-11-16 14:59:07 +0000)

are available in the git repository at:

git://git.openembedded.org/openembedded-core-contrib paule/devtool10
http://cgit.openembedded.org/cgit.cgi/openembedded-core-contrib/log/?h=paule/devtool10

Paul Eggleton (18):
devtool: upgrade: fix removing other recipes from workspace on reset
devtool: upgrade: fix updating PV and SRCREV
devtool: upgrade: remove erroneous error when not renaming recipe
devtool: upgrade: fetch remote repository before checking out new revision
devtool: upgrade: provide a means to update the source branch lib/oe/patch: improve extraction of patch header
devtool: disable creating workspace for extract and search subcommands
devtool: package: use DEPLOY_DIR_<pkgtype> to get deploy directory
devtool: drop unused plugin_init() functions
devtool: search: print SUMMARY value
devtool: ensure we change back to the original dir on error lib/oe/recipeutils: check in validate_pn() for names instead of filenames
devtool: build: use bbappend to set PARALLEL_MAKE
devtool: build: enable showing default task in help
devtool: clarify help text
lib/oe/recipeutils: refactor patch_recipe_file() to use edit_metadata()
recipetool: add setvar subcommand
recipetool: make plugin registration function name consistent with devtool

 meta/lib/oe/patch.py                |  58 +++++++++----
 meta/lib/oe/recipeutils.py          | 162 +++++++++++++++++++++---------------
 meta/lib/oeqa/selftest/devtool.py   |   8 +-
 scripts/devtool                     |   7 +-
 scripts/lib/devtool/__init__.py     |  26 +++---
 scripts/lib/devtool/build.py        |  54 ++++++------
 scripts/lib/devtool/package.py      |  24 +++---
 scripts/lib/devtool/sdk.py          |   4 -
 scripts/lib/devtool/search.py       |  79 ++++++++++--------
 scripts/lib/devtool/standard.py     |  32 ++++---
 scripts/lib/devtool/upgrade.py      |  98 ++++++++++++----------
 scripts/lib/recipetool/append.py    |   2 +-
 scripts/lib/recipetool/create.py    |   2 +-
 scripts/lib/recipetool/newappend.py |   7 +-
 scripts/lib/recipetool/setvar.py    |  75 +++++++++++++++++
 scripts/recipetool                  |  10 ++-
 16 files changed, 397 insertions(+), 251 deletions(-)
 create mode 100644 scripts/lib/recipetool/setvar.py
Signature · 1 line
2.1.0
#1 lore raw

PATCH 1/18 devtool: upgrade: fix removing other recipes from workspace on reset

If you did a "devtool add" followed by "devtool upgrade" and then did a "devtool reset" on the recipe you upgraded, the first recipe would also be deleted from the workspace - this was because we were erroneously adding the entire "recipes" subdirectory and its contents to be tracked for removal on reset. Remove the unnecessary call to os.path.dirname() that caused this.

---
 scripts/lib/devtool/standard.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
scripts/lib/devtool/standard.py +1 -1
index 1285974..ace3a4b 100644
@@ -456,7 +456,7 @@ def _add_md5(config, recipename, filename):
             f.write('%s|%s|%s\n' % (recipename, os.path.relpath(fn, config.workspace_path), md5))
 
     if os.path.isdir(filename):
-        for root, _, files in os.walk(os.path.dirname(filename)):
+        for root, _, files in os.walk(filename):
             for f in files:
                 addfile(os.path.join(root, f))
     else:
Signature · 1 line
2.1.0
#2 lore raw

PATCH 2/18 devtool: upgrade: fix updating PV and SRCREV

In reply to Paul Eggleton

This code was clearly never tested. Fix the following issues:
* Actually set SRCREV if it's been specified
* Enable history tracking and reparse so that we handle if variables are set in an inc file next to the recipe
* Use a more accurate check for PV being in the recipe which will work if it's in an inc file next to the recipe

---
 scripts/lib/devtool/upgrade.py | 34 ++++++++++++++++++++--------------
 1 file changed, 20 insertions(+), 14 deletions(-)
scripts/lib/devtool/upgrade.py +20 -14
index d387623..ace0b76 100644
@@ -91,15 +91,13 @@ def _remove_patch_dirs(recipefolder):
         for d in dirs:
             shutil.rmtree(os.path.join(root,d))
 
-def _recipe_contains(rf, var):
-    import re
-    found = False
-    with open(rf) as f:
-        for line in f:
-            if re.match("^%s.*=.*" % var, line):
-                found = True
-                break
-    return found
+def _recipe_contains(rd, var):
+    rf = rd.getVar('FILE', True)
+    varfiles = oe.recipeutils.get_var_files(rf, [var], rd)
+    for var, fn in varfiles.iteritems():
+        if fn and fn.startswith(os.path.dirname(rf) + os.sep):
+            return True
+    return False
 
 def _rename_recipe_dirs(oldpv, newpv, path):
     for root, dirs, files in os.walk(path):
@@ -257,7 +255,7 @@ def _extract_new_source(newpv, srctree, no_patch, srcrev, branch, keep_temp, tin
 
     return (rev, md5, sha256)
 
-def _create_new_recipe(newpv, md5, sha256, workspace, rd):
+def _create_new_recipe(newpv, md5, sha256, srcrev, workspace, tinfoil, rd):
     """Creates the new recipe under workspace"""
     crd = rd.createCopy()
 
@@ -271,8 +269,16 @@ def _create_new_recipe(newpv, md5, sha256, workspace, rd):
         newpv = oldpv
     fullpath = _rename_recipe_files(bpn, oldpv, newpv, path)
 
-    if _recipe_contains(fullpath, 'PV') and newpv != oldpv:
-        oe.recipeutils.patch_recipe(d, fullpath, {'PV':newpv})
+    newvalues = {}
+    if _recipe_contains(rd, 'PV') and newpv != oldpv:
+        newvalues['PV'] = newpv
+
+    if srcrev:
+        newvalues['SRCREV'] = srcrev
+
+    if newvalues:
+        rd = oe.recipeutils.parse_recipe(fullpath, None, tinfoil.config_data)
+        oe.recipeutils.patch_recipe(rd, fullpath, newvalues)
 
     if md5 and sha256:
         # Unfortunately, oe.recipeutils.patch_recipe cannot update flags.
@@ -294,7 +300,7 @@ def upgrade(args, config, basepath, workspace):
     if reason:
         raise DevtoolError(reason)
 
-    tinfoil = setup_tinfoil(basepath=basepath)
+    tinfoil = setup_tinfoil(basepath=basepath, tracking=True)
 
     rd = parse_recipe(config, tinfoil, args.recipename, True)
     if not rd:
@@ -316,7 +322,7 @@ def upgrade(args, config, basepath, workspace):
         rev2, md5, sha256 = _extract_new_source(args.version, args.srctree, args.no_patch,
                                                 args.srcrev, args.branch, args.keep_temp,
                                                 tinfoil, rd)
-        rf = _create_new_recipe(args.version, md5, sha256, config.workspace_path, rd)
+        rf = _create_new_recipe(args.version, md5, sha256, args.srcrev, config.workspace_path, tinfoil, rd)
     except bb.process.CmdError as e:
         _upgrade_error(e, rf, args.srctree)
     except DevtoolError as e:
Signature · 1 line
2.1.0
#3 lore raw

PATCH 3/18 devtool: upgrade: remove erroneous error when not renaming recipe

In reply to Paul Eggleton

If we're upgrading a git recipe the recipe file usually won't need renaming; for some unknown reason we were throwing an error here which isn't correct.

---
 scripts/lib/devtool/upgrade.py | 1 -
 1 file changed, 1 deletion(-)
scripts/lib/devtool/upgrade.py +0 -1
index ace0b76..f033fc8 100644
@@ -117,7 +117,6 @@ def _rename_recipe_file(bpn, oldpv, newpv, path):
         recipe = "%s_git.bb" % bpn
         if os.path.isfile(os.path.join(path, recipe)):
             newrecipe = recipe
-            raise DevtoolError("Original recipe not found on workspace")
     return os.path.join(path, newrecipe)
 
 def _rename_recipe_files(bpn, oldpv, newpv, path):
Signature · 1 line
2.1.0
#4 lore raw

PATCH 4/18 devtool: upgrade: fetch remote repository before checking out new revision

In reply to Paul Eggleton

If we're upgrading a recipe that fetches from git, and we've simply fetched a tarball of the repo instead of directly from the upstream repo (this can happen if you have PREMIRRORS set up as in poky with a core recipe, e.g. kernelshark) then we won't have any new revisions, and the checkout will fail with "fatal: reference is not a tree: <hash>". To avoid this, do a "git fetch" before checking out the new revision.

---
 scripts/lib/devtool/upgrade.py | 1 +
 1 file changed, 1 insertion(+)
scripts/lib/devtool/upgrade.py +1 -0
index f033fc8..6bac44b 100644
@@ -199,6 +199,7 @@ def _extract_new_source(newpv, srctree, no_patch, srcrev, branch, keep_temp, tin
     if srcrev:
         rev = srcrev
     if uri.startswith('git://'):
+        __run('git fetch')
         __run('git checkout %s' % rev)
         __run('git tag -f devtool-base-new')
         md5 = None
Signature · 1 line
2.1.0
#5 lore raw

PATCH 5/18 devtool: upgrade: provide a means to update the source branch

In reply to Paul Eggleton

If you're upgrading a git recipe to a revision on a release branch that's different to the branch for the current revision, then you'll need to update the branch parameter in SRC_URI, so add a --srcbranch/-B command-line parameter to let you do that easily. It handles both when the branch is stated verbatim in the recipe, and when a reference to another variable is used (a common convention is to use a SRCBRANCH variable for this, though the code doesn't care what variable is used if any).

---
 scripts/lib/devtool/upgrade.py | 32 ++++++++++++++++++++++++++++++--
 1 file changed, 30 insertions(+), 2 deletions(-)
scripts/lib/devtool/upgrade.py +30 -2
index 6bac44b..9d9ab0f 100644
@@ -255,7 +255,7 @@ def _extract_new_source(newpv, srctree, no_patch, srcrev, branch, keep_temp, tin
 
     return (rev, md5, sha256)
 
-def _create_new_recipe(newpv, md5, sha256, srcrev, workspace, tinfoil, rd):
+def _create_new_recipe(newpv, md5, sha256, srcrev, srcbranch, workspace, tinfoil, rd):
     """Creates the new recipe under workspace"""
     crd = rd.createCopy()
 
@@ -276,6 +276,31 @@ def _create_new_recipe(newpv, md5, sha256, srcrev, workspace, tinfoil, rd):
     if srcrev:
         newvalues['SRCREV'] = srcrev
 
+    if srcbranch:
+        src_uri = oe.recipeutils.split_var_value(rd.getVar('SRC_URI', False) or '')
+        changed = False
+        replacing = True
+        new_src_uri = []
+        for entry in src_uri:
+            scheme, network, path, user, passwd, params = bb.fetch2.decodeurl(entry)
+            if replacing and scheme in ['git', 'gitsm']:
+                branch = params.get('branch', 'master')
+                if rd.expand(branch) != srcbranch:
+                    # Handle case where branch is set through a variable
+                    res = re.match(r'\$\{([^}@]+)\}', branch)
+                    if res:
+                        newvalues[res.group(1)] = srcbranch
+                        # We know we won't change SRC_URI now, so break out
+                        break
+                    else:
+                        params['branch'] = srcbranch
+                        entry = bb.fetch2.encodeurl((scheme, network, path, user, passwd, params))
+                        changed = True
+                replacing = False
+            new_src_uri.append(entry)
+        if changed:
+            newvalues['SRC_URI'] = ' '.join(new_src_uri)
+
     if newvalues:
         rd = oe.recipeutils.parse_recipe(fullpath, None, tinfoil.config_data)
         oe.recipeutils.patch_recipe(rd, fullpath, newvalues)
@@ -295,6 +320,8 @@ def upgrade(args, config, basepath, workspace):
         raise DevtoolError("recipe %s is already in your workspace" % args.recipename)
     if not args.version and not args.srcrev:
         raise DevtoolError("You must provide a version using the --version/-V option, or for recipes that fetch from an SCM such as git, the --srcrev/-S option")
+    if args.srcbranch and not args.srcrev:
+        raise DevtoolError("If you specify --srcbranch/-B then you must use --srcrev/-S to specify the revision" % args.recipename)
 
     reason = oe.recipeutils.validate_pn(args.recipename)
     if reason:
@@ -322,7 +349,7 @@ def upgrade(args, config, basepath, workspace):
         rev2, md5, sha256 = _extract_new_source(args.version, args.srctree, args.no_patch,
                                                 args.srcrev, args.branch, args.keep_temp,
                                                 tinfoil, rd)
-        rf = _create_new_recipe(args.version, md5, sha256, args.srcrev, config.workspace_path, tinfoil, rd)
+        rf = _create_new_recipe(args.version, md5, sha256, args.srcrev, args.srcbranch, config.workspace_path, tinfoil, rd)
     except bb.process.CmdError as e:
         _upgrade_error(e, rf, args.srctree)
     except DevtoolError as e:
@@ -343,6 +370,7 @@ def register_commands(subparsers, context):
     parser_upgrade.add_argument('srctree', help='Path to where to extract the source tree')
     parser_upgrade.add_argument('--version', '-V', help='Version to upgrade to (PV)')
     parser_upgrade.add_argument('--srcrev', '-S', help='Source revision to upgrade to (if fetching from an SCM such as git)')
+    parser_upgrade.add_argument('--srcbranch', '-B', help='Branch in source repository containing the revision to use (if fetching from an SCM such as git)')
     parser_upgrade.add_argument('--branch', '-b', default="devtool", help='Name for new development branch to checkout (default "%(default)s")')
     parser_upgrade.add_argument('--no-patch', action="store_true", help='Do not apply patches from the recipe to the new source code')
     group = parser_upgrade.add_mutually_exclusive_group()
Signature · 1 line
2.1.0
#6 lore raw

PATCH 6/18 lib/oe/patch: improve extraction of patch header

In reply to Paul Eggleton

For patches that we have to extract the header information by hand (i.e. will not apply with "git am"), make the following improvements:

* If we can't extract author/date/subject, then try to do so from the commit that added the patch in git (assuming the metadata is tracked by git)
* Take only first Signed-off-by line instead of last
* Accept any case for "Signed-off-by" in case author has typed it by hand
* Improve conditional - we can skip the other cases if one matches

Implements [YOCTO #7624].

---
 meta/lib/oe/patch.py | 58 ++++++++++++++++++++++++++++++++++++++--------------
 1 file changed, 43 insertions(+), 15 deletions(-)
meta/lib/oe/patch.py +43 -15
index 2bf501e..2255ce4 100644
@@ -282,33 +282,32 @@ class GitApplyTree(PatchTree):
         return lines
 
     @staticmethod
-    def prepareCommit(patchfile):
-        """
-        Prepare a git commit command line based on the header from a patch file
-        (typically this is useful for patches that cannot be applied with "git am" due to formatting)
-        """
-        import tempfile
+    def decodeAuthor(line):
+        from email.header import decode_header
+        authorval = line.split(':', 1)[1].strip().replace('"', '')
+        return decode_header(authorval)[0][0]
+
+    @staticmethod
+    def interpretPatchHeader(headerlines):
         import re
         author_re = re.compile('[\S ]+ <\S+@\S+\.\S+>')
-        # Process patch header and extract useful information
-        lines = GitApplyTree.extractPatchHeader(patchfile)
         outlines = []
         author = None
         date = None
-        for line in lines:
+        subject = None
+        for line in headerlines:
             if line.startswith('Subject: '):
                 subject = line.split(':', 1)[1]
                 # Remove any [PATCH][oe-core] etc.
                 subject = re.sub(r'\[.+?\]\s*', '', subject)
-                outlines.insert(0, '%s\n\n' % subject.strip())
                 continue
-            if line.startswith('From: ') or line.startswith('Author: '):
-                authorval = line.split(':', 1)[1].strip().replace('"', '')
+            elif line.startswith('From: ') or line.startswith('Author: '):
+                authorval = GitApplyTree.decodeAuthor(line)
                 # git is fussy about author formatting i.e. it must be Name <email@domain>
                 if author_re.match(authorval):
                     author = authorval
                     continue
-            if line.startswith('Date: '):
+            elif line.startswith('Date: '):
                 if date is None:
                     dateval = line.split(':', 1)[1].strip()
                     # Very crude check for date format, since git will blow up if it's not in the right
@@ -316,12 +315,41 @@ class GitApplyTree(PatchTree):
                     if len(dateval) > 12:
                         date = dateval
                 continue
-            if line.startswith('Signed-off-by: '):
-                authorval = line.split(':', 1)[1].strip().replace('"', '')
+            elif not author and line.lower().startswith('signed-off-by: '):
+                authorval = GitApplyTree.decodeAuthor(line)
                 # git is fussy about author formatting i.e. it must be Name <email@domain>
                 if author_re.match(authorval):
                     author = authorval
             outlines.append(line)
+        return outlines, author, date, subject
+
+    @staticmethod
+    def prepareCommit(patchfile):
+        """
+        Prepare a git commit command line based on the header from a patch file
+        (typically this is useful for patches that cannot be applied with "git am" due to formatting)
+        """
+        import tempfile
+        # Process patch header and extract useful information
+        lines = GitApplyTree.extractPatchHeader(patchfile)
+        outlines, author, date, subject = GitApplyTree.interpretPatchHeader(lines)
+        if not author or not subject:
+            try:
+                shellcmd = ["git", "log", "--format=email", "--diff-filter=A", "--", patchfile]
+                out = runcmd(["sh", "-c", " ".join(shellcmd)], os.path.dirname(patchfile))
+            except CmdError:
+                out = None
+            if out:
+                _, newauthor, newdate, newsubject = GitApplyTree.interpretPatchHeader(out.splitlines())
+                if not author or not date:
+                    # These really need to go together
+                    author = newauthor
+                    date = newdate
+                if not subject:
+                    subject = newsubject
+        if subject:
+            outlines.insert(0, '%s\n\n' % subject.strip())
+
         # Write out commit message to a file
         with tempfile.NamedTemporaryFile('w', delete=False) as tf:
             tmpfile = tf.name
Signature · 1 line
2.1.0
#7 lore raw

PATCH 7/18 devtool: disable creating workspace for extract and search subcommands

In reply to Paul Eggleton

For subcommands that don't actually involve the workspace, don't auto-create the workspace.

---
 meta/lib/oeqa/selftest/devtool.py | 8 ++++----
 scripts/devtool                   | 4 ++--
 scripts/lib/devtool/search.py     | 2 +-
 scripts/lib/devtool/standard.py   | 2 +-
 4 files changed, 8 insertions(+), 8 deletions(-)
meta/lib/oeqa/selftest/devtool.py +4 -4
index dcdef5a..0a44ae7 100644
@@ -823,10 +823,10 @@ class DevtoolTests(DevtoolBase):
         tempdir = tempfile.mkdtemp(prefix='devtoolqa')
         # Try devtool extract
         self.track_for_cleanup(tempdir)
-        self.track_for_cleanup(self.workspacedir)
-        self.add_command_to_tearDown('bitbake-layers remove-layer */workspace')
         result = runCmd('devtool extract remake %s' % tempdir)
         self.assertTrue(os.path.exists(os.path.join(tempdir, 'Makefile.am')), 'Extracted source could not be found')
+        # devtool extract shouldn't create the workspace
+        self.assertFalse(os.path.exists(self.workspacedir))
         self._check_src_repo(tempdir)
 
     @testcase(1379)
@@ -834,10 +834,10 @@ class DevtoolTests(DevtoolBase):
         tempdir = tempfile.mkdtemp(prefix='devtoolqa')
         # Try devtool extract
         self.track_for_cleanup(tempdir)
-        self.track_for_cleanup(self.workspacedir)
-        self.add_command_to_tearDown('bitbake-layers remove-layer */workspace')
         result = runCmd('devtool extract virtual/libx11 %s' % tempdir)
         self.assertTrue(os.path.exists(os.path.join(tempdir, 'Makefile.am')), 'Extracted source could not be found')
+        # devtool extract shouldn't create the workspace
+        self.assertFalse(os.path.exists(self.workspacedir))
         self._check_src_repo(tempdir)
 
     @testcase(1168)
scripts/devtool +2 -2
index e4d9db3..2a5a3d4 100755
@@ -261,7 +261,7 @@ def main():
                                                         description='Sets up a new workspace. NOTE: other devtool subcommands will create a workspace automatically as needed, so you only need to use %(prog)s if you want to specify where the workspace should be located.')
         parser_create_workspace.add_argument('layerpath', nargs='?', help='Path in which the workspace layer should be created')
         parser_create_workspace.add_argument('--create-only', action="store_true", help='Only create the workspace layer, do not alter configuration')
-        parser_create_workspace.set_defaults(func=create_workspace)
+        parser_create_workspace.set_defaults(func=create_workspace, no_workspace=True)
 
     for plugin in plugins:
         if hasattr(plugin, 'register_commands'):
@@ -269,7 +269,7 @@ def main():
 
     args = parser.parse_args(unparsed_args, namespace=global_args)
 
-    if args.subparser_name != 'create-workspace':
+    if not getattr(args, 'no_workspace', False):
         read_workspace()
 
     try:
scripts/lib/devtool/search.py +1 -1
index c2f420c..1c8eaff 100644
@@ -77,4 +77,4 @@ def register_commands(subparsers, context):
     parser_search = subparsers.add_parser('search', help='Search available recipes',
                                             description='Searches for available target recipes. Matches on recipe name, package name, description and installed files, and prints the recipe name on match.')
     parser_search.add_argument('keyword', help='Keyword to search for (regular expression syntax allowed)')
-    parser_search.set_defaults(func=search)
+    parser_search.set_defaults(func=search, no_workspace=True)
scripts/lib/devtool/standard.py +1 -1
index ace3a4b..bc92456 100644
@@ -1117,7 +1117,7 @@ def register_commands(subparsers, context):
     parser_extract.add_argument('srctree', help='Path to where to extract the source tree')
     parser_extract.add_argument('--branch', '-b', default="devtool", help='Name for development branch to checkout')
     parser_extract.add_argument('--keep-temp', action="store_true", help='Keep temporary directory (for debugging)')
-    parser_extract.set_defaults(func=extract)
+    parser_extract.set_defaults(func=extract, no_workspace=True)
 
     parser_update_recipe = subparsers.add_parser('update-recipe', help='Apply changes from external source tree to recipe',
                                        description='Applies changes from external source tree to a recipe (updating/adding/removing patches as necessary, or by updating SRCREV)')
Signature · 1 line
2.1.0
#8 lore raw

PATCH 8/18 devtool: package: use DEPLOY_DIR_<pkgtype> to get deploy directory

In reply to Paul Eggleton

Rather than reconstructing the output path for packages, use the proper variable.

---
 scripts/lib/devtool/package.py | 20 ++++++++++++--------
 1 file changed, 12 insertions(+), 8 deletions(-)
scripts/lib/devtool/package.py +12 -8
index b8d8423..53d3dc7 100644
@@ -32,14 +32,17 @@ def package(args, config, basepath, workspace):
     """Entry point for the devtool 'package' subcommand"""
     check_workspace_recipe(workspace, args.recipename)
 
-    image_pkgtype = config.get('Package', 'image_pkgtype', '')
-    if not image_pkgtype:
-        tinfoil = setup_tinfoil(basepath=basepath)
-        try:
-            tinfoil.prepare(config_only=True)
+    tinfoil = setup_tinfoil(basepath=basepath)
+    try:
+        tinfoil.prepare(config_only=True)
+
+        image_pkgtype = config.get('Package', 'image_pkgtype', '')
+        if not image_pkgtype:
             image_pkgtype = tinfoil.config_data.getVar('IMAGE_PKGTYPE', True)
-        finally:
-            tinfoil.shutdown()
+
+        deploy_dir_pkg = tinfoil.config_data.getVar('DEPLOY_DIR_%s' % image_pkgtype.upper(), True)
+    finally:
+        tinfoil.shutdown()
 
     package_task = config.get('Package', 'package_task', 'package_write_%s' % image_pkgtype)
     try:
@@ -47,7 +50,8 @@ def package(args, config, basepath, workspace):
     except bb.process.ExecutionError as e:
         # We've already seen the output since watch=True, so just ensure we return something to the user
         return e.exitcode
-    logger.info('Your packages are in %s/tmp/deploy/%s' % (basepath, image_pkgtype))
+
+    logger.info('Your packages are in %s' % deploy_dir_pkg)
 
     return 0
 
Signature · 1 line
2.1.0
#9 lore raw

PATCH 9/18 devtool: drop unused plugin_init() functions

In reply to Paul Eggleton

This function is no longer required to be defined for a plugin, so drop it where it's a no-op.

---
 scripts/lib/devtool/build.py        | 4 ----
 scripts/lib/devtool/package.py      | 4 ----
 scripts/lib/devtool/sdk.py          | 4 ----
 scripts/lib/devtool/upgrade.py      | 4 ----
 scripts/lib/recipetool/newappend.py | 5 -----
 5 files changed, 21 deletions(-)
scripts/lib/devtool/build.py +0 -4
index 9b58858..62d8599 100644
@@ -25,10 +25,6 @@ from devtool import exec_build_env_command, check_workspace_recipe, DevtoolError
 
 logger = logging.getLogger('devtool')
 
-def plugin_init(pluginlist):
-    """Plugin initialization"""
-    pass
-
 def _create_conf_file(values, conf_file=None):
     if not conf_file:
         fd, conf_file = tempfile.mkstemp(suffix='.conf')
scripts/lib/devtool/package.py +0 -4
index 53d3dc7..a296fce 100644
@@ -24,10 +24,6 @@ from devtool import exec_build_env_command, setup_tinfoil, check_workspace_recip
 
 logger = logging.getLogger('devtool')
 
-def plugin_init(pluginlist):
-    """Plugin initialization"""
-    pass
-
 def package(args, config, basepath, workspace):
     """Entry point for the devtool 'package' subcommand"""
     check_workspace_recipe(workspace, args.recipename)
scripts/lib/devtool/sdk.py +0 -4
index 7a842af..85c0fb1 100644
@@ -11,10 +11,6 @@ from devtool import exec_build_env_command, setup_tinfoil, DevtoolError
 
 logger = logging.getLogger('devtool')
 
-def plugin_init(pluginlist):
-    """Plugin initialization"""
-    pass
-
 def parse_locked_sigs(sigfile_path):
     """Return <pn:task>:<hash> dictionary"""
     sig_dict = {}
scripts/lib/devtool/upgrade.py +0 -4
index 9d9ab0f..761aa99 100644
@@ -33,10 +33,6 @@ from devtool import exec_build_env_command, setup_tinfoil, DevtoolError, parse_r
 
 logger = logging.getLogger('devtool')
 
-def plugin_init(pluginlist):
-    """Plugin initialization"""
-    pass
-
 def _run(cmd, cwd=''):
     logger.debug("Running command %s> %s" % (cwd,cmd))
     return bb.process.run('%s' % cmd, cwd=cwd)
scripts/lib/recipetool/newappend.py +0 -5
index 77b74cb..4eeac0e 100644
@@ -32,11 +32,6 @@ logger = logging.getLogger('recipetool')
 tinfoil = None
 
 
-def plugin_init(pluginlist):
-    # Don't need to do anything here right now, but plugins must have this function defined
-    pass
-
-
 def tinfoil_init(instance):
     global tinfoil
     tinfoil = instance
Signature · 1 line
2.1.0
#10 lore raw

PATCH 10/18 devtool: search: print SUMMARY value

In reply to Paul Eggleton

Print the SUMMARY value for each matched item assuming it's not the default.

---
 scripts/lib/devtool/search.py | 77 +++++++++++++++++++++++--------------------
 1 file changed, 42 insertions(+), 35 deletions(-)
scripts/lib/devtool/search.py +42 -35
index 1c8eaff..2ea4462 100644
@@ -22,53 +22,60 @@ import bb
 import logging
 import argparse
 import re
-from devtool import setup_tinfoil, DevtoolError
+from devtool import setup_tinfoil, parse_recipe, DevtoolError
 
 logger = logging.getLogger('devtool')
 
 def search(args, config, basepath, workspace):
     """Entry point for the devtool 'search' subcommand"""
 
-    tinfoil = setup_tinfoil(config_only=True, basepath=basepath)
-    pkgdata_dir = tinfoil.config_data.getVar('PKGDATA_DIR', True)
-    tinfoil.shutdown()
+    tinfoil = setup_tinfoil(config_only=False, basepath=basepath)
+    try:
+        pkgdata_dir = tinfoil.config_data.getVar('PKGDATA_DIR', True)
+        defsummary = tinfoil.config_data.getVar('SUMMARY', False) or ''
 
-    keyword_rc = re.compile(args.keyword)
+        keyword_rc = re.compile(args.keyword)
 
-    for fn in os.listdir(pkgdata_dir):
-        pfn = os.path.join(pkgdata_dir, fn)
-        if not os.path.isfile(pfn):
-            continue
+        for fn in os.listdir(pkgdata_dir):
+            pfn = os.path.join(pkgdata_dir, fn)
+            if not os.path.isfile(pfn):
+                continue
 
-        packages = []
-        match = False
-        if keyword_rc.search(fn):
-            match = True
+            packages = []
+            match = False
+            if keyword_rc.search(fn):
+                match = True
 
-        if not match:
-            with open(pfn, 'r') as f:
-                for line in f:
-                    if line.startswith('PACKAGES:'):
-                        packages = line.split(':', 1)[1].strip().split()
+            if not match:
+                with open(pfn, 'r') as f:
+                    for line in f:
+                        if line.startswith('PACKAGES:'):
+                            packages = line.split(':', 1)[1].strip().split()
 
-            for pkg in packages:
-                if keyword_rc.search(pkg):
-                    match = True
-                    break
-                if os.path.exists(os.path.join(pkgdata_dir, 'runtime', pkg + '.packaged')):
-                    with open(os.path.join(pkgdata_dir, 'runtime', pkg), 'r') as f:
-                        for line in f:
-                            if ': ' in line:
-                                splitline = line.split(':', 1)
-                                key = splitline[0]
-                                value = splitline[1].strip()
-                            if key in ['PKG_%s' % pkg, 'DESCRIPTION', 'FILES_INFO'] or key.startswith('FILERPROVIDES_'):
-                                if keyword_rc.search(value):
-                                    match = True
-                                    break
+                for pkg in packages:
+                    if keyword_rc.search(pkg):
+                        match = True
+                        break
+                    if os.path.exists(os.path.join(pkgdata_dir, 'runtime', pkg + '.packaged')):
+                        with open(os.path.join(pkgdata_dir, 'runtime', pkg), 'r') as f:
+                            for line in f:
+                                if ': ' in line:
+                                    splitline = line.split(':', 1)
+                                    key = splitline[0]
+                                    value = splitline[1].strip()
+                                if key in ['PKG_%s' % pkg, 'DESCRIPTION', 'FILES_INFO'] or key.startswith('FILERPROVIDES_'):
+                                    if keyword_rc.search(value):
+                                        match = True
+                                        break
 
-        if match:
-            print(fn)
+            if match:
+                rd = parse_recipe(config, tinfoil, fn, True)
+                summary = rd.getVar('SUMMARY', True)
+                if summary == rd.expand(defsummary):
+                    summary = ''
+                print("%s  %s" % (fn.ljust(20), summary))
+    finally:
+        tinfoil.shutdown()
 
     return 0
 
Signature · 1 line
2.1.0
#11 lore raw

PATCH 11/18 devtool: ensure we change back to the original dir on error

In reply to Paul Eggleton

This is just belt-and-braces but we ought to use try..finally in this kind of situation, so just do it.

---
 scripts/lib/devtool/__init__.py | 26 ++++++++++++++------------
 1 file changed, 14 insertions(+), 12 deletions(-)
scripts/lib/devtool/__init__.py +14 -12
index 50604e6..e617d60 100644
@@ -100,18 +100,20 @@ def setup_tinfoil(config_only=False, basepath=None, tracking=False):
     """Initialize tinfoil api from bitbake"""
     import scriptpath
     orig_cwd = os.path.abspath(os.curdir)
-    if basepath:
-        os.chdir(basepath)
-    bitbakepath = scriptpath.add_bitbake_lib_path()
-    if not bitbakepath:
-        logger.error("Unable to find bitbake by searching parent directory of this script or PATH")
-        sys.exit(1)
-
-    import bb.tinfoil
-    tinfoil = bb.tinfoil.Tinfoil(tracking=tracking)
-    tinfoil.prepare(config_only)
-    tinfoil.logger.setLevel(logger.getEffectiveLevel())
-    os.chdir(orig_cwd)
+    try:
+        if basepath:
+            os.chdir(basepath)
+        bitbakepath = scriptpath.add_bitbake_lib_path()
+        if not bitbakepath:
+            logger.error("Unable to find bitbake by searching parent directory of this script or PATH")
+            sys.exit(1)
+
+        import bb.tinfoil
+        tinfoil = bb.tinfoil.Tinfoil(tracking=tracking)
+        tinfoil.prepare(config_only)
+        tinfoil.logger.setLevel(logger.getEffectiveLevel())
+    finally:
+        os.chdir(orig_cwd)
     return tinfoil
 
 def get_recipe_file(cooker, pn):
Signature · 1 line
2.1.0
#12 lore raw

PATCH 12/18 lib/oe/recipeutils: check in validate_pn() for names instead of filenames

In reply to Paul Eggleton

Ensure that the user specifies just the name portion instead of a file name with extension. (We can't just look for . since there are recipe names such as "glib-2.0" that legitimately contain .).

---
 meta/lib/oe/recipeutils.py | 2 ++
 1 file changed, 2 insertions(+)
meta/lib/oe/recipeutils.py +2 -0
index 119a688..8918fac 100644
@@ -398,6 +398,8 @@ def validate_pn(pn):
         return 'Recipe name "%s" is invalid: is a reserved keyword' % pn
     elif pn.startswith('pn-'):
         return 'Recipe name "%s" is invalid: names starting with "pn-" are reserved' % pn
+    elif pn.endswith(('.bb', '.bbappend', '.bbclass', '.inc', '.conf')):
+        return 'Recipe name "%s" is invalid: should be just a name, not a file name' % pn
     return ''
 
 
Signature · 1 line
2.1.0
#14 lore raw

PATCH 14/18 devtool: build: enable showing default task in help

In reply to Paul Eggleton

Enable access to the configuration object in register_commands() so that we can read configuration values there; this allows us to show the task that will be run in the command line help for the build subcommand.

---
 scripts/devtool              | 1 +
 scripts/lib/devtool/build.py | 6 ++++--
 2 files changed, 5 insertions(+), 2 deletions(-)
scripts/devtool +1 -0
index 2a5a3d4..1fcb42c 100755
@@ -220,6 +220,7 @@ def main():
     config = ConfigHandler(os.path.join(basepath, 'conf', 'devtool.conf'))
     if not config.read():
         return -1
+    context.config = config
 
     bitbake_subdir = config.get('General', 'bitbake_subdir', '')
     if bitbake_subdir:
scripts/lib/devtool/build.py +4 -2
index 14f55e0..0b8e88b 100644
@@ -46,12 +46,14 @@ def _set_file_values(fn, values):
             f.writelines(newlines)
     return updated
 
+def _get_build_task(config):
+    return config.get('Build', 'build_task', 'populate_sysroot')
 
 def build(args, config, basepath, workspace):
     """Entry point for the devtool 'build' subcommand"""
     check_workspace_recipe(workspace, args.recipename)
 
-    build_task = config.get('Build', 'build_task', 'populate_sysroot')
+    build_task = _get_build_task(config)
 
     bbappend = workspace[args.recipename]['bbappend']
     if args.disable_parallel_make:
@@ -71,7 +73,7 @@ def build(args, config, basepath, workspace):
 def register_commands(subparsers, context):
     """Register devtool subcommands from this plugin"""
     parser_build = subparsers.add_parser('build', help='Build a recipe',
-                                         description='Builds the specified recipe using bitbake',
+                                         description='Builds the specified recipe using bitbake (up to and including do_%s)' % _get_build_task(context.config),
                                          formatter_class=argparse.ArgumentDefaultsHelpFormatter)
     parser_build.add_argument('recipename', help='Recipe to build')
     parser_build.add_argument('-s', '--disable-parallel-make', action="store_true", help='Disable make parallelism')
Signature · 1 line
2.1.0
#13 lore raw

PATCH 13/18 devtool: build: use bbappend to set PARALLEL_MAKE

In reply to Paul Eggleton

Use a bbappend file to set PARALLEL_MAKE instead of a postfile; this is a bit neater and only affects the specified recipe.

---
 scripts/lib/devtool/build.py | 45 +++++++++++++++++++++++++-------------------
 1 file changed, 26 insertions(+), 19 deletions(-)
scripts/lib/devtool/build.py +26 -19
index 62d8599..14f55e0 100644
@@ -25,16 +25,27 @@ from devtool import exec_build_env_command, check_workspace_recipe, DevtoolError
 
 logger = logging.getLogger('devtool')
 
-def _create_conf_file(values, conf_file=None):
-    if not conf_file:
-        fd, conf_file = tempfile.mkstemp(suffix='.conf')
-    elif not os.path.exists(os.path.dirname(conf_file)):
-        logger.debug("Creating folder %s" % os.path.dirname(conf_file))
-        bb.utils.mkdirhier(os.path.dirname(conf_file))
-    with open(conf_file, 'w') as f:
-        for key, value in values.iteritems():
-            f.write('%s = "%s"\n' % (key, value))
-    return conf_file
+
+def _set_file_values(fn, values):
+    remaining = values.keys()
+
+    def varfunc(varname, origvalue, op, newlines):
+        newvalue = values.get(varname, origvalue)
+        remaining.remove(varname)
+        return (newvalue, '=', 0, True)
+
+    with open(fn, 'r') as f:
+        (updated, newlines) = bb.utils.edit_metadata(f, values, varfunc)
+
+    for item in remaining:
+        updated = True
+        newlines.append('%s = "%s"' % (item, values[item]))
+
+    if updated:
+        with open(fn, 'w') as f:
+            f.writelines(newlines)
+    return updated
+
 
 def build(args, config, basepath, workspace):
     """Entry point for the devtool 'build' subcommand"""
@@ -42,22 +53,18 @@ def build(args, config, basepath, workspace):
 
     build_task = config.get('Build', 'build_task', 'populate_sysroot')
 
-    postfile_param = ""
-    postfile = ""
+    bbappend = workspace[args.recipename]['bbappend']
     if args.disable_parallel_make:
         logger.info("Disabling 'make' parallelism")
-        postfile = os.path.join(basepath, 'conf', 'disable_parallelism.conf')
-        _create_conf_file({'PARALLEL_MAKE':''}, postfile)
-        postfile_param = "-R %s" % postfile
+        _set_file_values(bbappend, {'PARALLEL_MAKE': ''})
     try:
-        exec_build_env_command(config.init_path, basepath, 'bitbake -c %s %s %s' % (build_task, postfile_param, args.recipename), watch=True)
+        exec_build_env_command(config.init_path, basepath, 'bitbake -c %s %s' % (build_task, args.recipename), watch=True)
     except bb.process.ExecutionError as e:
         # We've already seen the output since watch=True, so just ensure we return something to the user
         return e.exitcode
     finally:
-        if postfile:
-            logger.debug('Removing postfile')
-            os.remove(postfile)
+        if args.disable_parallel_make:
+            _set_file_values(bbappend, {'PARALLEL_MAKE': None})
 
     return 0
 
Signature · 1 line
2.1.0
#15 lore raw

PATCH 15/18 devtool: clarify help text

In reply to Paul Eggleton

* Make some minor clarifications to help text
* Drop ArgumentDefaultsHelpFormatter and just put the defaults in the text itself where needed (because otherwise you get defaults shown for store_true options which is somewhat confusing).

---
 scripts/devtool                 |  2 +-
 scripts/lib/devtool/build.py    |  3 +--
 scripts/lib/devtool/standard.py | 28 ++++++++++++----------------
 scripts/lib/devtool/upgrade.py  |  4 ++--
 4 files changed, 16 insertions(+), 21 deletions(-)
scripts/devtool +1 -1
index 1fcb42c..9d3287c 100755
@@ -258,7 +258,7 @@ def main():
 
     if not context.fixed_setup:
         parser_create_workspace = subparsers.add_parser('create-workspace',
-                                                        help='Set up a workspace',
+                                                        help='Set up workspace in an alternative location',
                                                         description='Sets up a new workspace. NOTE: other devtool subcommands will create a workspace automatically as needed, so you only need to use %(prog)s if you want to specify where the workspace should be located.')
         parser_create_workspace.add_argument('layerpath', nargs='?', help='Path in which the workspace layer should be created')
         parser_create_workspace.add_argument('--create-only', action="store_true", help='Only create the workspace layer, do not alter configuration')
scripts/lib/devtool/build.py +1 -2
index 0b8e88b..a9a0778 100644
@@ -73,8 +73,7 @@ def build(args, config, basepath, workspace):
 def register_commands(subparsers, context):
     """Register devtool subcommands from this plugin"""
     parser_build = subparsers.add_parser('build', help='Build a recipe',
-                                         description='Builds the specified recipe using bitbake (up to and including do_%s)' % _get_build_task(context.config),
-                                         formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+                                         description='Builds the specified recipe using bitbake (up to and including do_%s)' % _get_build_task(context.config))
     parser_build.add_argument('recipename', help='Recipe to build')
     parser_build.add_argument('-s', '--disable-parallel-make', action="store_true", help='Disable make parallelism')
     parser_build.set_defaults(func=build)
scripts/lib/devtool/standard.py +12 -16
index bc92456..274d0fc 100644
@@ -1085,8 +1085,8 @@ def reset(args, config, basepath, workspace):
 def register_commands(subparsers, context):
     """Register devtool subcommands from this plugin"""
     parser_add = subparsers.add_parser('add', help='Add a new recipe',
-                                       description='Adds a new recipe')
-    parser_add.add_argument('recipename', help='Name for new recipe to add')
+                                       description='Adds a new recipe to the workspace to build a specified source tree')
+    parser_add.add_argument('recipename', help='Name for new recipe to add (just name - no version, path or extension)')
     parser_add.add_argument('srctree', help='Path to external source tree')
     group = parser_add.add_mutually_exclusive_group()
     group.add_argument('--same-dir', '-s', help='Build in same directory as source', action="store_true")
@@ -1098,45 +1098,41 @@ def register_commands(subparsers, context):
     parser_add.set_defaults(func=add)
 
     parser_modify = subparsers.add_parser('modify', help='Modify the source for an existing recipe',
-                                       description='Enables modifying the source for an existing recipe',
-                                       formatter_class=argparse.ArgumentDefaultsHelpFormatter)
-    parser_modify.add_argument('recipename', help='Name for recipe to edit')
+                                       description='Enables modifying the source for an existing recipe')
+    parser_modify.add_argument('recipename', help='Name of existing recipe to edit (just name - no version, path or extension)')
     parser_modify.add_argument('srctree', help='Path to external source tree')
     parser_modify.add_argument('--wildcard', '-w', action="store_true", help='Use wildcard for unversioned bbappend')
     parser_modify.add_argument('--extract', '-x', action="store_true", help='Extract source as well')
     group = parser_modify.add_mutually_exclusive_group()
     group.add_argument('--same-dir', '-s', help='Build in same directory as source', action="store_true")
     group.add_argument('--no-same-dir', help='Force build in a separate build directory', action="store_true")
-    parser_modify.add_argument('--branch', '-b', default="devtool", help='Name for development branch to checkout (only when using -x)')
+    parser_modify.add_argument('--branch', '-b', default="devtool", help='Name for development branch to checkout (only when using -x) (default "%(default)s")')
     parser_modify.set_defaults(func=modify)
 
     parser_extract = subparsers.add_parser('extract', help='Extract the source for an existing recipe',
-                                       description='Extracts the source for an existing recipe',
-                                       formatter_class=argparse.ArgumentDefaultsHelpFormatter)
-    parser_extract.add_argument('recipename', help='Name for recipe to extract the source for')
+                                       description='Extracts the source for an existing recipe')
+    parser_extract.add_argument('recipename', help='Name of recipe to extract the source for')
     parser_extract.add_argument('srctree', help='Path to where to extract the source tree')
-    parser_extract.add_argument('--branch', '-b', default="devtool", help='Name for development branch to checkout')
+    parser_extract.add_argument('--branch', '-b', default="devtool", help='Name for development branch to checkout (default "%(default)s")')
     parser_extract.add_argument('--keep-temp', action="store_true", help='Keep temporary directory (for debugging)')
     parser_extract.set_defaults(func=extract, no_workspace=True)
 
     parser_update_recipe = subparsers.add_parser('update-recipe', help='Apply changes from external source tree to recipe',
-                                       description='Applies changes from external source tree to a recipe (updating/adding/removing patches as necessary, or by updating SRCREV)')
+                                       description='Applies changes from external source tree to a recipe (updating/adding/removing patches as necessary, or by updating SRCREV). Note that these changes need to have been committed to the git repository in order to be recognised.')
     parser_update_recipe.add_argument('recipename', help='Name of recipe to update')
     parser_update_recipe.add_argument('--mode', '-m', choices=['patch', 'srcrev', 'auto'], default='auto', help='Update mode (where %(metavar)s is %(choices)s; default is %(default)s)', metavar='MODE')
-    parser_update_recipe.add_argument('--initial-rev', help='Starting revision for patches')
+    parser_update_recipe.add_argument('--initial-rev', help='Override starting revision for patches')
     parser_update_recipe.add_argument('--append', '-a', help='Write changes to a bbappend in the specified layer instead of the recipe', metavar='LAYERDIR')
     parser_update_recipe.add_argument('--wildcard-version', '-w', help='In conjunction with -a/--append, use a wildcard to make the bbappend apply to any recipe version', action='store_true')
     parser_update_recipe.add_argument('--no-remove', '-n', action="store_true", help='Don\'t remove patches, only add or update')
     parser_update_recipe.set_defaults(func=update_recipe)
 
     parser_status = subparsers.add_parser('status', help='Show workspace status',
-                                          description='Lists recipes currently in your workspace and the paths to their respective external source trees',
-                                          formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+                                          description='Lists recipes currently in your workspace and the paths to their respective external source trees')
     parser_status.set_defaults(func=status)
 
     parser_reset = subparsers.add_parser('reset', help='Remove a recipe from your workspace',
-                                         description='Removes the specified recipe from your workspace (resetting its state)',
-                                         formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+                                         description='Removes the specified recipe from your workspace (resetting its state)')
     parser_reset.add_argument('recipename', nargs='?', help='Recipe to reset')
     parser_reset.add_argument('--all', '-a', action="store_true", help='Reset all recipes (clear workspace)')
     parser_reset.add_argument('--no-clean', '-n', action="store_true", help='Don\'t clean the sysroot to remove recipe output')
scripts/lib/devtool/upgrade.py +2 -2
index 761aa99..7bbb915 100644
@@ -361,8 +361,8 @@ def upgrade(args, config, basepath, workspace):
 def register_commands(subparsers, context):
     """Register devtool subcommands from this plugin"""
     parser_upgrade = subparsers.add_parser('upgrade', help='Upgrade an existing recipe',
-                                           description='Upgrades an existing recipe to a new upstream version')
-    parser_upgrade.add_argument('recipename', help='Name for recipe to extract the source for')
+                                           description='Upgrades an existing recipe to a new upstream version. Puts the upgraded recipe file into the workspace along with any associated files, and extracts the source tree to a specified location (in case patches need rebasing or adding to as a result of the upgrade).')
+    parser_upgrade.add_argument('recipename', help='Name of recipe to upgrade (just name - no version, path or extension)')
     parser_upgrade.add_argument('srctree', help='Path to where to extract the source tree')
     parser_upgrade.add_argument('--version', '-V', help='Version to upgrade to (PV)')
     parser_upgrade.add_argument('--srcrev', '-S', help='Source revision to upgrade to (if fetching from an SCM such as git)')
Signature · 1 line
2.1.0
#16 lore raw

PATCH 16/18 lib/oe/recipeutils: refactor patch_recipe_file() to use edit_metadata()

In reply to Paul Eggleton

Use bb.utils.edit_metadata() to replace some of the logic in this function; this avoids us effectively having two implementations of the same thing. In the process fix the following issues:

* Insert values before any leading comments for the next variable instead of after them
* Insert overridden variables (e.g. RDEPENDS_${PN}) in the correct place
* Properly handle replacing varflag settings (e.g. SRC_URI[md5sum])

---
 meta/lib/oe/recipeutils.py     | 160 ++++++++++++++++++++++++-----------------
 scripts/lib/devtool/upgrade.py |  30 ++------
 2 files changed, 97 insertions(+), 93 deletions(-)
meta/lib/oe/recipeutils.py +93 -67
index 8918fac..5e0fda5 100644
@@ -19,9 +19,9 @@ from collections import OrderedDict, defaultdict
 
 
 # Help us to find places to insert values
-recipe_progression = ['SUMMARY', 'DESCRIPTION', 'HOMEPAGE', 'BUGTRACKER', 'SECTION', 'LICENSE', 'LIC_FILES_CHKSUM', 'PROVIDES', 'DEPENDS', 'PR', 'PV', 'SRCREV', 'SRC_URI', 'S', 'do_fetch', 'do_unpack', 'do_patch', 'EXTRA_OECONF', 'do_configure', 'EXTRA_OEMAKE', 'do_compile', 'do_install', 'do_populate_sysroot', 'INITSCRIPT', 'USERADD', 'GROUPADD', 'PACKAGES', 'FILES', 'RDEPENDS', 'RRECOMMENDS', 'RSUGGESTS', 'RPROVIDES', 'RREPLACES', 'RCONFLICTS', 'ALLOW_EMPTY', 'do_package', 'do_deploy']
+recipe_progression = ['SUMMARY', 'DESCRIPTION', 'HOMEPAGE', 'BUGTRACKER', 'SECTION', 'LICENSE', 'LIC_FILES_CHKSUM', 'PROVIDES', 'DEPENDS', 'PR', 'PV', 'SRCREV', 'SRC_URI', 'S', 'do_fetch()', 'do_unpack()', 'do_patch()', 'EXTRA_OECONF', 'do_configure()', 'EXTRA_OEMAKE', 'do_compile()', 'do_install()', 'do_populate_sysroot()', 'INITSCRIPT', 'USERADD', 'GROUPADD', 'PACKAGES', 'FILES', 'RDEPENDS', 'RRECOMMENDS', 'RSUGGESTS', 'RPROVIDES', 'RREPLACES', 'RCONFLICTS', 'ALLOW_EMPTY', 'do_package()', 'do_deploy()']
 # Variables that sometimes are a bit long but shouldn't be wrapped
-nowrap_vars = ['SUMMARY', 'HOMEPAGE', 'BUGTRACKER']
+nowrap_vars = ['SUMMARY', 'HOMEPAGE', 'BUGTRACKER', 'SRC_URI[md5sum]', 'SRC_URI[sha256sum]']
 list_vars = ['SRC_URI', 'LIC_FILES_CHKSUM']
 meta_vars = ['SUMMARY', 'DESCRIPTION', 'HOMEPAGE', 'BUGTRACKER', 'SECTION']
 
@@ -164,85 +164,111 @@ def patch_recipe_file(fn, values, patch=False, relpath=''):
        Note that some manual inspection/intervention may be required
        since this cannot handle all situations.
     """
+
+    import bb.utils
+
+    recipe_progression_res = []
+    recipe_progression_restrs = []
+    for item in recipe_progression:
+        if item.endswith('()'):
+            key = item[:-2]
+        else:
+            key = item
+        restr = '%s(_[a-zA-Z0-9-_$(){}]+|\[[^\]]*\])?' % key
+        if item.endswith('()'):
+            recipe_progression_restrs.append(restr + '()')
+        else:
+            recipe_progression_restrs.append(restr)
+        recipe_progression_res.append(re.compile('^%s$' % restr))
+
+    def get_recipe_pos(variable):
+        for i, p in enumerate(recipe_progression_res):
+            if p.match(variable):
+                return i
+        return -1
+
     remainingnames = {}
     for k in values.keys():
-        remainingnames[k] = recipe_progression.index(k) if k in recipe_progression else -1
+        remainingnames[k] = get_recipe_pos(k)
     remainingnames = OrderedDict(sorted(remainingnames.iteritems(), key=lambda x: x[1]))
 
-    with tempfile.NamedTemporaryFile('w', delete=False) as tf:
-        def outputvalue(name):
-            rawtext = '%s = "%s"\n' % (name, values[name])
-            if name in nowrap_vars:
-                tf.write(rawtext)
-            elif name in list_vars:
-                splitvalue = split_var_value(values[name], assignment=False)
-                if len(splitvalue) > 1:
-                    linesplit = ' \\\n' + (' ' * (len(name) + 4))
-                    tf.write('%s = "%s%s"\n' % (name, linesplit.join(splitvalue), linesplit))
-                else:
-                    tf.write(rawtext)
+    modifying = False
+
+    def outputvalue(name, lines, rewindcomments=False):
+        if values[name] is None:
+            return
+        rawtext = '%s = "%s"\n' % (name, values[name])
+        addlines = []
+        if name in nowrap_vars:
+            addlines.append(rawtext)
+        elif name in list_vars:
+            splitvalue = split_var_value(values[name], assignment=False)
+            if len(splitvalue) > 1:
+                linesplit = ' \\\n' + (' ' * (len(name) + 4))
+                addlines.append('%s = "%s%s"\n' % (name, linesplit.join(splitvalue), linesplit))
             else:
-                wrapped = textwrap.wrap(rawtext)
-                for wrapline in wrapped[:-1]:
-                    tf.write('%s \\\n' % wrapline)
-                tf.write('%s\n' % wrapped[-1])
-
-        tfn = tf.name
-        with open(fn, 'r') as f:
-            # First runthrough - find existing names (so we know not to insert based on recipe_progression)
-            # Second runthrough - make the changes
-            existingnames = []
-            for runthrough in [1, 2]:
-                currname = None
-                for line in f:
-                    if not currname:
-                        insert = False
-                        for k in remainingnames.keys():
-                            for p in recipe_progression:
-                                if re.match('^%s(_prepend|_append)*[ ?:=(]' % p, line):
-                                    if remainingnames[k] > -1 and recipe_progression.index(p) > remainingnames[k] and runthrough > 1 and not k in existingnames:
-                                        outputvalue(k)
-                                        del remainingnames[k]
-                                    break
-                        for k in remainingnames.keys():
-                            if re.match('^%s[ ?:=]' % k, line):
-                                currname = k
-                                if runthrough == 1:
-                                    existingnames.append(k)
-                                else:
-                                    del remainingnames[k]
-                                break
-                        if currname and runthrough > 1:
-                            outputvalue(currname)
-
-                    if currname:
-                        sline = line.rstrip()
-                        if not sline.endswith('\\'):
-                            currname = None
-                        continue
-                    if runthrough > 1:
-                        tf.write(line)
-                f.seek(0)
-        if remainingnames:
-            tf.write('\n')
-            for k in remainingnames.keys():
-                outputvalue(k)
-
-    with open(tfn, 'U') as f:
-        tolines = f.readlines()
+                addlines.append(rawtext)
+        else:
+            wrapped = textwrap.wrap(rawtext)
+            for wrapline in wrapped[:-1]:
+                addlines.append('%s \\\n' % wrapline)
+            addlines.append('%s\n' % wrapped[-1])
+        if rewindcomments:
+            # Ensure we insert the lines before any leading comments
+            # (that we'd want to ensure remain leading the next value)
+            for i, ln in reversed(list(enumerate(lines))):
+                if ln[0] != '#':
+                    lines[i+1:i+1] = addlines
+                    break
+            else:
+                lines.extend(addlines)
+        else:
+            lines.extend(addlines)
+
+    existingnames = []
+    def patch_recipe_varfunc(varname, origvalue, op, newlines):
+        if modifying:
+            # Insert anything that should come before this variable
+            pos = get_recipe_pos(varname)
+            for k in remainingnames.keys()[:]:
+                if remainingnames[k] > -1 and pos >= remainingnames[k] and not k in existingnames:
+                    outputvalue(k, newlines, rewindcomments=True)
+                    del remainingnames[k]
+            # Now change this variable, if it needs to be changed
+            if varname in existingnames:
+                outputvalue(varname, newlines)
+                del remainingnames[varname]
+                return None, None, 0, True
+        else:
+            if varname in values:
+                existingnames.append(varname)
+        return origvalue, None, 0, True
+
+    # First run - establish which values we want to set are already in the file
+    varlist = [re.escape(item) for item in values.keys()]
+    with open(fn, 'r') as f:
+        changed, fromlines = bb.utils.edit_metadata(f, varlist, patch_recipe_varfunc)
+    # Second run - actually set everything
+    modifying = True
+    varlist.extend(recipe_progression_restrs)
+    changed, tolines = bb.utils.edit_metadata(fromlines, varlist, patch_recipe_varfunc, match_overrides=True)
+
+    if remainingnames:
+        if tolines[-1].strip() != '':
+            tolines.append('\n')
+        for k in remainingnames.keys():
+            outputvalue(k, tolines)
+
     if patch:
-        with open(fn, 'U') as f:
-            fromlines = f.readlines()
         relfn = os.path.relpath(fn, relpath)
         diff = difflib.unified_diff(fromlines, tolines, 'a/%s' % relfn, 'b/%s' % relfn)
-        os.remove(tfn)
         return diff
     else:
         with open(fn, 'w') as f:
             f.writelines(tolines)
-        os.remove(tfn)
         return None
 
+
 def localise_file_vars(fn, varfiles, varlist):
     """Given a list of variables and variable history (fetched with get_var_files())
     find where each variable should be set/changed. This handles for example where a
scripts/lib/devtool/upgrade.py +4 -26
index 7bbb915..6620018 100644
@@ -62,26 +62,6 @@ def _get_checksums(rf):
                     checksums[cs] = m.group(1)
     return checksums
 
-def _replace_checksums(rf, md5, sha256):
-    if not md5 and not sha256:
-        return
-    checksums = {'md5sum':md5, 'sha256sum':sha256}
-    with open(rf + ".tmp", "w+") as tmprf:
-        with open(rf) as f:
-            for line in f:
-                m = None
-                for cs in checksums.keys():
-                    m = re.match("^SRC_URI\[%s\].*=.*\"(.*)\"" % cs, line)
-                    if m:
-                        if checksums[cs]:
-                            oldcheck = m.group(1)
-                            newcheck = checksums[cs]
-                            line = line.replace(oldcheck, newcheck)
-                        break
-                tmprf.write(line)
-    os.rename(rf + ".tmp", rf)
-
-
 def _remove_patch_dirs(recipefolder):
     for root, dirs, files in os.walk(recipefolder):
         for d in dirs:
@@ -297,16 +277,14 @@ def _create_new_recipe(newpv, md5, sha256, srcrev, srcbranch, workspace, tinfoil
         if changed:
             newvalues['SRC_URI'] = ' '.join(new_src_uri)
 
+    if md5 and sha256:
+        newvalues['SRC_URI[md5sum]'] = md5
+        newvalues['SRC_URI[sha256sum]'] = sha256
+
     if newvalues:
         rd = oe.recipeutils.parse_recipe(fullpath, None, tinfoil.config_data)
         oe.recipeutils.patch_recipe(rd, fullpath, newvalues)
 
-    if md5 and sha256:
-        # Unfortunately, oe.recipeutils.patch_recipe cannot update flags.
-        # once the latter feature is implemented, we should call patch_recipe
-        # instead of the following function
-        _replace_checksums(fullpath, md5, sha256)
-
     return fullpath
 
 def upgrade(args, config, basepath, workspace):
Signature · 1 line
2.1.0
#17 lore raw

PATCH 17/18 recipetool: add setvar subcommand

In reply to Paul Eggleton

Add a recipetool subcommand "setvar" to set a variable in a file. This uses our existing logic such that it doesn't matter if the variable is already set in the recipe, if it's set in the recipe or some inc file, and if the variable is not currently set that the line setting the variable gets inserted in the right place in the file.

Implements [YOCTO #7676].

---
 scripts/lib/recipetool/setvar.py | 75 ++++++++++++++++++++++++++++++++++++++++
 scripts/recipetool               |  4 ++-
 2 files changed, 78 insertions(+), 1 deletion(-)
 create mode 100644 scripts/lib/recipetool/setvar.py
scripts/lib/recipetool/setvar.py new file +75 -0
new file mode 100644
index 0000000..18e3281
@@ -0,0 +1,75 @@
+# Recipe creation tool - set variable plugin
+#
+# Copyright (C) 2015 Intel Corporation
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+import sys
+import os
+import argparse
+import glob
+import fnmatch
+import re
+import logging
+import scriptutils
+
+logger = logging.getLogger('recipetool')
+
+tinfoil = None
+plugins = None
+
+def tinfoil_init(instance):
+    global tinfoil
+    tinfoil = instance
+
+def setvar(args):
+    import oe.recipeutils
+
+    if args.delete:
+        if args.value:
+            logger.error('-D/--delete and specifying a value are mutually exclusive')
+            return 1
+        value = None
+    else:
+        if args.value is None:
+            logger.error('You must specify a value if not using -D/--delete')
+            return 1
+        value = args.value
+    varvalues = {args.varname: value}
+
+    if args.recipe_only:
+        patches = [oe.recipeutils.patch_recipe_file(args.recipefile, varvalues, patch=args.patch)]
+    else:
+        rd = oe.recipeutils.parse_recipe(args.recipefile, None, tinfoil.config_data)
+        if not rd:
+            return 1
+        patches = oe.recipeutils.patch_recipe(rd, args.recipefile, varvalues, patch=args.patch)
+    if args.patch:
+        for patch in patches:
+            for line in patch:
+                sys.stdout.write(line)
+    return 0
+
+
+def register_command(subparsers):
+    parser_setvar = subparsers.add_parser('setvar',
+                                          help='Set a variable within a recipe',
+                                          description='Adds/updates the value a variable is set to in a recipe')
+    parser_setvar.add_argument('recipefile', help='Recipe file to update')
+    parser_setvar.add_argument('varname', help='Variable name to set')
+    parser_setvar.add_argument('value', nargs='?', help='New value to set the variable to')
+    parser_setvar.add_argument('--recipe-only', '-r', help='Do not set variable in any include file if present', action='store_true')
+    parser_setvar.add_argument('--patch', '-p', help='Create a patch to make the change instead of modifying the recipe', action='store_true')
+    parser_setvar.add_argument('--delete', '-D', help='Delete the specified value instead of setting it', action='store_true')
+    parser_setvar.set_defaults(func=setvar)
scripts/recipetool +3 -1
index 87fb35e..4af0bfb 100755
@@ -34,7 +34,7 @@ plugins = []
 def tinfoil_init(parserecipes):
     import bb.tinfoil
     import logging
-    tinfoil = bb.tinfoil.Tinfoil()
+    tinfoil = bb.tinfoil.Tinfoil(tracking=True)
     tinfoil.prepare(not parserecipes)
     tinfoil.logger.setLevel(logger.getEffectiveLevel())
     return tinfoil
@@ -96,7 +96,9 @@ def main():
 
     try:
         if getattr(args, 'parserecipes', False):
+            tinfoil.config_data.disableTracking()
             tinfoil.parseRecipes()
+            tinfoil.config_data.enableTracking()
         ret = args.func(args)
     except bb.BBHandledException:
         ret = 1
Signature · 1 line
2.1.0
#18 lore raw

PATCH 18/18 recipetool: make plugin registration function name consistent with devtool

In reply to Paul Eggleton

This should have been register_commands rather than register_command; I used register_commands in devtool so lets change this here to be consistent with that. (Since this is extensible through layers though we need to remain compatible with the old name, so fall back to that if the new function name isn't there.)

---
 scripts/lib/recipetool/append.py    | 2 +-
 scripts/lib/recipetool/create.py    | 2 +-
 scripts/lib/recipetool/newappend.py | 2 +-
 scripts/lib/recipetool/setvar.py    | 2 +-
 scripts/recipetool                  | 6 +++++-
 5 files changed, 9 insertions(+), 5 deletions(-)
scripts/lib/recipetool/append.py +1 -1
index 7fe4115..fc2008c 100644
@@ -433,7 +433,7 @@ def target_path(targetpath):
     return targetpath
 
 
-def register_command(subparsers):
+def register_commands(subparsers):
     common = argparse.ArgumentParser(add_help=False)
     common.add_argument('-m', '--machine', help='Make bbappend changes specific to a machine only', metavar='MACHINE')
     common.add_argument('-w', '--wildcard-version', help='Use wildcard to make the bbappend apply to any recipe version', action='store_true')
scripts/lib/recipetool/create.py +1 -1
index 15aa9bd..2d75046 100644
@@ -436,7 +436,7 @@ def convert_debian(debpath):
     return values
 
 
-def register_command(subparsers):
+def register_commands(subparsers):
     parser_create = subparsers.add_parser('create',
                                           help='Create a new recipe',
                                           description='Creates a new recipe from a source tree')
scripts/lib/recipetool/newappend.py +1 -1
index 4eeac0e..f9b8d85 100644
@@ -97,7 +97,7 @@ def newappend(args):
     print(append_path)
 
 
-def register_command(subparsers):
+def register_commands(subparsers):
     parser = subparsers.add_parser('newappend',
                                    help='Create a bbappend for the specified target in the specified layer')
     parser.add_argument('-w', '--wildcard-version', help='Use wildcard to make the bbappend apply to any recipe version', action='store_true')
scripts/lib/recipetool/setvar.py +1 -1
index 18e3281..657d2b6 100644
@@ -62,7 +62,7 @@ def setvar(args):
     return 0
 
 
-def register_command(subparsers):
+def register_commands(subparsers):
     parser_setvar = subparsers.add_parser('setvar',
                                           help='Set a variable within a recipe',
                                           description='Adds/updates the value a variable is set to in a recipe')
scripts/recipetool +5 -1
index 4af0bfb..791a66a 100755
@@ -82,7 +82,11 @@ def main():
 
     registered = False
     for plugin in plugins:
-        if hasattr(plugin, 'register_command'):
+        if hasattr(plugin, 'register_commands'):
+            registered = True
+            plugin.register_commands(subparsers)
+        elif hasattr(plugin, 'register_command'):
+            # Legacy function name
             registered = True
             plugin.register_command(subparsers)
         if hasattr(plugin, 'tinfoil_init'):
Signature · 1 line
2.1.0