devtool: upgrade feature

From: Leonardo Sandoval <[email protected]>

Upgrades a recipe to a particular version and downloads the source code into srctree. User can avoid patching the source code. These are the general steps of the upgrade function:
* Extract current recipe source code into srctree and create branch
* Extract upgrade recipe source code into srctree and rebase with previous branch. This step also creates a temporal recipe (created using recipetool), containing the correct checksums.
* Creates the new recipe under the workspace

[YOCTO #7642]

Signed-off-by Leonardo Sandoval <[email protected]>

---
  scripts/lib/devtool/standard.py |   4 +-
  scripts/lib/devtool/upgrade.py  | 314 ++++++++++++++++++++++++++++++++++++++++
  2 files changed, 317 insertions(+), 1 deletion(-)

create mode 100644 scripts/lib/devtool/upgrade.py

scripts/lib/devtool/standard.py +2 -0

index ea21877..cd5a3ed 100644
@@ -205,6 +205,8 @@ def extract(args, config, basepath, workspace):
 
      srctree = os.path.abspath(args.srctree)
      initial_rev = _extract_source(srctree, args.keep_temp, args.branch, rd)
+    logger.info('Source tree extracted to %s' % srctree)
+
      if initial_rev:
          return 0
      else:

scripts/lib/devtool/standard.py +1 -0

      finally:
          bb.logger.setLevel(origlevel)
 

@@ -439,6 +440,7 @@ def modify(args, config, basepath, workspace):
          initial_rev = _extract_source(args.srctree, False, args.branch, rd)
          if not initial_rev:
              return 1
+        logger.info('Source tree extracted to %s' % srctree)
          # Get list of commits since this revision
          (stdout, _) = bb.process.run('git rev-list --reverse %s..HEAD' % initial_rev, cwd=args.srctree)
          commits = stdout.split()

scripts/lib/devtool/upgrade.py new file +99 -0

new file mode 100644
index 0000000..9bef984
@@ -0,0 +1,314 @@
+# Development tool - upgrade command plugin
+#
+# Copyright (C) 2014-2015 Intel Corporation
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# DESCRIPTION
+# Created a new recipe under workspace/recipes/<recipename> and place the
+# source code into <srctree>.
+# The upgrade feature executes the following steps:
+#     * Extract current recipe source code into srctree and create branch
+#     * Extract upgrade recipe source code into srctree and rebase with
+#       previous branch. This step also creates a temporal recipe (created
+#       using recipetool), containing the correct checksums.
+#     * Creates the new recipe under the workspace
+"""Devtool upgrade plugin"""
+
+import os
+import sys
+import re
+import shutil
+import tempfile
+import logging
+import argparse
+import scriptutils
+import errno
+from devtool import standard
+from devtool import exec_build_env_command, setup_tinfoil, DevtoolError
+
+logger = logging.getLogger('devtool')
+
+def plugin_init(pluginlist):
+    """Plugin initialization"""
+    pass
+
+def _extract_upgrade_source(args, devbranch, config, basepath, d, recipepostfix='tmp'):
+    """Extract sources of a recipe with PV given on args.version
+
+    On the target source tree folder, a new branch (<devbranch>_<args.version>)
+    and tag (<devbranch>-base_<args.version>) will be created. In case patches
+    are applied, another tag is created (<devbranch>-patched_<args.version>).
+
+    Returns: 1) The (git) initial revision ID
+             2) The full path of a temporal recipe containing the correct checksums
+    """
+    import oe.recipeutils
+
+    initial_rev = None
+    srctree = os.path.abspath(args.srctree)
+
+    pn = d.getVar('PN', True)
+
+    standard._check_compatible_recipe(pn, d)
+
+    recipepath = os.path.join(config.workspace_path, 'recipes', pn)
+    bb.utils.mkdirhier(recipepath)
+    recipefile = os.path.join(recipepath, "%s-%s.bb" % (pn,recipepostfix))
+    tmpsrcbasetree = tempfile.mkdtemp(prefix='devtool')
+
+    # Change PV and get URL
+    crd = d.createCopy()
+    pv = d.getVar('PV', True)
+    crd.setVar('PV', args.version)
+    src_uri = crd.getVar('SRC_URI', True)
+    if src_uri:
+        url = src_uri.split()[0]
+
+    # Generate recipe and fetch new source
+    try:
+        cmdopts = "-o %s -x %s -V %s" % (recipefile, tmpsrcbasetree, args.version)
+        cmd = 'recipetool create "%s" %s' % (url, cmdopts)
+        stdout, _ = exec_build_env_command(config.init_path, basepath, cmd)
+    except bb.process.ExecutionError as e:
+        raise DevtoolError('Command \'%s\' failed:\n%s' % (e.command, e.stdout))
+
+    tmpsrctree = os.path.join(tmpsrcbasetree, pn + '-' + args.version)
+
+    try:
+        # branch from devtool-base (original source code without patches) before copying new source code
+        bb.process.run('git checkout -b %s devtool-base' % devbranch, cwd=srctree)
+        bb.process.run('git tag -f devtool-base_%s' % args.version, cwd=srctree)
+
+        # Copy tmpsrctree  into srctree
+        src_files = standard._ls_tree(tmpsrctree)
+        for path in src_files:
+            tgt_dir = os.path.join(srctree, os.path.dirname(path))
+            bb.utils.mkdirhier(tgt_dir)
+            tgt_path = os.path.join(srctree, path)

scripts/lib/devtool/upgrade.py new file +165 -0

+        (stdout, _) = bb.process.run('git rev-parse HEAD', cwd=srctree)
+        initial_rev = stdout.rstrip()
+
+        if args.no_patch:
+            patches = oe.recipeutils.get_recipe_patches(crd)
+            if len(patches):
+                logger.warn('By user choice, the following (%s_%s) patches will NOT be applied into' %(pn,pv))
+                for patch in patches:
+                    logger.warn("\t%s" % os.path.basename(patch))
+        else:
+            try:
+                bb.process.run('git rebase devtool-patched', cwd=srctree)
+            except bb.process.ExecutionError as e:
+                # this is the only case where we do not propagate the error
+                # user will have the change to correct merges
+                logger.error('Command \'%s\' failed:\n%s' % (e.command, e.stdout))
+            bb.process.run('git tag -f devtool-patched_%s' % args.version, cwd=srctree)
+    except bb.process.ExecutionError as e:
+        raise DevtoolError('Command \'%s\' failed:\n%s' % (e.command, e.stdout))
+    finally:
+        if args.keep_temp:
+            logger.info('Preserving temporary directory %s' % tmpsrcbasetree)
+        else:
+            shutil.rmtree(tmpsrcbasetree)
+    return initial_rev, recipefile
+
+def _create_upgrade_recipe(args, config, recipetmp, config_data, d):
+    """Creates the new recipe under workspace
+
+    Returns the full path on the new created recipe
+    """
+    import oe.recipeutils
+
+    def _get_checksums(recipefile):
+        import re
+        checksums = {}
+        with open(recipefile) as rf:
+            for line in rf:
+                for cs in ['md5sum', 'sha256sum']:
+                    m = re.match("^SRC_URI\[%s\].*=.*\"(.*)\"" % cs, line)
+                    if m:
+                        checksums[cs] = m.group(1)
+        return checksums
+
+    def _replace_checksums(recipefile, checksums):
+        import re
+        with open(recipefile + ".tmp", "w+") as tmprf:
+            with open(recipefile) as rf:
+                for line in rf:
+                    m = None
+                    for cs in ['md5sum', 'sha256sum']:
+                        m = re.match("^SRC_URI\[%s\].*=.*\"(.*)\"" % cs, line)
+                        if m:
+                            if cs in checksums:
+                                oldcheck = m.group(1)
+                                newcheck = checksums[cs]
+                                line = line.replace(oldcheck, newcheck)
+                            break
+                    tmprf.write(line)
+
+        os.rename(recipefile + ".tmp", recipefile)
+
+    def _rename_patch_dirs(recipefolder, oldpv, newpv):
+        for root, dirs, files in os.walk(recipefolder):
+            for olddir in dirs:
+                if olddir.find(oldpv) != -1:
+                    newdir = olddir.replace(oldpv, newpv)
+                    bb.process.run('mv %s %s' % (olddir, newdir))
+
+    def _remove_patch_dirs(recipefolder):
+        for root, dirs, files in os.walk(recipefolder):
+            for d in dirs:
+                shutil.rmtree(os.path.join(root,d))
+
+    def _recipe_contains(recipefile, var):
+        import re
+        found = False
+        with open(recipefile) as rf:
+            for line in rf:
+                if re.match("^%s.*=.*" % var, line):
+                    found = True
+                    break
+        return found
+
+    if not os.path.exists(recipetmp):
+        raise DevtoolError("Temporal recipe %s does not exist" % recipetmp)
+
+    # Copy current recipe into workspace
+    pn = d.getVar('PN', True)
+    recipepath = os.path.join(config.workspace_path, 'recipes', pn)
+    oe.recipeutils.copy_recipe_files(d, recipepath)
+
+    # Rename recipe
+    pv = d.getVar('PV', True)
+    recipename = "%s_%s.bb" % (pn, pv)
+    newrecipename = "%s_%s.bb" % (pn, args.version)
+    if os.path.isfile(os.path.join(recipepath, recipename)):
+        bb.process.run('mv %s %s' % (recipename, newrecipename), cwd=recipepath)
+    else:
+        # Check if it is a git recipe
+        recipename = newrecipename = "%s_git.bb" % pn
+        if not os.path.isfile(os.path.join(recipepath, recipename)):
+            raise DevtoolError("Original recipe not found on workspace")
+
+    # Rename folders
+    _rename_patch_dirs(recipepath, pv, args.version)
+
+    # Update PV, just in case it is present
+    if _recipe_contains(os.path.join(recipepath, newrecipename), 'PV'):
+        oe.recipeutils.patch_recipe(d, os.path.join(recipepath, newrecipename), {'PV':args.version})
+
+    # Update checksums
+    checksums = _get_checksums(os.path.join(recipepath, recipetmp))
+    _replace_checksums(os.path.join(recipepath, newrecipename), checksums)
+
+    # Remove recipe created by the recipe-tool
+    bb.process.run('rm %s' % recipetmp)
+
+    return os.path.join(recipepath,newrecipename)
+
+def upgrade(args, config, basepath, workspace):
+    """Entry point for the devtool 'upgrade' subcommand"""
+    import bb
+    import oe.recipeutils
+
+    if args.recipename in workspace:
+        raise DevtoolError("recipe %s is already in your workspace" %
+                            args.recipename)
+    if not args.version:
+        raise DevtoolError("Provide a version through the parameter --version/-V")
+
+    reason = oe.recipeutils.validate_pn(args.recipename)
+    if reason:
+        raise DevtoolError(reason)
+
+    tinfoil = setup_tinfoil()
+
+    rd = standard._parse_recipe(config, tinfoil, args.recipename, True)
+    if not rd:
+        return 1
+
+    pn = rd.getVar('PV', True)
+    if pn == args.version:
+        raise DevtoolError("Current and upgrade versions are the same %s" % pn)
+
+    srctree = os.path.abspath(args.srctree)
+
+    try:
+        # Extract source from current recipe
+        initial_rev_base = standard._extract_source(srctree, False, args.branch, rd)
+
+        # We need to shutdown tinfoil temporally because recipetool will be used
+        tinfoil.shutdown()
+
+        # Extract new source code
+        branch_upgrade = "%s_%s" % (args.branch, args.version)
+        initial_rev_upgrade, recipe_file_tmp = _extract_upgrade_source(args, branch_upgrade, config, basepath, rd)
+        # Start again tinfoil
+        tinfoil = setup_tinfoil()
+
+        recipe_file = _create_upgrade_recipe(args, config, recipe_file_tmp, tinfoil.config_data, rd)
+
+    except DevtoolError as e:
+        logger.error(e)
+        # remove workspace recipe and srctree

scripts/lib/devtool/upgrade.py new file +30 -0

+        raise DevtoolError(e)
+
+    appendpath = os.path.join(config.workspace_path, 'appends')
+    if not os.path.exists(appendpath):
+        os.makedirs(appendpath)
+
+    recipe_file_base = os.path.basename(os.path.splitext(recipe_file)[0])
+    appendfile = os.path.join(appendpath, '%s.bbappend' % recipe_file_base)
+    with open(appendfile, 'w') as f:
+        f.write('inherit externalsrc\n')
+        f.write('EXTERNALSRC = "%s"\n' % srctree)
+        f.write('EXTERNALSRC_BUILD = "%s"\n' % srctree)
+        f.write('\n# initial_rev: %s\n' % initial_rev_upgrade)
+
+    standard._add_md5(config, args.recipename, appendfile)
+    logger.info('Source tree extracted to %s' % srctree)
+    return 0
+
+def register_commands(subparsers, context):
+    """Register devtool subcommands from this plugin"""
+    parser_upgrade = subparsers.add_parser('upgrade', help='Upgrade an existing recipe',
+                                       description='Upgrades an existing recipe',
+                                       formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+    parser_upgrade.add_argument('recipename', help='Name for recipe to extract the source for')
+    parser_upgrade.add_argument('srctree', help='Path to where to extract the source tree')
+    parser_upgrade.add_argument('--version', '-V', help='Version to upgrade (PV)')
+    parser_upgrade.add_argument('--branch', '-b', default="devtool", help='Name for development branch to checkout')
+    parser_upgrade.add_argument('--keep-temp', action="store_true", help='Keep temporary directory (for debugging)')
+    parser_upgrade.add_argument('--no-patch', action="store_true", help='Do not patch the new source code')
+    parser_upgrade.set_defaults(func=upgrade)

Comments below,

Quoted 34 lines · On 26/08/15 02:43, [email protected] wrote:

From: Leonardo Sandoval <[email protected]>

Upgrades a recipe to a particular version and downloads the source code into srctree. User can avoid patching the source code. These are the general steps of the upgrade function:
* Extract current recipe source code into srctree and create branch
* Extract upgrade recipe source code into srctree and rebase with previous branch. This step also creates a temporal recipe (created using recipetool), containing the correct checksums.
* Creates the new recipe under the workspace

[YOCTO #7642]

Signed-off-by: Leonardo Sandoval
<[email protected]>

---
  scripts/lib/devtool/standard.py |   4 +-
  scripts/lib/devtool/upgrade.py  | 314
++++++++++++++++++++++++++++++++++++++++
  2 files changed, 317 insertions(+), 1 deletion(-)

create mode 100644 scripts/lib/devtool/upgrade.py

scripts/lib/devtool/standard.py +2 -0

index ea21877..cd5a3ed 100644
@@ -205,6 +205,8 @@ def extract(args, config, basepath, workspace):
      srctree = os.path.abspath(args.srctree)
      initial_rev = _extract_source(srctree, args.keep_temp,
 args.branch, rd)
+    logger.info('Source tree extracted to %s' % srctree)
+
      if initial_rev:

Quoted 7 lines

scripts/lib/devtool/standard.py +0 -0

          return 0
      else:

@@ -360,7 +362,6 @@ def _extract_source(srctree, keep_temp, devbranch,

d):
bb.process.run('git checkout patches', cwd=srcsubdir)
shutil.move(srcsubdir, srctree)
- logger.info('Source tree extracted to %s' % srctree)

Why you move this logging outside?

Quoted 142 lines

finally:
bb.logger.setLevel(origlevel)

scripts/lib/devtool/standard.py +1 -0

@@ -439,6 +440,7 @@ def modify(args, config, basepath, workspace):
          initial_rev = _extract_source(args.srctree, False,
 args.branch, rd)
          if not initial_rev:
              return 1
+        logger.info('Source tree extracted to %s' % srctree)
          # Get list of commits since this revision
          (stdout, _) = bb.process.run('git rev-list --reverse

%s..HEAD' % initial_rev, cwd=args.srctree)
commits = stdout.split()

scripts/lib/devtool/upgrade.py new file +14 -0

new file mode 100644
index 0000000..9bef984
@@ -0,0 +1,314 @@
+# Development tool - upgrade command plugin
+#
+# Copyright (C) 2014-2015 Intel Corporation
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License

along
+# with this program; if not, write to the Free Software Foundation, Inc.,

scripts/lib/devtool/upgrade.py new file +4 -0

+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# DESCRIPTION
+# Created a new recipe under workspace/recipes/<recipename> and place

the

scripts/lib/devtool/upgrade.py new file +3 -0

+# source code into <srctree>.
+# The upgrade feature executes the following steps:
+#     * Extract current recipe source code into srctree and create

branch

scripts/lib/devtool/upgrade.py new file +2 -0

+#     * Extract upgrade recipe source code into srctree and rebase with
+#       previous branch. This step also creates a temporal recipe

(created

scripts/lib/devtool/upgrade.py new file +23 -0

+#       using recipetool), containing the correct checksums.
+#     * Creates the new recipe under the workspace
+"""Devtool upgrade plugin"""
+
+import os
+import sys
+import re
+import shutil
+import tempfile
+import logging
+import argparse
+import scriptutils
+import errno
+from devtool import standard
+from devtool import exec_build_env_command, setup_tinfoil, DevtoolError
+
+logger = logging.getLogger('devtool')
+
+def plugin_init(pluginlist):
+    """Plugin initialization"""
+    pass
+
+def _extract_upgrade_source(args, devbranch, config, basepath, d,

recipepostfix='tmp'):

scripts/lib/devtool/upgrade.py new file +3 -0

+    """Extract sources of a recipe with PV given on args.version
+
+    On the target source tree folder, a new branch

(<devbranch>_<args.version>)
+ and tag (<devbranch>-base_<args.version>) will be created. In
case patches
+ are applied, another tag is created
(<devbranch>-patched_<args.version>).

scripts/lib/devtool/upgrade.py new file +3 -0

+
+    Returns: 1) The (git) initial revision ID
+             2) The full path of a temporal recipe containing the

correct checksums

scripts/lib/devtool/upgrade.py new file +13 -0

+    """
+    import oe.recipeutils
+
+    initial_rev = None
+    srctree = os.path.abspath(args.srctree)
+
+    pn = d.getVar('PN', True)
+
+    standard._check_compatible_recipe(pn, d)
+
+    recipepath = os.path.join(config.workspace_path, 'recipes', pn)
+    bb.utils.mkdirhier(recipepath)
+    recipefile = os.path.join(recipepath, "%s-%s.bb" %

(pn,recipepostfix))

scripts/lib/devtool/upgrade.py new file +13 -0

+    tmpsrcbasetree = tempfile.mkdtemp(prefix='devtool')
+
+    # Change PV and get URL
+    crd = d.createCopy()
+    pv = d.getVar('PV', True)
+    crd.setVar('PV', args.version)
+    src_uri = crd.getVar('SRC_URI', True)
+    if src_uri:
+        url = src_uri.split()[0]
+
+    # Generate recipe and fetch new source
+    try:
+        cmdopts = "-o %s -x %s -V %s" % (recipefile, tmpsrcbasetree,

args.version)

scripts/lib/devtool/upgrade.py new file +2 -0

+        cmd = 'recipetool create "%s" %s' % (url, cmdopts)
+        stdout, _ = exec_build_env_command(config.init_path,

basepath, cmd)

scripts/lib/devtool/upgrade.py new file +2 -0

+    except bb.process.ExecutionError as e:
+        raise DevtoolError('Command \'%s\' failed:\n%s' % (e.command,

e.stdout))

scripts/lib/devtool/upgrade.py new file +5 -0

+
+    tmpsrctree = os.path.join(tmpsrcbasetree, pn + '-' + args.version)
+
+    try:
+        # branch from devtool-base (original source code without

patches) before copying new source code
+ bb.process.run('git checkout -b %s devtool-base' % devbranch,
cwd=srctree)
+ bb.process.run('git tag -f devtool-base_%s' % args.version,
cwd=srctree)

scripts/lib/devtool/upgrade.py new file +13 -1

+
+        # Copy tmpsrctree  into srctree
+        src_files = standard._ls_tree(tmpsrctree)
+        for path in src_files:
+            tgt_dir = os.path.join(srctree, os.path.dirname(path))
+            bb.utils.mkdirhier(tgt_dir)
+            tgt_path = os.path.join(srctree, path)
+            os.rename(os.path.join(tmpsrctree, path), tgt_path)
+
+        # Track modified and untracked files
+        (stdout,_) = bb.process.run('git ls-files --modified --others
--exclude-standard', cwd=srctree)
+        add_files = stdout.splitlines()
+        for add_file in add_files:

Quoted 5 lines

scripts/lib/devtool/upgrade.py new file +4 -0

+            bb.process.run('git add "%s"' % add_file, cwd=srctree)
+
+        if len(add_files):
+            bb.process.run('git commit -q -m "Initial commit from

upstream at version %s"' % args.version, cwd=srctree)

I don't know if is good to commit changes without user intervention. This tool is an interactive one may be ask to the
user?

Quoted 184 lines

scripts/lib/devtool/upgrade.py new file +7 -0

+        (stdout, _) = bb.process.run('git rev-parse HEAD', cwd=srctree)
+        initial_rev = stdout.rstrip()
+
+        if args.no_patch:
+            patches = oe.recipeutils.get_recipe_patches(crd)
+            if len(patches):
+                logger.warn('By user choice, the following (%s_%s)

patches will NOT be applied into' %(pn,pv))

scripts/lib/devtool/upgrade.py new file +5 -0

+                for patch in patches:
+                    logger.warn("\t%s" % os.path.basename(patch))
+        else:
+            try:
+                bb.process.run('git rebase devtool-patched',

cwd=srctree)

scripts/lib/devtool/upgrade.py new file +2 -0

+            except bb.process.ExecutionError as e:
+                # this is the only case where we do not propagate the

error

scripts/lib/devtool/upgrade.py new file +2 -0

+                # user will have the change to correct merges
+                logger.error('Command \'%s\' failed:\n%s' %

(e.command, e.stdout))
+ bb.process.run('git tag -f devtool-patched_%s' %
args.version, cwd=srctree)

scripts/lib/devtool/upgrade.py new file +2 -0

+    except bb.process.ExecutionError as e:
+        raise DevtoolError('Command \'%s\' failed:\n%s' % (e.command,

e.stdout))

scripts/lib/devtool/upgrade.py new file +3 -0

+    finally:
+        if args.keep_temp:
+            logger.info('Preserving temporary directory %s' %

tmpsrcbasetree)

scripts/lib/devtool/upgrade.py new file +18 -0

+        else:
+            shutil.rmtree(tmpsrcbasetree)
+    return initial_rev, recipefile
+
+def _create_upgrade_recipe(args, config, recipetmp, config_data, d):
+    """Creates the new recipe under workspace
+
+    Returns the full path on the new created recipe
+    """
+    import oe.recipeutils
+
+    def _get_checksums(recipefile):
+        import re
+        checksums = {}
+        with open(recipefile) as rf:
+            for line in rf:
+                for cs in ['md5sum', 'sha256sum']:
+                    m = re.match("^SRC_URI\[%s\].*=.*\"(.*)\"" % cs,

line)

scripts/lib/devtool/upgrade.py new file +12 -0

+                    if m:
+                        checksums[cs] = m.group(1)
+        return checksums
+
+    def _replace_checksums(recipefile, checksums):
+        import re
+        with open(recipefile + ".tmp", "w+") as tmprf:
+            with open(recipefile) as rf:
+                for line in rf:
+                    m = None
+                    for cs in ['md5sum', 'sha256sum']:
+                        m = re.match("^SRC_URI\[%s\].*=.*\"(.*)\"" %

cs, line)

scripts/lib/devtool/upgrade.py new file +34 -0

+                        if m:
+                            if cs in checksums:
+                                oldcheck = m.group(1)
+                                newcheck = checksums[cs]
+                                line = line.replace(oldcheck, newcheck)
+                            break
+                    tmprf.write(line)
+
+        os.rename(recipefile + ".tmp", recipefile)
+
+    def _rename_patch_dirs(recipefolder, oldpv, newpv):
+        for root, dirs, files in os.walk(recipefolder):
+            for olddir in dirs:
+                if olddir.find(oldpv) != -1:
+                    newdir = olddir.replace(oldpv, newpv)
+                    bb.process.run('mv %s %s' % (olddir, newdir))
+
+    def _remove_patch_dirs(recipefolder):
+        for root, dirs, files in os.walk(recipefolder):
+            for d in dirs:
+                shutil.rmtree(os.path.join(root,d))
+
+    def _recipe_contains(recipefile, var):
+        import re
+        found = False
+        with open(recipefile) as rf:
+            for line in rf:
+                if re.match("^%s.*=.*" % var, line):
+                    found = True
+                    break
+        return found
+
+    if not os.path.exists(recipetmp):
+        raise DevtoolError("Temporal recipe %s does not exist" %

recipetmp)

scripts/lib/devtool/upgrade.py new file +12 -0

+
+    # Copy current recipe into workspace
+    pn = d.getVar('PN', True)
+    recipepath = os.path.join(config.workspace_path, 'recipes', pn)
+    oe.recipeutils.copy_recipe_files(d, recipepath)
+
+    # Rename recipe
+    pv = d.getVar('PV', True)
+    recipename = "%s_%s.bb" % (pn, pv)
+    newrecipename = "%s_%s.bb" % (pn, args.version)
+    if os.path.isfile(os.path.join(recipepath, recipename)):
+        bb.process.run('mv %s %s' % (recipename, newrecipename),

cwd=recipepath)

scripts/lib/devtool/upgrade.py new file +12 -0

+    else:
+        # Check if it is a git recipe
+        recipename = newrecipename = "%s_git.bb" % pn
+        if not os.path.isfile(os.path.join(recipepath, recipename)):
+            raise DevtoolError("Original recipe not found on workspace")
+
+    # Rename folders
+    _rename_patch_dirs(recipepath, pv, args.version)
+
+    # Update PV, just in case it is present
+    if _recipe_contains(os.path.join(recipepath, newrecipename), 'PV'):
+        oe.recipeutils.patch_recipe(d, os.path.join(recipepath,

newrecipename), {'PV':args.version})

scripts/lib/devtool/upgrade.py new file +4 -0

+
+    # Update checksums
+    checksums = _get_checksums(os.path.join(recipepath, recipetmp))
+    _replace_checksums(os.path.join(recipepath, newrecipename),

checksums)

scripts/lib/devtool/upgrade.py new file +30 -1

+
+    # Remove recipe created by the recipe-tool
+    bb.process.run('rm %s' % recipetmp)
+
+    return os.path.join(recipepath,newrecipename)
+
+def upgrade(args, config, basepath, workspace):
+    """Entry point for the devtool 'upgrade' subcommand"""
+    import bb
+    import oe.recipeutils
+
+    if args.recipename in workspace:
+        raise DevtoolError("recipe %s is already in your workspace" %
+                            args.recipename)
+    if not args.version:
+        raise DevtoolError("Provide a version through the parameter
--version/-V")
+
+    reason = oe.recipeutils.validate_pn(args.recipename)
+    if reason:
+        raise DevtoolError(reason)
+
+    tinfoil = setup_tinfoil()
+
+    rd = standard._parse_recipe(config, tinfoil, args.recipename, True)
+    if not rd:
+        return 1
+
+    pn = rd.getVar('PV', True)
+    if pn == args.version:
+        raise DevtoolError("Current and upgrade versions are the same

%s" % pn)

scripts/lib/devtool/upgrade.py new file +6 -0

+
+    srctree = os.path.abspath(args.srctree)
+
+    try:
+        # Extract source from current recipe
+        initial_rev_base = standard._extract_source(srctree, False,

args.branch, rd)

scripts/lib/devtool/upgrade.py new file +2 -0

+
+        # We need to shutdown tinfoil temporally because recipetool

will be used

scripts/lib/devtool/upgrade.py new file +5 -0

+        tinfoil.shutdown()
+
+        # Extract new source code
+        branch_upgrade = "%s_%s" % (args.branch, args.version)
+        initial_rev_upgrade, recipe_file_tmp =

_extract_upgrade_source(args, branch_upgrade, config, basepath, rd)

scripts/lib/devtool/upgrade.py new file +4 -0

+        # Start again tinfoil
+        tinfoil = setup_tinfoil()
+
+        recipe_file = _create_upgrade_recipe(args, config,

recipe_file_tmp, tinfoil.config_data, rd)

scripts/lib/devtool/upgrade.py new file +4 -0

+
+    except DevtoolError as e:
+        logger.error(e)
+        # remove workspace recipe and srctree

scripts/lib/devtool/upgrade.py line 284

Quoted 10 lines

scripts/lib/devtool/upgrade.py new file +10 -0

+        pn = rd.getVar('PN', True)
+        recipespath = os.path.join(config.workspace_path, 'recipes')
+        recipepath = os.path.join(recipespath, pn)
+        if os.path.exists(recipepath):
+            shutil.rmtree(recipepath)
+            if not len(os.listdir(recipespath)):
+                os.rmdir(recipespath)
+        # remove srctree
+        if os.path.exists(srctree):
+            shutil.rmtree(srctree)

May be will be better to put this inside a clean function?

Quoted 41 lines

scripts/lib/devtool/upgrade.py new file +7 -0

+        raise DevtoolError(e)
+
+    appendpath = os.path.join(config.workspace_path, 'appends')
+    if not os.path.exists(appendpath):
+        os.makedirs(appendpath)
+
+    recipe_file_base =

os.path.basename(os.path.splitext(recipe_file)[0])
+ appendfile = os.path.join(appendpath, '%s.bbappend' %
recipe_file_base)

scripts/lib/devtool/upgrade.py new file +13 -0

+    with open(appendfile, 'w') as f:
+        f.write('inherit externalsrc\n')
+        f.write('EXTERNALSRC = "%s"\n' % srctree)
+        f.write('EXTERNALSRC_BUILD = "%s"\n' % srctree)
+        f.write('\n# initial_rev: %s\n' % initial_rev_upgrade)
+
+    standard._add_md5(config, args.recipename, appendfile)
+    logger.info('Source tree extracted to %s' % srctree)
+    return 0
+
+def register_commands(subparsers, context):
+    """Register devtool subcommands from this plugin"""
+    parser_upgrade = subparsers.add_parser('upgrade', help='Upgrade

an existing recipe',
+ description='Upgrades an
existing recipe',
+ formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+ parser_upgrade.add_argument('recipename', help='Name for recipe
to extract the source for')
+ parser_upgrade.add_argument('srctree', help='Path to where to
extract the source tree')
+ parser_upgrade.add_argument('--version', '-V', help='Version to
upgrade (PV)')
+ parser_upgrade.add_argument('--branch', '-b', default="devtool",
help='Name for development branch to checkout')
+ parser_upgrade.add_argument('--keep-temp', action="store_true",
help='Keep temporary directory (for debugging)')
+ parser_upgrade.add_argument('--no-patch', action="store_true",
help='Do not patch the new source code')
+ parser_upgrade.set_defaults(func=upgrade)

From: Leonardo Sandoval <[email protected]>

Upgrades a recipe to a particular version and downloads the source code into srctree. User can avoid patching the source code. These are the general steps of the upgrade function:
* Extract current recipe source code into srctree and create branch
* Extract upgrade recipe source code into srctree and rebase with previous branch. This step also creates a temporal recipe (created using recipetool), containing the correct checksums.
* Creates the new recipe under the workspace

[YOCTO #7642]

Signed-off-by: Leonardo Sandoval
<[email protected]> ---

 scripts/lib/devtool/standard.py |   4 +-
 scripts/lib/devtool/upgrade.py  | 314
++++++++++++++++++++++++++++++++++++++++ 2 files changed, 317
insertions(+), 1 deletion(-)
 create mode 100644 scripts/lib/devtool/upgrade.py

(quoted diff) +0 -0

b/scripts/lib/devtool/standard.py index ea21877..cd5a3ed 100644

scripts/lib/devtool/standard.py +2 -1

@@ -205,6 +205,8 @@ def extract(args, config, basepath, workspace):
 
     srctree = os.path.abspath(args.srctree)
     initial_rev = _extract_source(srctree, args.keep_temp, args.branch, rd)
+    logger.info('Source tree extracted to %s' % srctree)
+
     if initial_rev:
         return 0
     else:

@@ -360,7 +362,6 @@ def _extract_source(srctree, keep_temp, devbranch, d):
                 bb.process.run('git checkout patches', cwd=srcsubdir)
 
         shutil.move(srcsubdir, srctree)
-        logger.info('Source tree extracted to %s' % srctree)
     finally:
         bb.logger.setLevel(origlevel)
 

@@ -439,6 +440,7 @@ def modify(args, config, basepath, workspace):
         initial_rev = _extract_source(args.srctree, False, args.branch, rd)

if not initial_rev:
return 1
+ logger.info('Source tree extracted to %s' % srctree)
# Get list of commits since this revision
(stdout, _) = bb.process.run('git rev-list --reverse %s..HEAD' %
initial_rev, cwd=args.srctree) commits = stdout.split()

scripts/lib/devtool/upgrade.py new file +47 -0

new file mode 100644
index 0000000..9bef984
@@ -0,0 +1,314 @@
+# Development tool - upgrade command plugin
+#
+# Copyright (C) 2014-2015 Intel Corporation
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# DESCRIPTION
+# Created a new recipe under workspace/recipes/<recipename> and place the
+# source code into <srctree>.
+# The upgrade feature executes the following steps:
+#     * Extract current recipe source code into srctree and create branch
+#     * Extract upgrade recipe source code into srctree and rebase with
+#       previous branch. This step also creates a temporal recipe (created
+#       using recipetool), containing the correct checksums.
+#     * Creates the new recipe under the workspace
+"""Devtool upgrade plugin"""
+
+import os
+import sys
+import re
+import shutil
+import tempfile
+import logging
+import argparse
+import scriptutils
+import errno
+from devtool import standard
+from devtool import exec_build_env_command, setup_tinfoil, DevtoolError
+
+logger = logging.getLogger('devtool')
+
+def plugin_init(pluginlist):
+    """Plugin initialization"""
+    pass
+
+def _extract_upgrade_source(args, devbranch, config, basepath, d,

recipepostfix='tmp'): + """Extract sources of a recipe with PV given on
args.version

scripts/lib/devtool/upgrade.py new file +2 -0

+
+    On the target source tree folder, a new branch

(<devbranch>_<args.version>) + and tag (<devbranch>-base_<args.version>)
will be created. In case patches + are applied, another tag is created
(<devbranch>-patched_<args.version>). +

scripts/lib/devtool/upgrade.py new file +2 -0

+    Returns: 1) The (git) initial revision ID
+             2) The full path of a temporal recipe containing the correct

checksums + """

scripts/lib/devtool/upgrade.py new file +25 -0

+    import oe.recipeutils
+
+    initial_rev = None
+    srctree = os.path.abspath(args.srctree)
+
+    pn = d.getVar('PN', True)
+
+    standard._check_compatible_recipe(pn, d)
+
+    recipepath = os.path.join(config.workspace_path, 'recipes', pn)
+    bb.utils.mkdirhier(recipepath)
+    recipefile = os.path.join(recipepath, "%s-%s.bb" % (pn,recipepostfix))
+    tmpsrcbasetree = tempfile.mkdtemp(prefix='devtool')
+
+    # Change PV and get URL
+    crd = d.createCopy()
+    pv = d.getVar('PV', True)
+    crd.setVar('PV', args.version)
+    src_uri = crd.getVar('SRC_URI', True)
+    if src_uri:
+        url = src_uri.split()[0]
+
+    # Generate recipe and fetch new source
+    try:
+        cmdopts = "-o %s -x %s -V %s" % (recipefile, tmpsrcbasetree,

args.version) + cmd = 'recipetool create "%s" %s' % (url, cmdopts)

scripts/lib/devtool/upgrade.py new file +3 -0

+        stdout, _ = exec_build_env_command(config.init_path, basepath, cmd)
+    except bb.process.ExecutionError as e:
+        raise DevtoolError('Command \'%s\' failed:\n%s' % (e.command,

e.stdout)) +

scripts/lib/devtool/upgrade.py new file +4 -0

+    tmpsrctree = os.path.join(tmpsrcbasetree, pn + '-' + args.version)
+
+    try:
+        # branch from devtool-base (original source code without patches)

before copying new source code + bb.process.run('git checkout -b %s
devtool-base' % devbranch, cwd=srctree) + bb.process.run('git tag -f
devtool-base_%s' % args.version, cwd=srctree) +

scripts/lib/devtool/upgrade.py new file +15 -1

+        # Copy tmpsrctree  into srctree
+        src_files = standard._ls_tree(tmpsrctree)
+        for path in src_files:
+            tgt_dir = os.path.join(srctree, os.path.dirname(path))
+            bb.utils.mkdirhier(tgt_dir)
+            tgt_path = os.path.join(srctree, path)
+            os.rename(os.path.join(tmpsrctree, path), tgt_path)
+
+        # Track modified and untracked files
+        (stdout,_) = bb.process.run('git ls-files --modified --others
--exclude-standard', cwd=srctree) +        add_files = stdout.splitlines()
+        for add_file in add_files:
+            bb.process.run('git add "%s"' % add_file, cwd=srctree)
+
+        if len(add_files):
+            bb.process.run('git commit -q -m "Initial commit from upstream

at version %s"' % args.version, cwd=srctree) + (stdout, _) =
bb.process.run('git rev-parse HEAD', cwd=srctree) + initial_rev =
stdout.rstrip()

scripts/lib/devtool/upgrade.py new file +5 -0

+
+        if args.no_patch:
+            patches = oe.recipeutils.get_recipe_patches(crd)
+            if len(patches):
+                logger.warn('By user choice, the following (%s_%s) patches

will NOT be applied into' %(pn,pv)) + for patch in patches:

scripts/lib/devtool/upgrade.py new file +8 -0

+                    logger.warn("\t%s" % os.path.basename(patch))
+        else:
+            try:
+                bb.process.run('git rebase devtool-patched', cwd=srctree)
+            except bb.process.ExecutionError as e:
+                # this is the only case where we do not propagate the error
+                # user will have the change to correct merges
+                logger.error('Command \'%s\' failed:\n%s' % (e.command,

e.stdout))
+ bb.process.run('git tag -f devtool-patched_%s' %
args.version, cwd=srctree)

scripts/lib/devtool/upgrade.py new file +2 -0

+    except bb.process.ExecutionError as e:
+        raise DevtoolError('Command \'%s\' failed:\n%s' % (e.command,

e.stdout)) + finally:

scripts/lib/devtool/upgrade.py new file +2 -0

+        if args.keep_temp:
+            logger.info('Preserving temporary directory %s' %

tmpsrcbasetree) + else:

scripts/lib/devtool/upgrade.py new file +29 -0

+            shutil.rmtree(tmpsrcbasetree)
+    return initial_rev, recipefile
+
+def _create_upgrade_recipe(args, config, recipetmp, config_data, d):
+    """Creates the new recipe under workspace
+
+    Returns the full path on the new created recipe
+    """
+    import oe.recipeutils
+
+    def _get_checksums(recipefile):
+        import re
+        checksums = {}
+        with open(recipefile) as rf:
+            for line in rf:
+                for cs in ['md5sum', 'sha256sum']:
+                    m = re.match("^SRC_URI\[%s\].*=.*\"(.*)\"" % cs, line)
+                    if m:
+                        checksums[cs] = m.group(1)
+        return checksums
+
+    def _replace_checksums(recipefile, checksums):
+        import re
+        with open(recipefile + ".tmp", "w+") as tmprf:
+            with open(recipefile) as rf:
+                for line in rf:
+                    m = None
+                    for cs in ['md5sum', 'sha256sum']:
+                        m = re.match("^SRC_URI\[%s\].*=.*\"(.*)\"" % cs,

line) + if m:

scripts/lib/devtool/upgrade.py new file +36 -0

+    def _rename_patch_dirs(recipefolder, oldpv, newpv):
+        for root, dirs, files in os.walk(recipefolder):
+            for olddir in dirs:
+                if olddir.find(oldpv) != -1:
+                    newdir = olddir.replace(oldpv, newpv)
+                    bb.process.run('mv %s %s' % (olddir, newdir))
+
+    def _remove_patch_dirs(recipefolder):
+        for root, dirs, files in os.walk(recipefolder):
+            for d in dirs:
+                shutil.rmtree(os.path.join(root,d))
+
+    def _recipe_contains(recipefile, var):
+        import re
+        found = False
+        with open(recipefile) as rf:
+            for line in rf:
+                if re.match("^%s.*=.*" % var, line):
+                    found = True
+                    break
+        return found
+
+    if not os.path.exists(recipetmp):
+        raise DevtoolError("Temporal recipe %s does not exist" % recipetmp)
+
+    # Copy current recipe into workspace
+    pn = d.getVar('PN', True)
+    recipepath = os.path.join(config.workspace_path, 'recipes', pn)
+    oe.recipeutils.copy_recipe_files(d, recipepath)
+
+    # Rename recipe
+    pv = d.getVar('PV', True)
+    recipename = "%s_%s.bb" % (pn, pv)
+    newrecipename = "%s_%s.bb" % (pn, args.version)
+    if os.path.isfile(os.path.join(recipepath, recipename)):
+        bb.process.run('mv %s %s' % (recipename, newrecipename),

cwd=recipepath)

scripts/lib/devtool/upgrade.py new file +12 -0

+    else:
+        # Check if it is a git recipe
+        recipename = newrecipename = "%s_git.bb" % pn
+        if not os.path.isfile(os.path.join(recipepath, recipename)):
+            raise DevtoolError("Original recipe not found on workspace")
+
+    # Rename folders
+    _rename_patch_dirs(recipepath, pv, args.version)
+
+    # Update PV, just in case it is present
+    if _recipe_contains(os.path.join(recipepath, newrecipename), 'PV'):
+        oe.recipeutils.patch_recipe(d, os.path.join(recipepath,

newrecipename), {'PV':args.version}) +

scripts/lib/devtool/upgrade.py new file +32 -1

+    # Update checksums
+    checksums = _get_checksums(os.path.join(recipepath, recipetmp))
+    _replace_checksums(os.path.join(recipepath, newrecipename), checksums)
+
+    # Remove recipe created by the recipe-tool
+    bb.process.run('rm %s' % recipetmp)
+
+    return os.path.join(recipepath,newrecipename)
+
+def upgrade(args, config, basepath, workspace):
+    """Entry point for the devtool 'upgrade' subcommand"""
+    import bb
+    import oe.recipeutils
+
+    if args.recipename in workspace:
+        raise DevtoolError("recipe %s is already in your workspace" %
+                            args.recipename)
+    if not args.version:
+        raise DevtoolError("Provide a version through the parameter
--version/-V") +
+    reason = oe.recipeutils.validate_pn(args.recipename)
+    if reason:
+        raise DevtoolError(reason)
+
+    tinfoil = setup_tinfoil()
+
+    rd = standard._parse_recipe(config, tinfoil, args.recipename, True)
+    if not rd:
+        return 1
+
+    pn = rd.getVar('PV', True)
+    if pn == args.version:
+        raise DevtoolError("Current and upgrade versions are the same %s" %

pn) +

scripts/lib/devtool/upgrade.py new file +5 -0

+    srctree = os.path.abspath(args.srctree)
+
+    try:
+        # Extract source from current recipe
+        initial_rev_base = standard._extract_source(srctree, False,

args.branch, rd) +
+ # We need to shutdown tinfoil temporally because recipetool will be
used

scripts/lib/devtool/upgrade.py new file +5 -0

+        tinfoil.shutdown()
+
+        # Extract new source code
+        branch_upgrade = "%s_%s" % (args.branch, args.version)
+        initial_rev_upgrade, recipe_file_tmp =

_extract_upgrade_source(args, branch_upgrade, config, basepath, rd) +
# Start again tinfoil

scripts/lib/devtool/upgrade.py new file +3 -0

+        tinfoil = setup_tinfoil()
+
+        recipe_file = _create_upgrade_recipe(args, config, recipe_file_tmp,

tinfoil.config_data, rd) +

scripts/lib/devtool/upgrade.py new file +15 -0

+    except DevtoolError as e:
+        logger.error(e)
+        # remove workspace recipe and srctree
+        pn = rd.getVar('PN', True)
+        recipespath = os.path.join(config.workspace_path, 'recipes')
+        recipepath = os.path.join(recipespath, pn)
+        if os.path.exists(recipepath):
+            shutil.rmtree(recipepath)
+            if not len(os.listdir(recipespath)):
+                os.rmdir(recipespath)
+        # remove srctree
+        if os.path.exists(srctree):
+            shutil.rmtree(srctree)
+        raise DevtoolError(e)
+

Quoted 10 lines · On Wednesday 26 August 2015 07:43:23 [email protected]…

From: Leonardo Sandoval <[email protected]>

Upgrades a recipe to a particular version and downloads the source code into srctree. User can avoid patching the source code. These are the general steps of the upgrade function:
* Extract current recipe source code into srctree and create branch
* Extract upgrade recipe source code into srctree and rebase with previous branch. This step also creates a temporal recipe (created using recipetool), containing the correct checksums.
* Creates the new recipe under the workspace

OK, I haven't tested this yet but some comments just looking over the code below.

Quoted 171 lines

[YOCTO #7642]

Signed-off-by: Leonardo Sandoval
<[email protected]> ---

  scripts/lib/devtool/standard.py |   4 +-
  scripts/lib/devtool/upgrade.py  | 314

++++++++++++++++++++++++++++++++++++++++ 2 files changed, 317 insertions(+), 1 deletion(-)
create mode 100644 scripts/lib/devtool/upgrade.py

(quoted diff) +0 -0

b/scripts/lib/devtool/standard.py index ea21877..cd5a3ed 100644

scripts/lib/devtool/standard.py +2 -1

@@ -205,6 +205,8 @@ def extract(args, config, basepath, workspace):
 
      srctree = os.path.abspath(args.srctree)
      initial_rev = _extract_source(srctree, args.keep_temp, args.branch, rd)
+    logger.info('Source tree extracted to %s' % srctree)
+
      if initial_rev:
          return 0
      else:

@@ -360,7 +362,6 @@ def _extract_source(srctree, keep_temp, devbranch, d):
                  bb.process.run('git checkout patches', cwd=srcsubdir)
 
          shutil.move(srcsubdir, srctree)
-        logger.info('Source tree extracted to %s' % srctree)
      finally:
          bb.logger.setLevel(origlevel)
 

@@ -439,6 +440,7 @@ def modify(args, config, basepath, workspace):
          initial_rev = _extract_source(args.srctree, False, args.branch, rd)

if not initial_rev:
return 1
+ logger.info('Source tree extracted to %s' % srctree)
# Get list of commits since this revision
(stdout, _) = bb.process.run('git rev-list --reverse %s..HEAD' %
initial_rev, cwd=args.srctree) commits = stdout.split()

scripts/lib/devtool/upgrade.py new file +47 -0

new file mode 100644
index 0000000..9bef984
@@ -0,0 +1,314 @@
+# Development tool - upgrade command plugin
+#
+# Copyright (C) 2014-2015 Intel Corporation
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# DESCRIPTION
+# Created a new recipe under workspace/recipes/<recipename> and place the
+# source code into <srctree>.
+# The upgrade feature executes the following steps:
+#     * Extract current recipe source code into srctree and create branch
+#     * Extract upgrade recipe source code into srctree and rebase with
+#       previous branch. This step also creates a temporal recipe (created
+#       using recipetool), containing the correct checksums.
+#     * Creates the new recipe under the workspace
+"""Devtool upgrade plugin"""
+
+import os
+import sys
+import re
+import shutil
+import tempfile
+import logging
+import argparse
+import scriptutils
+import errno
+from devtool import standard
+from devtool import exec_build_env_command, setup_tinfoil, DevtoolError
+
+logger = logging.getLogger('devtool')
+
+def plugin_init(pluginlist):
+    """Plugin initialization"""
+    pass
+
+def _extract_upgrade_source(args, devbranch, config, basepath, d,

recipepostfix='tmp'): + """Extract sources of a recipe with PV given on
args.version

scripts/lib/devtool/upgrade.py new file +2 -0

+
+    On the target source tree folder, a new branch

(<devbranch>_<args.version>) + and tag (<devbranch>-base_<args.version>)
will be created. In case patches + are applied, another tag is created
(<devbranch>-patched_<args.version>). +

scripts/lib/devtool/upgrade.py new file +2 -0

+    Returns: 1) The (git) initial revision ID
+             2) The full path of a temporal recipe containing the correct

checksums + """

scripts/lib/devtool/upgrade.py new file +25 -0

+    import oe.recipeutils
+
+    initial_rev = None
+    srctree = os.path.abspath(args.srctree)
+
+    pn = d.getVar('PN', True)
+
+    standard._check_compatible_recipe(pn, d)
+
+    recipepath = os.path.join(config.workspace_path, 'recipes', pn)
+    bb.utils.mkdirhier(recipepath)
+    recipefile = os.path.join(recipepath, "%s-%s.bb" % (pn,recipepostfix))
+    tmpsrcbasetree = tempfile.mkdtemp(prefix='devtool')
+
+    # Change PV and get URL
+    crd = d.createCopy()
+    pv = d.getVar('PV', True)
+    crd.setVar('PV', args.version)
+    src_uri = crd.getVar('SRC_URI', True)
+    if src_uri:
+        url = src_uri.split()[0]
+
+    # Generate recipe and fetch new source
+    try:
+        cmdopts = "-o %s -x %s -V %s" % (recipefile, tmpsrcbasetree,

args.version) + cmd = 'recipetool create "%s" %s' % (url, cmdopts)

scripts/lib/devtool/upgrade.py new file +3 -0

+        stdout, _ = exec_build_env_command(config.init_path, basepath, cmd)
+    except bb.process.ExecutionError as e:
+        raise DevtoolError('Command \'%s\' failed:\n%s' % (e.command,

e.stdout)) +

scripts/lib/devtool/upgrade.py new file +4 -0

+    tmpsrctree = os.path.join(tmpsrcbasetree, pn + '-' + args.version)
+
+    try:
+        # branch from devtool-base (original source code without patches)

before copying new source code + bb.process.run('git checkout -b %s
devtool-base' % devbranch, cwd=srctree) + bb.process.run('git tag -f
devtool-base_%s' % args.version, cwd=srctree) +

scripts/lib/devtool/upgrade.py new file +15 -1

+        # Copy tmpsrctree  into srctree
+        src_files = standard._ls_tree(tmpsrctree)
+        for path in src_files:
+            tgt_dir = os.path.join(srctree, os.path.dirname(path))
+            bb.utils.mkdirhier(tgt_dir)
+            tgt_path = os.path.join(srctree, path)
+            os.rename(os.path.join(tmpsrctree, path), tgt_path)
+
+        # Track modified and untracked files
+        (stdout,_) = bb.process.run('git ls-files --modified --others
--exclude-standard', cwd=srctree) +        add_files = stdout.splitlines()
+        for add_file in add_files:
+            bb.process.run('git add "%s"' % add_file, cwd=srctree)
+
+        if len(add_files):
+            bb.process.run('git commit -q -m "Initial commit from upstream

at version %s"' % args.version, cwd=srctree) + (stdout, _) =
bb.process.run('git rev-parse HEAD', cwd=srctree) + initial_rev =
stdout.rstrip()

scripts/lib/devtool/upgrade.py new file +5 -0

+
+        if args.no_patch:
+            patches = oe.recipeutils.get_recipe_patches(crd)
+            if len(patches):
+                logger.warn('By user choice, the following (%s_%s) patches

will NOT be applied into' %(pn,pv)) + for patch in patches:

scripts/lib/devtool/upgrade.py new file +5 -0

+                    logger.warn("\t%s" % os.path.basename(patch))
+        else:
+            try:
+                bb.process.run('git rebase devtool-patched', cwd=srctree)
+            except bb.process.ExecutionError as e:

Quoted 6 lines

scripts/lib/devtool/upgrade.py new file +3 -0

+                # this is the only case where we do not propagate the error
+                # user will have the change to correct merges
+                logger.error('Command \'%s\' failed:\n%s' % (e.command,

e.stdout))
+ bb.process.run('git tag -f devtool-patched_%s' %
args.version, cwd=srctree)

Could you use 'devtool-patched-%s' so we avoid mixing - and _ here ?

Quoted 41 lines

scripts/lib/devtool/upgrade.py new file +2 -0

+    except bb.process.ExecutionError as e:
+        raise DevtoolError('Command \'%s\' failed:\n%s' % (e.command,

e.stdout)) + finally:

scripts/lib/devtool/upgrade.py new file +2 -0

+        if args.keep_temp:
+            logger.info('Preserving temporary directory %s' %

tmpsrcbasetree) + else:

scripts/lib/devtool/upgrade.py new file +29 -0

+            shutil.rmtree(tmpsrcbasetree)
+    return initial_rev, recipefile
+
+def _create_upgrade_recipe(args, config, recipetmp, config_data, d):
+    """Creates the new recipe under workspace
+
+    Returns the full path on the new created recipe
+    """
+    import oe.recipeutils
+
+    def _get_checksums(recipefile):
+        import re
+        checksums = {}
+        with open(recipefile) as rf:
+            for line in rf:
+                for cs in ['md5sum', 'sha256sum']:
+                    m = re.match("^SRC_URI\[%s\].*=.*\"(.*)\"" % cs, line)
+                    if m:
+                        checksums[cs] = m.group(1)
+        return checksums
+
+    def _replace_checksums(recipefile, checksums):
+        import re
+        with open(recipefile + ".tmp", "w+") as tmprf:
+            with open(recipefile) as rf:
+                for line in rf:
+                    m = None
+                    for cs in ['md5sum', 'sha256sum']:
+                        m = re.match("^SRC_URI\[%s\].*=.*\"(.*)\"" % cs,

line) + if m:

scripts/lib/devtool/upgrade.py new file +4 -0

+                            if cs in checksums:
+                                oldcheck = m.group(1)
+                                newcheck = checksums[cs]
+                                line = line.replace(oldcheck, newcheck)

scripts/lib/devtool/upgrade.py line 170

Quoted 4 lines

scripts/lib/devtool/upgrade.py new file +4 -0

+                            break
+                    tmprf.write(line)
+
+        os.rename(recipefile + ".tmp", recipefile)

We already have code for editing recipes like this in recipeutils (and in lib/bb/utils as well) - in fact I see you're using patch_recipe() later on, so why do it by hand here?

Quoted 123 lines

scripts/lib/devtool/upgrade.py new file +36 -0

+    def _rename_patch_dirs(recipefolder, oldpv, newpv):
+        for root, dirs, files in os.walk(recipefolder):
+            for olddir in dirs:
+                if olddir.find(oldpv) != -1:
+                    newdir = olddir.replace(oldpv, newpv)
+                    bb.process.run('mv %s %s' % (olddir, newdir))
+
+    def _remove_patch_dirs(recipefolder):
+        for root, dirs, files in os.walk(recipefolder):
+            for d in dirs:
+                shutil.rmtree(os.path.join(root,d))
+
+    def _recipe_contains(recipefile, var):
+        import re
+        found = False
+        with open(recipefile) as rf:
+            for line in rf:
+                if re.match("^%s.*=.*" % var, line):
+                    found = True
+                    break
+        return found
+
+    if not os.path.exists(recipetmp):
+        raise DevtoolError("Temporal recipe %s does not exist" % recipetmp)
+
+    # Copy current recipe into workspace
+    pn = d.getVar('PN', True)
+    recipepath = os.path.join(config.workspace_path, 'recipes', pn)
+    oe.recipeutils.copy_recipe_files(d, recipepath)
+
+    # Rename recipe
+    pv = d.getVar('PV', True)
+    recipename = "%s_%s.bb" % (pn, pv)
+    newrecipename = "%s_%s.bb" % (pn, args.version)
+    if os.path.isfile(os.path.join(recipepath, recipename)):
+        bb.process.run('mv %s %s' % (recipename, newrecipename),

cwd=recipepath)

scripts/lib/devtool/upgrade.py new file +12 -0

+    else:
+        # Check if it is a git recipe
+        recipename = newrecipename = "%s_git.bb" % pn
+        if not os.path.isfile(os.path.join(recipepath, recipename)):
+            raise DevtoolError("Original recipe not found on workspace")
+
+    # Rename folders
+    _rename_patch_dirs(recipepath, pv, args.version)
+
+    # Update PV, just in case it is present
+    if _recipe_contains(os.path.join(recipepath, newrecipename), 'PV'):
+        oe.recipeutils.patch_recipe(d, os.path.join(recipepath,

newrecipename), {'PV':args.version}) +

scripts/lib/devtool/upgrade.py new file +32 -1

+    # Update checksums
+    checksums = _get_checksums(os.path.join(recipepath, recipetmp))
+    _replace_checksums(os.path.join(recipepath, newrecipename), checksums)
+
+    # Remove recipe created by the recipe-tool
+    bb.process.run('rm %s' % recipetmp)
+
+    return os.path.join(recipepath,newrecipename)
+
+def upgrade(args, config, basepath, workspace):
+    """Entry point for the devtool 'upgrade' subcommand"""
+    import bb
+    import oe.recipeutils
+
+    if args.recipename in workspace:
+        raise DevtoolError("recipe %s is already in your workspace" %
+                            args.recipename)
+    if not args.version:
+        raise DevtoolError("Provide a version through the parameter
--version/-V") +
+    reason = oe.recipeutils.validate_pn(args.recipename)
+    if reason:
+        raise DevtoolError(reason)
+
+    tinfoil = setup_tinfoil()
+
+    rd = standard._parse_recipe(config, tinfoil, args.recipename, True)
+    if not rd:
+        return 1
+
+    pn = rd.getVar('PV', True)
+    if pn == args.version:
+        raise DevtoolError("Current and upgrade versions are the same %s" %

pn) +

scripts/lib/devtool/upgrade.py new file +5 -0

+    srctree = os.path.abspath(args.srctree)
+
+    try:
+        # Extract source from current recipe
+        initial_rev_base = standard._extract_source(srctree, False,

args.branch, rd) +
+ # We need to shutdown tinfoil temporally because recipetool will be
used

scripts/lib/devtool/upgrade.py new file +5 -0

+        tinfoil.shutdown()
+
+        # Extract new source code
+        branch_upgrade = "%s_%s" % (args.branch, args.version)
+        initial_rev_upgrade, recipe_file_tmp =

_extract_upgrade_source(args, branch_upgrade, config, basepath, rd) +
# Start again tinfoil

scripts/lib/devtool/upgrade.py new file +3 -0

+        tinfoil = setup_tinfoil()
+
+        recipe_file = _create_upgrade_recipe(args, config, recipe_file_tmp,

tinfoil.config_data, rd) +

scripts/lib/devtool/upgrade.py new file +20 -0

+    except DevtoolError as e:
+        logger.error(e)
+        # remove workspace recipe and srctree
+        pn = rd.getVar('PN', True)
+        recipespath = os.path.join(config.workspace_path, 'recipes')
+        recipepath = os.path.join(recipespath, pn)
+        if os.path.exists(recipepath):
+            shutil.rmtree(recipepath)
+            if not len(os.listdir(recipespath)):
+                os.rmdir(recipespath)
+        # remove srctree
+        if os.path.exists(srctree):
+            shutil.rmtree(srctree)
+        raise DevtoolError(e)
+
+    appendpath = os.path.join(config.workspace_path, 'appends')
+    if not os.path.exists(appendpath):
+        os.makedirs(appendpath)
+
+    recipe_file_base = os.path.basename(os.path.splitext(recipe_file)[0])

scripts/lib/devtool/upgrade.py line 296

Quoted 5 lines

scripts/lib/devtool/upgrade.py new file +5 -0

+    appendfile = os.path.join(appendpath, '%s.bbappend' % recipe_file_base)
+    with open(appendfile, 'w') as f:
+        f.write('inherit externalsrc\n')
+        f.write('EXTERNALSRC = "%s"\n' % srctree)
+        f.write('EXTERNALSRC_BUILD = "%s"\n' % srctree)

Why is this unconditionally setting EXTERNALSRC_BUILD? Surely we should be using the same logic as used elsewhere for this?

Cheers,
Paul

Quoted 5 lines · Paul Eggleton

We already have code for editing recipes like this in recipeutils (and in lib/bb/utils as well) - in fact I see you're using patch_recipe() later on, so
why do it by hand here?

can 'oe.recipeutils.patch_recipe' patch variable's flags? I did not try but looking at the function definition, seems like it does not.

Quoted 7 lines · Leonardo Sandoval

Quoted 5 lines · Paul Eggleton

We already have code for editing recipes like this in recipeutils (and in lib/bb/utils as well) - in fact I see you're using patch_recipe() later on, so
why do it by hand here?

Quoted 2 lines

can 'oe.recipeutils.patch_recipe' patch variable's flags? I did not try but looking at the function definition, seems like it does not.

So add this feature there if you need it.

From: Leonardo Sandoval <[email protected]>

Basic tests for the devtool's upgrade feature, including:
* Parameter check
* Upgrading a real recipe (e2fsprogrs) without patching and checing its output
* Devtool status after upgrade

Signed-off-by Leonardo Sandoval <[email protected]>

---
  meta/lib/oeqa/selftest/devtool.py | 36 ++++++++++++++++++++++++++++++++++++
  1 file changed, 36 insertions(+)

meta/lib/oeqa/selftest/devtool.py +2 -0

index b59db15..f72e010 100644
@@ -857,3 +857,39 @@ class DevtoolTests(DevtoolBase):
              result = runCmd('devtool undeploy-target -c %s root@%s' % (testrecipe, qemu.ip))
              result = runCmd('ssh %s root@%s %s' % (sshargs, qemu.ip, testcommand), ignore_status=True)
              self.assertNotEqual(result, 0, 'undeploy-target did not remove command as it should have')
+
+    def test_devtool_upgrade(self):

meta/lib/oeqa/selftest/devtool.py +24 -0

+        tempdir = tempfile.mkdtemp(prefix='devtoolqa')
+        # Check that recipe is not already under devtool control
+        result = runCmd('devtool status')
+        self.assertNotIn(recipe, result.output)
+        # Check upgrade. Code does not check if new PV is older or newer that current PV, so, it may be that
+        # we are downgrading instead of upgrading.
+        result = runCmd('devtool upgrade %s %s -V %s --no-patch' % (recipe, tempdir, version))
+        # Check if srctree at least is populated
+        self.assertTrue(len(os.listdir(tempdir)) > 0, 'scrtree (%s) should be populated with new (%s) source code' % (tempdir, version))
+        # Check new recipe folder is present
+        self.assertTrue(os.path.exists(os.path.join(workspacedir,'recipes',recipe)), 'Recipe folder should exist')
+        # Check new recipe file is present
+        self.assertTrue(os.path.exists(os.path.join(workspacedir,'recipes',recipe,"%s_%s.bb" % (recipe,version))), 'Recipe folder should exist')
+        # Check devtool status and make sure recipe is present
+        result = runCmd('devtool status')
+        self.assertIn(recipe, result.output)
+        self.assertIn(tempdir, result.output)
+        # Check devtool reset recipe
+        result = runCmd('devtool reset %s -n' % recipe)
+        result = runCmd('devtool status')
+        self.assertNotIn(recipe, result.output)
+        self.track_for_cleanup(tempdir)
+        self.track_for_cleanup(workspacedir)
+        self.add_command_to_tearDown('bitbake-layers remove-layer */workspace')

Comments below.

Quoted 36 lines · On 26/08/15 02:43, [email protected] wrote:

From: Leonardo Sandoval <[email protected]>

Basic tests for the devtool's upgrade feature, including:
* Parameter check
* Upgrading a real recipe (e2fsprogrs) without patching and checing its output
* Devtool status after upgrade

Signed-off-by: Leonardo Sandoval
<[email protected]>

---
  meta/lib/oeqa/selftest/devtool.py | 36
++++++++++++++++++++++++++++++++++++
  1 file changed, 36 insertions(+)

meta/lib/oeqa/selftest/devtool.py +0 -0

index b59db15..f72e010 100644
@@ -857,3 +857,39 @@ class DevtoolTests(DevtoolBase):
              result = runCmd('devtool undeploy-target -c %s root@%s'
 % (testrecipe, qemu.ip))
              result = runCmd('ssh %s root@%s %s' % (sshargs, qemu.ip,

testcommand), ignore_status=True)
self.assertNotEqual(result, 0, 'undeploy-target did not
remove command as it should have')

meta/lib/oeqa/selftest/devtool.py +5 -0

+
+    def test_devtool_upgrade(self):
+        # Check preconditions
+        workspacedir = os.path.join(self.builddir, 'workspace')
+        self.assertTrue(not os.path.exists(workspacedir), 'This test

cannot be run with a workspace directory under the build directory')

meta/lib/oeqa/selftest/devtool.py +3 -0

+        # Check parameters
+        result = runCmd('devtool upgrade -h')
+        for param in 'recipename srctree --version -V --branch -b

meta/lib/oeqa/selftest/devtool.py class DevtoolTests(DevtoolBase): · line 871

Quoted 5 lines

--keep-temp --no-patch'.split():

meta/lib/oeqa/selftest/devtool.py +4 -0

+            self.assertIn(param, result.output)
+        # For the moment, we are using a real recipe.
+        recipe='e2fsprogs'
+        version='1.42.13'

Isn't a guarantee that this version will be newer ever (i.e. if someone upgrade e2fsprogs to 1.42.13) what happen in this case?

Quoted 32 lines

meta/lib/oeqa/selftest/devtool.py +5 -0

+        tempdir = tempfile.mkdtemp(prefix='devtoolqa')
+        # Check that recipe is not already under devtool control
+        result = runCmd('devtool status')
+        self.assertNotIn(recipe, result.output)
+        # Check upgrade. Code does not check if new PV is older or

newer that current PV, so, it may be that

meta/lib/oeqa/selftest/devtool.py +2 -0

+        # we are downgrading instead of upgrading.
+        result = runCmd('devtool upgrade %s %s -V %s --no-patch' %

(recipe, tempdir, version))

meta/lib/oeqa/selftest/devtool.py +2 -0

+        # Check if srctree at least is populated
+        self.assertTrue(len(os.listdir(tempdir)) > 0, 'scrtree (%s)

should be populated with new (%s) source code' % (tempdir, version))

meta/lib/oeqa/selftest/devtool.py +2 -0

+        # Check new recipe folder is present
+

self.assertTrue(os.path.exists(os.path.join(workspacedir,'recipes',recipe)), 'Recipe folder should exist')

meta/lib/oeqa/selftest/devtool.py +2 -0

+        # Check new recipe file is present
+

self.assertTrue(os.path.exists(os.path.join(workspacedir,'recipes',recipe,"%s_%s.bb" % (recipe,version))), 'Recipe folder should exist')

meta/lib/oeqa/selftest/devtool.py +11 -0

+        # Check devtool status and make sure recipe is present
+        result = runCmd('devtool status')
+        self.assertIn(recipe, result.output)
+        self.assertIn(tempdir, result.output)
+        # Check devtool reset recipe
+        result = runCmd('devtool reset %s -n' % recipe)
+        result = runCmd('devtool status')
+        self.assertNotIn(recipe, result.output)
+        self.track_for_cleanup(tempdir)
+        self.track_for_cleanup(workspacedir)
+        self.add_command_to_tearDown('bitbake-layers remove-layer

*/workspace')

Quoted 50 lines · Aníbal Limón

Comments below.

Quoted 47 lines · On 26/08/15 02:43, [email protected] wrote:

From: Leonardo Sandoval <[email protected]>

Basic tests for the devtool's upgrade feature, including:
* Parameter check
* Upgrading a real recipe (e2fsprogrs) without patching and

checing its output

* Devtool status after upgrade

Signed-off-by: Leonardo Sandoval
<[email protected]>
---

  meta/lib/oeqa/selftest/devtool.py | 36

++++++++++++++++++++++++++++++++++++

  1 file changed, 36 insertions(+)

meta/lib/oeqa/selftest/devtool.py +0 -0

index b59db15..f72e010 100644
@@ -857,3 +857,39 @@ class DevtoolTests(DevtoolBase):
              result = runCmd('devtool undeploy-target -c %s root@%s'
 
 % (testrecipe, qemu.ip))
 
              result = runCmd('ssh %s root@%s %s' % (sshargs, qemu.ip,

testcommand), ignore_status=True)

self.assertNotEqual(result, 0, 'undeploy-target did not

remove command as it should have')

meta/lib/oeqa/selftest/devtool.py +5 -0

+
+    def test_devtool_upgrade(self):
+        # Check preconditions
+        workspacedir = os.path.join(self.builddir, 'workspace')
+        self.assertTrue(not os.path.exists(workspacedir), 'This test

cannot be run with a workspace directory under the build directory')

meta/lib/oeqa/selftest/devtool.py +3 -0

+        # Check parameters
+        result = runCmd('devtool upgrade -h')
+        for param in 'recipename srctree --version -V --branch -b

Quoted 8 lines

meta/lib/oeqa/selftest/devtool.py class DevtoolTests(DevtoolBase): · line 871

Quoted 5 lines

--keep-temp --no-patch'.split():

meta/lib/oeqa/selftest/devtool.py +4 -0

+            self.assertIn(param, result.output)
+        # For the moment, we are using a real recipe.
+        recipe='e2fsprogs'
+        version='1.42.13'

Isn't a guarantee that this version will be newer ever (i.e. if someone upgrade e2fsprogs to 1.42.13) what happen in this case?

Good question. In fact, the code will fail just in case current and upgrade version are the same. The tool can also do downgrades, so at the end it does not matter what the version is. I will correct this point and send a v3.

As mentioned by Paul in a previous message, we need 'upgradable' recipes which whenever the test is done, it can be upgrade. The only point here is that we need a tarball somewhere, so the URL is always live. Not sure if poky repository is a good place for the latter, any suggestion?

Quoted 65 lines · On Wednesday 26 August 2015 11:29:37 Leonardo Sandoval wrote:

Quoted 50 lines · Aníbal Limón

Comments below.

Quoted 47 lines · On 26/08/15 02:43, [email protected] wrote:

From: Leonardo Sandoval <[email protected]>

Basic tests for the devtool's upgrade feature, including:
* Parameter check
* Upgrading a real recipe (e2fsprogrs) without patching and

checing its output

* Devtool status after upgrade

Signed-off-by: Leonardo Sandoval
<[email protected]>
---

   meta/lib/oeqa/selftest/devtool.py | 36

++++++++++++++++++++++++++++++++++++

   1 file changed, 36 insertions(+)

meta/lib/oeqa/selftest/devtool.py +0 -0

index b59db15..f72e010 100644
@@ -857,3 +857,39 @@ class DevtoolTests(DevtoolBase):
               result = runCmd('devtool undeploy-target -c %s root@%s'
 
 % (testrecipe, qemu.ip))
 
               result = runCmd('ssh %s root@%s %s' % (sshargs, qemu.ip,

testcommand), ignore_status=True)

self.assertNotEqual(result, 0, 'undeploy-target did not

remove command as it should have')

meta/lib/oeqa/selftest/devtool.py +5 -0

+
+    def test_devtool_upgrade(self):
+        # Check preconditions
+        workspacedir = os.path.join(self.builddir, 'workspace')
+        self.assertTrue(not os.path.exists(workspacedir), 'This test

cannot be run with a workspace directory under the build directory')

meta/lib/oeqa/selftest/devtool.py +3 -0

+        # Check parameters
+        result = runCmd('devtool upgrade -h')
+        for param in 'recipename srctree --version -V --branch -b

Quoted 8 lines

meta/lib/oeqa/selftest/devtool.py class DevtoolTests(DevtoolBase): · line 871

Quoted 5 lines

--keep-temp --no-patch'.split():

meta/lib/oeqa/selftest/devtool.py +4 -0

+            self.assertIn(param, result.output)
+        # For the moment, we are using a real recipe.
+        recipe='e2fsprogs'
+        version='1.42.13'

Isn't a guarantee that this version will be newer ever (i.e. if someone upgrade e2fsprogs to 1.42.13) what happen in this case?

Good question. In fact, the code will fail just in case current and upgrade version are the same. The tool can also do downgrades, so at the end it does not matter what the version is. I will correct this point and send a v3.

Quoted 4 lines

As mentioned by Paul in a previous message, we need 'upgradable' recipes which whenever the test is done, it can be upgrade. The only point here is that we need a tarball somewhere, so the URL is always live. Not sure if poky repository is a good place for the latter, any suggestion?

This is what we have meta-selftest for. We can put a recipe (or multiple recipes) in there that's named specially so as not to overlap with one in OE- Core and is a known old version.

Cheers,
Paul