devtool / recipetool improvements

index ca474fc..1532735 100644
@@ -613,9 +613,9 @@ def create_recipe(args):
 
     if args.src_subdir:
         srcsubdir = os.path.join(srcsubdir, args.src_subdir)
-        srctree_use = os.path.join(srctree, args.src_subdir)
+        srctree_use = os.path.abspath(os.path.join(srctree, args.src_subdir))
     else:
-        srctree_use = srctree
+        srctree_use = os.path.abspath(srctree)
 
     if args.outfile and os.path.isdir(args.outfile):
         outfile = None

index 1532735..055bdef 100644
@@ -156,10 +156,12 @@ class RecipeHandler(object):
                         RecipeHandler.recipebinmap[prog] = pn
 
     @staticmethod
-    def checkfiles(path, speclist, recursive=False):
+    def checkfiles(path, speclist, recursive=False, excludedirs=None):
         results = []
         if recursive:
-            for root, _, files in os.walk(path):
+            for root, dirs, files in os.walk(path, topdown=True):
+                if excludedirs:
+                    dirs[:] = [d for d in dirs if d not in excludedirs]
                 for fn in files:
                     for spec in speclist:
                         if fnmatch.fnmatch(fn, spec):

index 7cf188d..4569b53 100644
@@ -40,7 +40,7 @@ class KernelModuleRecipeHandler(RecipeHandler):
 
         makefiles = []
 
-        files = RecipeHandler.checkfiles(srctree, ['*.c', '*.h'], recursive=True)
+        files = RecipeHandler.checkfiles(srctree, ['*.c', '*.h'], recursive=True, excludedirs=['contrib', 'test', 'examples'])
         if files:
             for cfile in files:
                 # Look in same dir or parent for Makefile

index c8570ac..2f818bc 100644
@@ -801,7 +801,7 @@ def bbappend_recipe(rd, destlayerdir, srcfiles, install=None, wildcardver=False,
 
 def find_layerdir(fn):
     """ Figure out the path to the base of the layer containing a file (e.g. a recipe)"""
-    pth = fn
+    pth = os.path.abspath(fn)
     layerdir = ''
     while pth:
         if os.path.exists(os.path.join(pth, 'conf', 'layer.conf')):

index 2f818bc..cab8e40 100644
@@ -188,6 +188,11 @@ def patch_recipe_lines(fromlines, values, trailing_newline=True):
             for wrapline in wrapped[:-1]:
                 addlines.append('%s \\%s' % (wrapline, newline))
             addlines.append('%s%s' % (wrapped[-1], newline))
+
+        # Split on newlines - this isn't strictly necessary if you are only
+        # going to write the output to disk, but if you want to compare it
+        # (as patch_recipe_file() will do if patch=True) then it's important.
+        addlines = [line for l in addlines for line in l.splitlines(True)]
         if rewindcomments:
             # Ensure we insert the lines before any leading comments
             # (that we'd want to ensure remain leading the next value)

index f1b3ff0..4cfab0c 100644
@@ -433,7 +433,7 @@ def upgrade(args, config, basepath, workspace):
                         copied, config.workspace_path, rd)
         standard._add_md5(config, pn, af)
 
-        update_unlockedsigs(basepath, workspace, [pn], args.fixed_setup)
+        update_unlockedsigs(basepath, workspace, args.fixed_setup, [pn])
 
         logger.info('Upgraded source extracted to %s' % srctree)
         logger.info('New recipe is %s' % rf)

index 4cfab0c..073002b 100644
@@ -252,7 +252,7 @@ def _extract_new_source(newpv, srctree, no_patch, srcrev, srcbranch, branch, kee
 
         (stdout,_) = __run('git ls-files --modified --others --exclude-standard')
         for f in stdout.splitlines():
-            __run('git add "%s"' % f)
+            __run('git add -A "%s"' % f)
 
         useroptions = []
         oe.patch.GitApplyTree.gitCommandUserOptions(useroptions, d=rd)

index 073002b..441dd35 100644
@@ -251,8 +251,15 @@ def _extract_new_source(newpv, srctree, no_patch, srcrev, srcbranch, branch, kee
         _copy_source_code(tmpsrctree, srctree)
 
         (stdout,_) = __run('git ls-files --modified --others --exclude-standard')
-        for f in stdout.splitlines():
-            __run('git add -A "%s"' % f)
+        filelist = stdout.splitlines()
+        pbar = bb.ui.knotty.BBProgress('Adding changed files', len(filelist))
+        pbar.start()
+        batchsize = 100
+        for i in range(0, len(filelist), batchsize):
+            batch = filelist[i:i+batchsize]
+            __run('git add -A %s' % ' '.join(['"%s"' % item for item in batch]))
+            pbar.update(i)
+        pbar.finish()
 
         useroptions = []
         oe.patch.GitApplyTree.gitCommandUserOptions(useroptions, d=rd)

index 5292f18..87bb5c8 100755
@@ -113,6 +113,7 @@ def read_workspace():
     externalsrc_re = re.compile(r'^EXTERNALSRC(_pn-([^ =]+))? *= *"([^"]*)"$')
     for fn in glob.glob(os.path.join(config.workspace_path, 'appends', '*.bbappend')):
         with open(fn, 'r') as f:
+            pnvalues = {}
             for line in f:
                 res = externalsrc_re.match(line.rstrip())
                 if res:

@@ -125,10 +126,16 @@ def read_workspace():
                                                         bbfile))
                     if recipefile:
                         recipefile = recipefile[0]
-                    workspace[pn] = {'srctree': res.group(3),
-                                     'bbappend': fn,
-                                     'recipefile': recipefile}
-                    logger.debug('Found recipe %s' % workspace[pn])
+                    pnvalues['srctree'] = res.group(3)
+                    pnvalues['bbappend'] = fn
+                    pnvalues['recipefile'] = recipefile
+                elif line.startswith('# srctreebase: '):
+                    pnvalues['srctreebase'] = line.split(':', 1)[1].strip()
+            if pnvalues:
+                if not pnvalues.get('srctreebase', None):
+                    pnvalues['srctreebase'] = pnvalues['srctree']
+                logger.debug('Found recipe %s' % pnvalues)
+                workspace[pn] = pnvalues
 
 def create_workspace(args, config, basepath, workspace):
     if args.layerpath:

index beea0d4c2..8e4c7f7 100644
@@ -734,6 +734,9 @@ def modify(args, config, basepath, workspace):
                     (stdout, _) = bb.process.run('git rev-parse HEAD', cwd=srctree)
                     initial_rev = stdout.rstrip()
 
+        # Need to grab this here in case the source is within a subdirectory
+        srctreebase = srctree
+
         # Check that recipe isn't using a shared workdir
         s = os.path.abspath(rd.getVar('S'))
         workdir = os.path.abspath(rd.getVar('WORKDIR'))

@@ -748,7 +751,8 @@ def modify(args, config, basepath, workspace):
             # Local files can be modified/tracked in separate subdir under srctree
             # Mostly useful for packages with S != WORKDIR
             f.write('FILESPATH_prepend := "%s:"\n' %
-                    os.path.join(srctree, 'oe-local-files'))
+                    os.path.join(srctreebase, 'oe-local-files'))
+            f.write('# srctreebase: %s\n' % srctreebase)
 
             f.write('\ninherit externalsrc\n')
             f.write('# NOTE: We use pn- overrides here to avoid affecting multiple variants in the case where the recipe uses BBCLASSEXTEND\n')

@@ -1166,7 +1170,7 @@ def _create_kconfig_diff(srctree, rd, outfile):
     return False
 
 
-def _export_local_files(srctree, rd, destdir):
+def _export_local_files(srctree, rd, destdir, srctreebase):
     """Copy local files from srctree to given location.
        Returns three-tuple of dicts:
          1. updated - files that already exist in SRCURI

@@ -1186,7 +1190,7 @@ def _export_local_files(srctree, rd, destdir):
     updated = OrderedDict()
     added = OrderedDict()
     removed = OrderedDict()
-    local_files_dir = os.path.join(srctree, 'oe-local-files')
+    local_files_dir = os.path.join(srctreebase, 'oe-local-files')
     git_files = _git_ls_tree(srctree)
     if 'oe-local-files' in git_files:
         # If tracked by Git, take the files from srctree HEAD. First get

@@ -1199,9 +1203,9 @@ def _export_local_files(srctree, rd, destdir):
         new_set = list(_git_ls_tree(srctree, tree, True).keys())
     elif os.path.isdir(local_files_dir):
         # If not tracked by Git, just copy from working copy
-        new_set = _ls_tree(os.path.join(srctree, 'oe-local-files'))
+        new_set = _ls_tree(local_files_dir)
         bb.process.run(['cp', '-ax',
-                        os.path.join(srctree, 'oe-local-files', '.'), destdir])
+                        os.path.join(local_files_dir, '.'), destdir])
     else:
         new_set = []
 

@@ -1266,7 +1270,7 @@ def _determine_files_dir(rd):
     return os.path.join(recipedir, rd.getVar('BPN'))
 
 
-def _update_recipe_srcrev(srctree, rd, appendlayerdir, wildcard_version, no_remove):
+def _update_recipe_srcrev(recipename, workspace, srctree, rd, appendlayerdir, wildcard_version, no_remove):
     """Implement the 'srcrev' mode of update-recipe"""
     import bb
     import oe.recipeutils

@@ -1294,7 +1298,8 @@ def _update_recipe_srcrev(srctree, rd, appendlayerdir, wildcard_version, no_remo
     update_srcuri = False
     try:
         local_files_dir = tempfile.mkdtemp(dir=tempdir)
-        upd_f, new_f, del_f = _export_local_files(srctree, rd, local_files_dir)
+        srctreebase = workspace[recipename]['srctreebase']
+        upd_f, new_f, del_f = _export_local_files(srctree, rd, local_files_dir, srctreebase)
         if not no_remove:
             # Find list of existing patches in recipe file
             patches_dir = tempfile.mkdtemp(dir=tempdir)

@@ -1372,7 +1377,8 @@ def _update_recipe_patch(recipename, workspace, srctree, rd, appendlayerdir, wil
     tempdir = tempfile.mkdtemp(prefix='devtool')
     try:
         local_files_dir = tempfile.mkdtemp(dir=tempdir)
-        upd_f, new_f, del_f = _export_local_files(srctree, rd, local_files_dir)
+        srctreebase = workspace[recipename]['srctreebase']
+        upd_f, new_f, del_f = _export_local_files(srctree, rd, local_files_dir, srctreebase)
 
         remove_files = []
         if not no_remove:

@@ -1495,7 +1501,7 @@ def _update_recipe(recipename, workspace, rd, mode, appendlayerdir, wildcard_ver
         mode = _guess_recipe_update_mode(srctree, rd)
 
     if mode == 'srcrev':
-        updated = _update_recipe_srcrev(srctree, rd, appendlayerdir, wildcard_version, no_remove)
+        updated = _update_recipe_srcrev(recipename, workspace, srctree, rd, appendlayerdir, wildcard_version, no_remove)
     elif mode == 'patch':
         updated = _update_recipe_patch(recipename, workspace, srctree, rd, appendlayerdir, wildcard_version, no_remove, initial_rev)
     else:

index 94e3d7d..20ab83f 100644
@@ -337,3 +337,16 @@ def update_unlockedsigs(basepath, workspace, fixed_setup, extra=None):
             for pn in newunlocked:
                 f.write('    ' + pn)
             f.write('"')
+
+def check_prerelease_version(ver, operation):
+    if 'pre' in ver or 'rc' in ver:
+        logger.warning('Version "%s" looks like a pre-release version. '
+                       'If that is the case, in order to ensure that the '
+                       'version doesn\'t appear to go backwards when you '
+                       'later upgrade to the final release version, it is '
+                       'recommmended that instead you use '
+                       '<current version>+<pre-release version> e.g. if '
+                       'upgrading from 1.9 to 2.0-rc2 use "1.9+2.0-rc2". '
+                       'If you prefer not to reset and re-try, you can change '
+                       'the version after %s succeeds using "devtool rename" '
+                       'with -V/--version.' % (ver, operation))

index 8e4c7f7..b6e532b 100644
@@ -30,7 +30,7 @@ import errno
 import glob
 import filecmp
 from collections import OrderedDict
-from devtool import exec_build_env_command, setup_tinfoil, check_workspace_recipe, use_external_build, setup_git_repo, recipe_to_append, get_bbclassextend_targets, update_unlockedsigs, DevtoolError
+from devtool import exec_build_env_command, setup_tinfoil, check_workspace_recipe, use_external_build, setup_git_repo, recipe_to_append, get_bbclassextend_targets, update_unlockedsigs, check_prerelease_version, DevtoolError
 from devtool import parse_recipe
 
 logger = logging.getLogger('devtool')

@@ -298,6 +298,8 @@ def add(args, config, basepath, workspace):
 
         _add_md5(config, recipename, appendfile)
 
+        check_prerelease_version(rd.getVar('PV'), 'devtool add')
+
         logger.info('Recipe %s has been automatically created; further editing may be required to make it fully functional' % recipefile)
 
     finally:

index 441dd35..ab7acd1 100644
@@ -33,7 +33,7 @@ sys.path = sys.path + [devtool_path]
 
 import oe.recipeutils
 from devtool import standard
-from devtool import exec_build_env_command, setup_tinfoil, DevtoolError, parse_recipe, use_external_build, update_unlockedsigs
+from devtool import exec_build_env_command, setup_tinfoil, DevtoolError, parse_recipe, use_external_build, update_unlockedsigs, check_prerelease_version
 
 logger = logging.getLogger('devtool')
 

@@ -420,8 +420,13 @@ def upgrade(args, config, basepath, workspace):
             old_srcrev = None
         if old_srcrev and not args.srcrev:
             raise DevtoolError("Recipe specifies a SRCREV value; you must specify a new one when upgrading")
-        if rd.getVar('PV') == args.version and old_srcrev == args.srcrev:
+        old_ver = rd.getVar('PV')
+        if old_ver == args.version and old_srcrev == args.srcrev:
             raise DevtoolError("Current and upgrade versions are the same version")
+        if args.version:
+            if bb.utils.vercmp_string(args.version, old_ver) < 0:
+                logger.warning('Upgrade version %s compares as less than the current version %s. If you are using a package feed for on-target upgrades or providing this recipe for general consumption, then you should increment PE in the recipe (or if there is no current PE value set, set it to "1")' % (args.version, old_ver))
+            check_prerelease_version(args.version, 'devtool upgrade')
 
         rf = None
         try:

index b745116..7cd139f 100644
@@ -32,6 +32,12 @@ logger = logging.getLogger('devtool')
 
 def _find_recipe_path(args, config, basepath, workspace):
     if args.any_recipe:
+        logger.warning('-a/--any-recipe option is now always active, and thus the option will be removed in a future release')
+    if args.recipename in workspace:
+        recipefile = workspace[args.recipename]['recipefile']
+    else:
+        recipefile = None
+    if not recipefile:
         tinfoil = setup_tinfoil(config_only=False, basepath=basepath)
         try:
             rd = parse_recipe(config, tinfoil, args.recipename, True)

@@ -40,12 +46,6 @@ def _find_recipe_path(args, config, basepath, workspace):
             recipefile = rd.getVar('FILE')
         finally:
             tinfoil.shutdown()
-    else:
-        check_workspace_recipe(workspace, args.recipename)
-        recipefile = workspace[args.recipename]['recipefile']
-        if not recipefile:
-            raise DevtoolError("Recipe file for %s is not under the workspace" %
-                               args.recipename)
     return recipefile
 
 

@@ -222,19 +222,21 @@ The ./configure %s output for %s follows.
 
 def register_commands(subparsers, context):
     """Register devtool subcommands from this plugin"""
-    parser_edit_recipe = subparsers.add_parser('edit-recipe', help='Edit a recipe file in your workspace',
-                                         description='Runs the default editor (as specified by the EDITOR variable) on the specified recipe. Note that the recipe file itself must be in the workspace (i.e. as a result of "devtool add" or "devtool upgrade"); you can override this with the -a/--any-recipe option.',
+    parser_edit_recipe = subparsers.add_parser('edit-recipe', help='Edit a recipe file',
+                                         description='Runs the default editor (as specified by the EDITOR variable) on the specified recipe. Note that this will be quicker for recipes in the workspace as the cache does not need to be loaded in that case.',
                                          group='working')
     parser_edit_recipe.add_argument('recipename', help='Recipe to edit')
-    parser_edit_recipe.add_argument('--any-recipe', '-a', action="store_true", help='Edit any recipe, not just where the recipe file itself is in the workspace')
+    # FIXME drop -a at some point in future
+    parser_edit_recipe.add_argument('--any-recipe', '-a', action="store_true", help='Does nothing (exists for backwards-compatibility)')
     parser_edit_recipe.set_defaults(func=edit_recipe)
 
     # Find-recipe
-    parser_find_recipe = subparsers.add_parser('find-recipe', help='Find a recipe file in your workspace',
-                                         description='By default, this will find a recipe file in your workspace; you can override this with the -a/--any-recipe option.',
+    parser_find_recipe = subparsers.add_parser('find-recipe', help='Find a recipe file',
+                                         description='Finds a recipe file. Note that this will be quicker for recipes in the workspace as the cache does not need to be loaded in that case.',
                                          group='working')
     parser_find_recipe.add_argument('recipename', help='Recipe to find')
-    parser_find_recipe.add_argument('--any-recipe', '-a', action="store_true", help='Find any recipe, not just where the recipe file itself is in the workspace')
+    # FIXME drop -a at some point in future
+    parser_find_recipe.add_argument('--any-recipe', '-a', action="store_true", help='Does nothing (exists for backwards-compatibility)')
     parser_find_recipe.set_defaults(func=find_recipe)
 
     # NOTE: Needed to override the usage string here since the default

index b6e532b..9bd2d86 100644
@@ -1620,16 +1620,16 @@ def _reset(recipes, no_clean, config, basepath, workspace):
         # We don't automatically create this dir next to appends, but the user can
         preservedir(os.path.join(config.workspace_path, 'appends', pn))
 
-        srctree = workspace[pn]['srctree']
-        if os.path.isdir(srctree):
-            if os.listdir(srctree):
+        srctreebase = workspace[pn]['srctreebase']
+        if os.path.isdir(srctreebase):
+            if os.listdir(srctreebase):
                 # We don't want to risk wiping out any work in progress
                 logger.info('Leaving source tree %s as-is; if you no '
                             'longer need it then please delete it manually'
-                            % srctree)
+                            % srctreebase)
             else:
                 # This is unlikely, but if it's empty we can just remove it
-                os.rmdir(srctree)
+                os.rmdir(srctreebase)
 
         clean_preferred_provider(pn, config.workspace_path)
 

index 20ab83f..07d774d 100644
@@ -350,3 +350,20 @@ def check_prerelease_version(ver, operation):
                        'If you prefer not to reset and re-try, you can change '
                        'the version after %s succeeds using "devtool rename" '
                        'with -V/--version.' % (ver, operation))
+
+def check_git_repo_dirty(repodir):
+    """Check if a git repository is clean or not"""
+    stdout, _ = bb.process.run('git status --porcelain', cwd=repodir)
+    return stdout
+
+def check_git_repo_op(srctree, ignoredirs=None):
+    """Check if a git repository is in the middle of a rebase"""
+    stdout, _ = bb.process.run('git rev-parse --show-toplevel', cwd=srctree)
+    topleveldir = stdout.strip()
+    if ignoredirs and topleveldir in ignoredirs:
+        return
+    gitdir = os.path.join(topleveldir, '.git')
+    if os.path.exists(os.path.join(gitdir, 'rebase-merge')):
+        raise DevtoolError("Source tree %s appears to be in the middle of a rebase - please resolve this first" % srctree)
+    if os.path.exists(os.path.join(gitdir, 'rebase-apply')):
+        raise DevtoolError("Source tree %s appears to be in the middle of 'git am' or 'git apply' - please resolve this first" % srctree)

index 9bd2d86..7fee304 100644
@@ -30,7 +30,7 @@ import errno
 import glob
 import filecmp
 from collections import OrderedDict
-from devtool import exec_build_env_command, setup_tinfoil, check_workspace_recipe, use_external_build, setup_git_repo, recipe_to_append, get_bbclassextend_targets, update_unlockedsigs, check_prerelease_version, DevtoolError
+from devtool import exec_build_env_command, setup_tinfoil, check_workspace_recipe, use_external_build, setup_git_repo, recipe_to_append, get_bbclassextend_targets, update_unlockedsigs, check_prerelease_version, check_git_repo_dirty, check_git_repo_op, DevtoolError
 from devtool import parse_recipe
 
 logger = logging.getLogger('devtool')

@@ -1675,6 +1675,18 @@ def finish(args, config, basepath, workspace):
 
     check_workspace_recipe(workspace, args.recipename)
 
+    # Grab the equivalent of COREBASE without having to initialise tinfoil
+    corebasedir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', '..'))
+
+    srctree = workspace[args.recipename]['srctree']
+    check_git_repo_op(srctree, [corebasedir])
+    dirty = check_git_repo_dirty(srctree)
+    if dirty:
+        if args.force:
+            logger.warning('Source tree is not clean, continuing as requested by -f/--force')
+        else:
+            raise DevtoolError('Source tree is not clean:\n\n%s\nEnsure you have committed your changes or use -f/--force if you are sure there\'s nothing that needs to be committed' % dirty)
+
     no_clean = False
     tinfoil = setup_tinfoil(basepath=basepath, tracking=True)
     try:

@@ -1867,10 +1879,11 @@ def register_commands(subparsers, context):
     parser_reset.set_defaults(func=reset)
 
     parser_finish = subparsers.add_parser('finish', help='Finish working on a recipe in your workspace',
-                                         description='Pushes any committed changes to the specified recipe to the specified layer and removes it from your workspace. Roughly equivalent to an update-recipe followed by reset, except the update-recipe step will do the "right thing" depending on the recipe and the destination layer specified.',
+                                         description='Pushes any committed changes to the specified recipe to the specified layer and removes it from your workspace. Roughly equivalent to an update-recipe followed by reset, except the update-recipe step will do the "right thing" depending on the recipe and the destination layer specified. Note that your changes must have been committed to the git repository in order to be recognised.',
                                          group='working', order=-100)
     parser_finish.add_argument('recipename', help='Recipe to finish')
     parser_finish.add_argument('destination', help='Layer/path to put recipe into. Can be the name of a layer configured in your bblayers.conf, the path to the base of a layer, or a partial path inside a layer. %(prog)s will attempt to complete the path based on the layer\'s structure.')
     parser_finish.add_argument('--mode', '-m', choices=['patch', 'srcrev', 'auto'], default='auto', help='Update mode (where %(metavar)s is %(choices)s; default is %(default)s)', metavar='MODE')
     parser_finish.add_argument('--initial-rev', help='Override starting revision for patches')
+    parser_finish.add_argument('--force', '-f', action="store_true", help='Force continuing even if there are uncommitted changes in the source tree repository')
     parser_finish.set_defaults(func=finish)

index 7fee304..166862f 100644
@@ -1714,6 +1714,8 @@ def finish(args, config, basepath, workspace):
                 elif line.startswith('# original_files:'):
                     origfilelist = line.split(':')[1].split()
 
+        destlayerbasedir = oe.recipeutils.find_layerdir(destlayerdir)
+
         if origlayerdir == config.workspace_path:
             # Recipe file itself is in workspace, update it there first
             appendlayerdir = None

@@ -1727,7 +1729,7 @@ def finish(args, config, basepath, workspace):
                 raise DevtoolError("Unable to determine destination layer path - check that %s specifies an actual layer and %s/conf/layer.conf specifies BBFILES. You may also need to specify a more complete path." % (args.destination, destlayerdir))
             # Warn if the layer isn't in bblayers.conf (the code to create a bbappend will do this in other cases)
             layerdirs = [os.path.abspath(layerdir) for layerdir in rd.getVar('BBLAYERS').split()]
-            if not os.path.abspath(destlayerdir) in layerdirs:
+            if not os.path.abspath(destlayerbasedir) in layerdirs:
                 bb.warn('Specified destination layer is not currently enabled in bblayers.conf, so the %s recipe will now be unavailable in your current configuration until you add the layer there' % args.recipename)
 
         elif destlayerdir == origlayerdir:

@@ -1740,7 +1742,7 @@ def finish(args, config, basepath, workspace):
             destpath = None
 
         # Remove any old files in the case of an upgrade
-        if origpath and origfilelist and oe.recipeutils.find_layerdir(origpath) == oe.recipeutils.find_layerdir(destlayerdir):
+        if origpath and origfilelist and oe.recipeutils.find_layerdir(origpath) == destlayerbasedir:
             for fn in origfilelist:
                 fnp = os.path.join(origpath, fn)
                 try:

index cab8e40..cab94b1 100644
@@ -22,7 +22,7 @@ from collections import OrderedDict, defaultdict
 # Help us to find places to insert values
 recipe_progression = ['SUMMARY', 'DESCRIPTION', 'HOMEPAGE', 'BUGTRACKER', 'SECTION', 'LICENSE', 'LICENSE_FLAGS', 'LIC_FILES_CHKSUM', 'PROVIDES', 'DEPENDS', 'PR', 'PV', 'SRCREV', 'SRCPV', 'SRC_URI', 'S', 'do_fetch()', 'do_unpack()', 'do_patch()', 'EXTRA_OECONF', 'EXTRA_OECMAKE', 'EXTRA_OESCONS', 'do_configure()', 'EXTRA_OEMAKE', 'do_compile()', 'do_install()', 'do_populate_sysroot()', 'INITSCRIPT', 'USERADD', 'GROUPADD', 'PACKAGES', 'FILES', 'RDEPENDS', 'RRECOMMENDS', 'RSUGGESTS', 'RPROVIDES', 'RREPLACES', 'RCONFLICTS', 'ALLOW_EMPTY', 'populate_packages()', 'do_package()', 'do_deploy()']
 # Variables that sometimes are a bit long but shouldn't be wrapped
-nowrap_vars = ['SUMMARY', 'HOMEPAGE', 'BUGTRACKER', 'SRC_URI[md5sum]', 'SRC_URI[sha256sum]']
+nowrap_vars = ['SUMMARY', 'HOMEPAGE', 'BUGTRACKER', 'SRC_URI\[(.+\.)?md5sum\]', 'SRC_URI\[(.+\.)?sha256sum\]']
 list_vars = ['SRC_URI', 'LIC_FILES_CHKSUM']
 meta_vars = ['SUMMARY', 'DESCRIPTION', 'HOMEPAGE', 'BUGTRACKER', 'SECTION']
 

@@ -142,6 +142,10 @@ def patch_recipe_lines(fromlines, values, trailing_newline=True):
     else:
         newline = ''
 
+    nowrap_vars_res = []
+    for item in nowrap_vars:
+        nowrap_vars_res.append(re.compile('^%s$' % item))
+
     recipe_progression_res = []
     recipe_progression_restrs = []
     for item in recipe_progression:

@@ -174,7 +178,12 @@ def patch_recipe_lines(fromlines, values, trailing_newline=True):
             return
         rawtext = '%s = "%s"%s' % (name, values[name], newline)
         addlines = []
-        if name in nowrap_vars:
+        nowrap = False
+        for nowrap_re in nowrap_vars_res:
+            if nowrap_re.match(name):
+                nowrap = True
+                break
+        if nowrap:
             addlines.append(rawtext)
         elif name in list_vars:
             splitvalue = split_var_value(values[name], assignment=False)

66 lines not on this page · raw message

index ae53972..fb57e70 100644
@@ -73,7 +73,6 @@ class NpmRecipeHandler(RecipeHandler):
                     license = license.replace(' ', '_')
                     if not license[0] == '(':
                         license = '(' + license + ')'
-                    print('LICENSE: {}'.format(license))
                 else:
                     license = license.replace('AND', '&')
                     if license[0] == '(':

index 87bb5c8..a651d8f 100755
@@ -189,7 +189,11 @@ def _enable_workspace_layer(workspacedir, config, basepath):
     if not os.path.exists(bblayers_conf):
         logger.error('Unable to find bblayers.conf')
         return
-    _, added = bb.utils.edit_bblayers_conf(bblayers_conf, workspacedir, config.workspace_path)
+    if os.path.abspath(workspacedir) != os.path.abspath(config.workspace_path):
+        removedir = config.workspace_path
+    else:
+        removedir = None
+    _, added = bb.utils.edit_bblayers_conf(bblayers_conf, workspacedir, removedir)
     if added:
         logger.info('Enabling workspace layer in bblayers.conf')
     if config.workspace_path != workspacedir:

index 055bdef..54e7e85 100644
@@ -433,6 +433,9 @@ def create_recipe(args):
         source = 'file://%s' % os.path.abspath(source)
 
     if scriptutils.is_src_url(source):
+        # Warn about github archive URLs
+        if re.match('https?://github.com/[^/]+/[^/]+/archive/.+(\.tar\..*|\.zip)$', source):
+            logger.warn('github archive files are not guaranteed to be stable and may be re-generated over time. If the latter occurs, the checksums will likely change and the recipe will fail at do_fetch. It is recommended that you point to an actual commit or tag in the repository instead (using the repository URL in conjunction with the -S/--srcrev option).')
         # Fetch a URL
         fetchuri = reformat_git_uri(urldefrag(source)[0])
         if args.binary:

index 6d51958..e7d47b9 100644
@@ -457,7 +457,9 @@ def upgrade(args, config, basepath, workspace):
 
         rf = None
         try:
+            logger.info('Extracting current version source...')
             rev1 = standard._extract_source(srctree, False, 'devtool-orig', False, config, basepath, workspace, args.fixed_setup, rd, tinfoil)
+            logger.info('Extracting upgraded version source...')
             rev2, md5, sha256, srcbranch = _extract_new_source(args.version, srctree, args.no_patch,
                                                     args.srcrev, args.srcbranch, args.branch, args.keep_temp,
                                                     tinfoil, rd)

index 166862f..5ac678b 100644
@@ -409,7 +409,7 @@ def extract(args, config, basepath, workspace):
             return 1
 
         srctree = os.path.abspath(args.srctree)
-        initial_rev = _extract_source(srctree, args.keep_temp, args.branch, False, config, basepath, workspace, args.fixed_setup, rd, tinfoil)
+        initial_rev, _ = _extract_source(srctree, args.keep_temp, args.branch, False, config, basepath, workspace, args.fixed_setup, rd, tinfoil)
         logger.info('Source tree extracted to %s' % srctree)
 
         if initial_rev:

@@ -433,7 +433,7 @@ def sync(args, config, basepath, workspace):
             return 1
 
         srctree = os.path.abspath(args.srctree)
-        initial_rev = _extract_source(srctree, args.keep_temp, args.branch, True, config, basepath, workspace, args.fixed_setup, rd, tinfoil)
+        initial_rev, _ = _extract_source(srctree, args.keep_temp, args.branch, True, config, basepath, workspace, args.fixed_setup, rd, tinfoil)
         logger.info('Source tree %s synchronized' % srctree)
 
         if initial_rev:

@@ -549,6 +549,7 @@ def _extract_source(srctree, keep_temp, devbranch, sync, config, basepath, works
 
         with open(os.path.join(tempdir, 'srcsubdir'), 'r') as f:
             srcsubdir = f.read()
+        srcsubdir_rel = os.path.relpath(srcsubdir, os.path.join(tempdir, 'workdir'))
 
         tempdir_localdir = os.path.join(tempdir, 'oe-local-files')
         srctree_localdir = os.path.join(srctree, 'oe-local-files')

@@ -615,7 +616,7 @@ def _extract_source(srctree, keep_temp, devbranch, sync, config, basepath, works
             logger.info('Preserving temporary directory %s' % tempdir)
         else:
             shutil.rmtree(tempdir)
-    return initial_rev
+    return initial_rev, srcsubdir_rel
 
 def _add_md5(config, recipename, filename):
     """Record checksum of a file (or recursively for a directory) to the md5-file of the workspace"""

@@ -713,7 +714,7 @@ def modify(args, config, basepath, workspace):
         initial_rev = None
         commits = []
         if not args.no_extract:
-            initial_rev = _extract_source(srctree, args.keep_temp, args.branch, False, config, basepath, workspace, args.fixed_setup, rd, tinfoil)
+            initial_rev, _ = _extract_source(srctree, args.keep_temp, args.branch, False, config, basepath, workspace, args.fixed_setup, rd, tinfoil)
             if not initial_rev:
                 return 1
             logger.info('Source tree extracted to %s' % srctree)

index e7d47b9..0db2a50 100644
@@ -191,6 +191,8 @@ def _extract_new_source(newpv, srctree, no_patch, srcrev, srcbranch, branch, kee
         __run('git tag -f devtool-base-new')
         md5 = None
         sha256 = None
+        _, _, _, _, _, params = bb.fetch2.decodeurl(uri)
+        srcsubdir_rel = params.get('destsuffix', 'git')
         if not srcbranch:
             check_branch, check_branch_err = __run('git branch -r --contains %s' % srcrev)
             get_branch = [x.strip() for x in check_branch.splitlines()]

@@ -225,6 +227,7 @@ def _extract_new_source(newpv, srctree, no_patch, srcrev, srcbranch, branch, kee
 
         tmpsrctree = _get_srctree(tmpdir)
         srctree = os.path.abspath(srctree)
+        srcsubdir_rel = os.path.relpath(tmpsrctree, tmpdir)
 
         # Delete all sources so we ensure no stray files are left over
         for item in os.listdir(srctree):

@@ -288,9 +291,9 @@ def _extract_new_source(newpv, srctree, no_patch, srcrev, srcbranch, branch, kee
         else:
             shutil.rmtree(tmpsrctree)
 
-    return (rev, md5, sha256, srcbranch)
+    return (rev, md5, sha256, srcbranch, srcsubdir_rel)
 
-def _create_new_recipe(newpv, md5, sha256, srcrev, srcbranch, workspace, tinfoil, rd):
+def _create_new_recipe(newpv, md5, sha256, srcrev, srcbranch, srcsubdir_old, srcsubdir_new, workspace, tinfoil, rd):
     """Creates the new recipe under workspace"""
 
     bpn = rd.getVar('BPN')

@@ -384,6 +387,21 @@ def _create_new_recipe(newpv, md5, sha256, srcrev, srcbranch, workspace, tinfoil
         newvalues['SRC_URI[%smd5sum]' % nameprefix] = md5
         newvalues['SRC_URI[%ssha256sum]' % nameprefix] = sha256
 
+    if srcsubdir_new != srcsubdir_old:
+        s_subdir_old = os.path.relpath(os.path.abspath(rd.getVar('S')), rd.getVar('WORKDIR'))
+        s_subdir_new = os.path.relpath(os.path.abspath(crd.getVar('S')), crd.getVar('WORKDIR'))
+        if srcsubdir_old == s_subdir_old and srcsubdir_new != s_subdir_new:
+            # Subdir for old extracted source matches what S points to (it should!)
+            # but subdir for new extracted source doesn't match what S will be
+            newvalues['S'] = '${WORKDIR}/%s' % srcsubdir_new.replace(newpv, '${PV}')
+            if crd.expand(newvalues['S']) == crd.expand('${WORKDIR}/${BP}'):
+                # It's the default, drop it
+                # FIXME what if S is being set in a .inc?
+                newvalues['S'] = None
+                logger.info('Source subdirectory has changed, dropping S value since it now matches the default ("${WORKDIR}/${BP}")')
+            else:
+                logger.info('Source subdirectory has changed, updating S value')
+
     rd = tinfoil.parse_recipe_file(fullpath, False)
     oe.recipeutils.patch_recipe(rd, fullpath, newvalues)
 

@@ -458,12 +476,12 @@ def upgrade(args, config, basepath, workspace):
         rf = None
         try:
             logger.info('Extracting current version source...')
-            rev1 = standard._extract_source(srctree, False, 'devtool-orig', False, config, basepath, workspace, args.fixed_setup, rd, tinfoil)
+            rev1, srcsubdir1 = standard._extract_source(srctree, False, 'devtool-orig', False, config, basepath, workspace, args.fixed_setup, rd, tinfoil)
             logger.info('Extracting upgraded version source...')
-            rev2, md5, sha256, srcbranch = _extract_new_source(args.version, srctree, args.no_patch,
+            rev2, md5, sha256, srcbranch, srcsubdir2 = _extract_new_source(args.version, srctree, args.no_patch,
                                                     args.srcrev, args.srcbranch, args.branch, args.keep_temp,
                                                     tinfoil, rd)
-            rf, copied = _create_new_recipe(args.version, md5, sha256, args.srcrev, srcbranch, config.workspace_path, tinfoil, rd)
+            rf, copied = _create_new_recipe(args.version, md5, sha256, args.srcrev, srcbranch, srcsubdir1, srcsubdir2, config.workspace_path, tinfoil, rd)
         except bb.process.CmdError as e:
             _upgrade_error(e, rf, srctree)
         except DevtoolError as e:

index 0db2a50..3cb523c 100644
@@ -263,10 +263,8 @@ def _extract_new_source(newpv, srctree, no_patch, srcrev, srcbranch, branch, kee
 
     if no_patch:
         patches = oe.recipeutils.get_recipe_patches(crd)
-        if len(patches):
-            logger.warn('By user choice, the following patches will NOT be applied')
-            for patch in patches:
-                logger.warn("%s" % os.path.basename(patch))
+        if patches:
+            logger.warn('By user choice, the following patches will NOT be applied to the new source tree:\n  %s' % '\n  '.join([os.path.basename(patch) for patch in patches]))
     else:
         __run('git checkout devtool-patched -b %s' % branch)
         skiptag = False

index 5ac678b..1e7d707 100644
@@ -544,11 +544,14 @@ def _extract_source(srctree, keep_temp, devbranch, sync, config, basepath, works
         if not res:
             raise DevtoolError('Extracting source for %s failed' % pn)
 
-        with open(os.path.join(tempdir, 'initial_rev'), 'r') as f:
-            initial_rev = f.read()
+        try:
+            with open(os.path.join(tempdir, 'initial_rev'), 'r') as f:
+                initial_rev = f.read()
 
-        with open(os.path.join(tempdir, 'srcsubdir'), 'r') as f:
-            srcsubdir = f.read()
+            with open(os.path.join(tempdir, 'srcsubdir'), 'r') as f:
+                srcsubdir = f.read()
+        except FileNotFoundError as e:
+            raise DevtoolError('Something went wrong with source extraction - the devtool-source class was not active or did not function correctly:\n%s' % str(e))
         srcsubdir_rel = os.path.relpath(srcsubdir, os.path.join(tempdir, 'workdir'))
 
         tempdir_localdir = os.path.join(tempdir, 'oe-local-files')

index 1e7d707..9d9031c 100644
@@ -1054,7 +1054,7 @@ def _replace_srcuri_entry(srcuri, filename, newentry):
             srcuri.insert(i, newentry)
             break
 
-def _remove_source_files(append, files, destpath):
+def _remove_source_files(append, files, destpath, no_report_remove=False):
     """Unlink existing patch files"""
     for path in files:
         if append:

@@ -1063,7 +1063,8 @@ def _remove_source_files(append, files, destpath):
             path = os.path.join(destpath, os.path.basename(path))
 
         if os.path.exists(path):
-            logger.info('Removing file %s' % path)
+            if not no_report_remove:
+                logger.info('Removing file %s' % path)
             # FIXME "git rm" here would be nice if the file in question is
             #       tracked
             # FIXME there's a chance that this file is referred to by

@@ -1276,7 +1277,7 @@ def _determine_files_dir(rd):
     return os.path.join(recipedir, rd.getVar('BPN'))
 
 
-def _update_recipe_srcrev(recipename, workspace, srctree, rd, appendlayerdir, wildcard_version, no_remove):
+def _update_recipe_srcrev(recipename, workspace, srctree, rd, appendlayerdir, wildcard_version, no_remove, no_report_remove):
     """Implement the 'srcrev' mode of update-recipe"""
     import bb
     import oe.recipeutils

@@ -1357,10 +1358,10 @@ def _update_recipe_srcrev(recipename, workspace, srctree, rd, appendlayerdir, wi
                     'point to a git repository where you have pushed your '
                     'changes')
 
-    _remove_source_files(appendlayerdir, remove_files, destpath)
+    _remove_source_files(appendlayerdir, remove_files, destpath, no_report_remove)
     return True
 
-def _update_recipe_patch(recipename, workspace, srctree, rd, appendlayerdir, wildcard_version, no_remove, initial_rev):
+def _update_recipe_patch(recipename, workspace, srctree, rd, appendlayerdir, wildcard_version, no_remove, no_report_remove, initial_rev):
     """Implement the 'patch' mode of update-recipe"""
     import bb
     import oe.recipeutils

@@ -1476,7 +1477,7 @@ def _update_recipe_patch(recipename, workspace, srctree, rd, appendlayerdir, wil
     finally:
         shutil.rmtree(tempdir)
 
-    _remove_source_files(appendlayerdir, remove_files, destpath)
+    _remove_source_files(appendlayerdir, remove_files, destpath, no_report_remove)
     return True
 
 def _guess_recipe_update_mode(srctree, rdata):

@@ -1501,15 +1502,15 @@ def _guess_recipe_update_mode(srctree, rdata):
 
     return 'patch'
 
-def _update_recipe(recipename, workspace, rd, mode, appendlayerdir, wildcard_version, no_remove, initial_rev):
+def _update_recipe(recipename, workspace, rd, mode, appendlayerdir, wildcard_version, no_remove, initial_rev, no_report_remove=False):
     srctree = workspace[recipename]['srctree']
     if mode == 'auto':
         mode = _guess_recipe_update_mode(srctree, rd)
 
     if mode == 'srcrev':
-        updated = _update_recipe_srcrev(recipename, workspace, srctree, rd, appendlayerdir, wildcard_version, no_remove)
+        updated = _update_recipe_srcrev(recipename, workspace, srctree, rd, appendlayerdir, wildcard_version, no_remove, no_report_remove)
     elif mode == 'patch':
-        updated = _update_recipe_patch(recipename, workspace, srctree, rd, appendlayerdir, wildcard_version, no_remove, initial_rev)
+        updated = _update_recipe_patch(recipename, workspace, srctree, rd, appendlayerdir, wildcard_version, no_remove, no_report_remove, initial_rev)
     else:
         raise DevtoolError('update_recipe: invalid mode %s' % mode)
     return updated

@@ -1745,24 +1746,27 @@ def finish(args, config, basepath, workspace):
             appendlayerdir = destlayerdir
             destpath = None
 
+        # Actually update the recipe / bbappend
+        removing_original = (origpath and origfilelist and oe.recipeutils.find_layerdir(origpath) == destlayerbasedir)
+        _update_recipe(args.recipename, workspace, rd, args.mode, appendlayerdir, wildcard_version=True, no_remove=False, initial_rev=args.initial_rev, no_report_remove=removing_original)
+
         # Remove any old files in the case of an upgrade
-        if origpath and origfilelist and oe.recipeutils.find_layerdir(origpath) == destlayerbasedir:
+        recipedir = os.path.dirname(rd.getVar('FILE'))
+        if removing_original:
             for fn in origfilelist:
                 fnp = os.path.join(origpath, fn)
+                if not os.path.exists(os.path.join(recipedir, fn)):
+                    logger.info('Removing file %s' % fnp)
                 try:
                     os.remove(fnp)
                 except FileNotFoundError:
                     pass
 
-        # Actually update the recipe / bbappend
-        _update_recipe(args.recipename, workspace, rd, args.mode, appendlayerdir, wildcard_version=True, no_remove=False, initial_rev=args.initial_rev)
-
         if origlayerdir == config.workspace_path and destpath:
             # Recipe file itself is in the workspace - need to move it and any
             # associated files to the specified layer
             no_clean = True
             logger.info('Moving recipe file to %s' % destpath)
-            recipedir = os.path.dirname(rd.getVar('FILE'))
             for root, _, files in os.walk(recipedir):
                 for fn in files:
                     srcpath = os.path.join(root, fn)

index cab94b1..a1e191a 100644
@@ -251,7 +251,7 @@ def patch_recipe_lines(fromlines, values, trailing_newline=True):
     return changed, tolines
 
 
-def patch_recipe_file(fn, values, patch=False, relpath=''):
+def patch_recipe_file(fn, values, patch=False, relpath='', redirect_output=None):
     """Update or insert variable values into a recipe file (assuming you
        have already identified the exact file you want to update.)
        Note that some manual inspection/intervention may be required

@@ -263,7 +263,11 @@ def patch_recipe_file(fn, values, patch=False, relpath=''):
 
     _, tolines = patch_recipe_lines(fromlines, values)
 
-    if patch:
+    if redirect_output:
+        with open(os.path.join(redirect_output, os.path.basename(fn)), 'w') as f:
+            f.writelines(tolines)
+        return None
+    elif patch:
         relfn = os.path.relpath(fn, relpath)
         diff = difflib.unified_diff(fromlines, tolines, 'a/%s' % relfn, 'b/%s' % relfn)
         return diff

@@ -313,7 +317,7 @@ def localise_file_vars(fn, varfiles, varlist):
 
     return filevars
 
-def patch_recipe(d, fn, varvalues, patch=False, relpath=''):
+def patch_recipe(d, fn, varvalues, patch=False, relpath='', redirect_output=None):
     """Modify a list of variable values in the specified recipe. Handles inc files if
     used by the recipe.
     """

@@ -323,7 +327,7 @@ def patch_recipe(d, fn, varvalues, patch=False, relpath=''):
     patches = []
     for f,v in locs.items():
         vals = {k: varvalues[k] for k in v}
-        patchdata = patch_recipe_file(f, vals, patch, relpath)
+        patchdata = patch_recipe_file(f, vals, patch, relpath, redirect_output)
         if patch:
             patches.append(patchdata)
 

@@ -584,7 +588,7 @@ def get_bbappend_path(d, destlayerdir, wildcardver=False):
     return (appendpath, pathok)
 
 
-def bbappend_recipe(rd, destlayerdir, srcfiles, install=None, wildcardver=False, machine=None, extralines=None, removevalues=None):
+def bbappend_recipe(rd, destlayerdir, srcfiles, install=None, wildcardver=False, machine=None, extralines=None, removevalues=None, redirect_output=None):
     """
     Writes a bbappend file for a recipe
     Parameters:

@@ -611,6 +615,9 @@ def bbappend_recipe(rd, destlayerdir, srcfiles, install=None, wildcardver=False,
             value pairs, or simply a list of the lines.
         removevalues:
             Variable values to remove - a dict of names/values.
+        redirect_output:
+            If specified, redirects writing the output file to the
+            specified directory (for dry-run purposes)
     """
 
     if not removevalues:

@@ -625,7 +632,8 @@ def bbappend_recipe(rd, destlayerdir, srcfiles, install=None, wildcardver=False,
         bb.warn('Unable to determine correct subdirectory path for bbappend file - check that what %s adds to BBFILES also matches .bbappend files. Using %s for now, but until you fix this the bbappend will not be applied.' % (os.path.join(destlayerdir, 'conf', 'layer.conf'), os.path.dirname(appendpath)))
 
     appenddir = os.path.dirname(appendpath)
-    bb.utils.mkdirhier(appenddir)
+    if not redirect_output:
+        bb.utils.mkdirhier(appenddir)
 
     # FIXME check if the bbappend doesn't get overridden by a higher priority layer?
 

@@ -702,9 +710,18 @@ def bbappend_recipe(rd, destlayerdir, srcfiles, install=None, wildcardver=False,
         if instfunclines:
             bbappendlines.append(('do_install_append%s()' % appendoverride, '', instfunclines))
 
-    bb.note('Writing append file %s' % appendpath)
+    if redirect_output:
+        bb.note('Writing append file %s (dry-run)' % appendpath)
+        outfile = os.path.join(redirect_output, os.path.basename(appendpath))
+        # Only take a copy if the file isn't already there (this function may be called
+        # multiple times per operation when we're handling overrides)
+        if os.path.exists(appendpath) and not os.path.exists(outfile):
+            shutil.copy2(appendpath, outfile)
+    else:
+        bb.note('Writing append file %s' % appendpath)
+        outfile = appendpath
 
-    if os.path.exists(appendpath):
+    if os.path.exists(outfile):
         # Work around lack of nonlocal in python 2
         extvars = {'destsubdir': destsubdir}
 

@@ -776,7 +793,7 @@ def bbappend_recipe(rd, destlayerdir, srcfiles, install=None, wildcardver=False,
         if removevalues:
             varnames.extend(list(removevalues.keys()))
 
-        with open(appendpath, 'r') as f:
+        with open(outfile, 'r') as f:
             (updated, newlines) = bb.utils.edit_metadata(f, varnames, appendfile_varfunc)
 
         destsubdir = extvars['destsubdir']

@@ -793,20 +810,27 @@ def bbappend_recipe(rd, destlayerdir, srcfiles, install=None, wildcardver=False,
         updated = True
 
     if updated:
-        with open(appendpath, 'w') as f:
+        with open(outfile, 'w') as f:
             f.writelines(newlines)
 
     if copyfiles:
         if machine:
             destsubdir = os.path.join(destsubdir, machine)
+        if redirect_output:
+            outdir = redirect_output
+        else:
+            outdir = appenddir
         for newfile, srcfile in copyfiles.items():
-            filedest = os.path.join(appenddir, destsubdir, os.path.basename(srcfile))
+            filedest = os.path.join(outdir, destsubdir, os.path.basename(srcfile))
             if os.path.abspath(newfile) != os.path.abspath(filedest):
                 if newfile.startswith(tempfile.gettempdir()):
                     newfiledisp = os.path.basename(newfile)
                 else:
                     newfiledisp = newfile
-                bb.note('Copying %s to %s' % (newfiledisp, filedest))
+                if redirect_output:
+                    bb.note('Copying %s to %s (dry-run)' % (newfiledisp, os.path.join(appenddir, destsubdir, os.path.basename(srcfile))))
+                else:
+                    bb.note('Copying %s to %s' % (newfiledisp, filedest))
                 bb.utils.mkdirhier(os.path.dirname(filedest))
                 shutil.copyfile(newfile, filedest)
 

462 lines not on this page · raw message

index 8f5bc86..56882a4 100644
@@ -152,9 +152,65 @@ python devtool_pre_patch() {
 }
 
 python devtool_post_patch() {
+    import shutil
     tempdir = d.getVar('DEVTOOL_TEMPDIR')
     with open(os.path.join(tempdir, 'srcsubdir'), 'r') as f:
         srcsubdir = f.read()
+    with open(os.path.join(tempdir, 'initial_rev'), 'r') as f:
+        initial_rev = f.read()
+
+    def rm_patches():
+        patches_dir = os.path.join(srcsubdir, 'patches')
+        if os.path.exists(patches_dir):
+            shutil.rmtree(patches_dir)
+        # Restore any "patches" directory that was actually part of the source tree
+        try:
+            bb.process.run('git checkout -- patches', cwd=srcsubdir)
+        except bb.process.ExecutionError:
+            pass
+
+    extra_overrides = d.getVar('DEVTOOL_EXTRA_OVERRIDES')
+    if extra_overrides:
+        extra_override_list = extra_overrides.split(':')
+        devbranch = d.getVar('DEVTOOL_DEVBRANCH')
+        default_overrides = d.getVar('OVERRIDES').split(':')
+        no_overrides = []
+        # First, we may have some overrides that are referred to in the recipe set in
+        # our configuration, so we need to make a branch that excludes those
+        for override in default_overrides:
+            if override not in extra_override_list:
+                no_overrides.append(override)
+        if default_overrides != no_overrides:
+            # Some overrides are active in the current configuration, so
+            # we need to create a branch where none of the overrides are active
+            bb.process.run('git checkout %s -b devtool-no-overrides' % initial_rev, cwd=srcsubdir)
+            # Run do_patch function with the override applied
+            localdata = bb.data.createCopy(d)
+            localdata.setVar('OVERRIDES', ':'.join(no_overrides))
+            bb.build.exec_func('do_patch', localdata)
+            rm_patches()
+            # Now we need to reconcile the dev branch with the no-overrides one
+            # (otherwise we'd likely be left with identical commits that have different hashes)
+            bb.process.run('git checkout %s' % devbranch, cwd=srcsubdir)
+            bb.process.run('git rebase devtool-no-overrides', cwd=srcsubdir)
+        else:
+            bb.process.run('git checkout %s -b devtool-no-overrides' % devbranch, cwd=srcsubdir)
+
+        for override in extra_override_list:
+            localdata = bb.data.createCopy(d)
+            if override in default_overrides:
+                bb.process.run('git branch devtool-override-%s %s' % (override, devbranch), cwd=srcsubdir)
+            else:
+                # Reset back to the initial commit on a new branch
+                bb.process.run('git checkout %s -b devtool-override-%s' % (initial_rev, override), cwd=srcsubdir)
+                # Run do_patch function with the override applied
+                localdata.appendVar('OVERRIDES', ':%s' % override)
+                bb.build.exec_func('do_patch', localdata)
+                rm_patches()
+                # Now we need to reconcile the new branch with the no-overrides one
+                # (otherwise we'd likely be left with identical commits that have different hashes)
+                bb.process.run('git rebase devtool-no-overrides', cwd=srcsubdir)
+        bb.process.run('git checkout %s' % devbranch, cwd=srcsubdir)
     bb.process.run('git tag -f devtool-patched', cwd=srcsubdir)
 }
 

359 lines not on this page · raw message

index 3cb523c..f6141bf 100644
@@ -474,7 +474,7 @@ def upgrade(args, config, basepath, workspace):
         rf = None
         try:
             logger.info('Extracting current version source...')
-            rev1, srcsubdir1 = standard._extract_source(srctree, False, 'devtool-orig', False, config, basepath, workspace, args.fixed_setup, rd, tinfoil)
+            rev1, srcsubdir1 = standard._extract_source(srctree, False, 'devtool-orig', False, config, basepath, workspace, args.fixed_setup, rd, tinfoil, no_overrides=args.no_overrides)
             logger.info('Extracting upgraded version source...')
             rev2, md5, sha256, srcbranch, srcsubdir2 = _extract_new_source(args.version, srctree, args.no_patch,
                                                     args.srcrev, args.srcbranch, args.branch, args.keep_temp,

@@ -513,6 +513,7 @@ def register_commands(subparsers, context):
     parser_upgrade.add_argument('--srcbranch', '-B', help='Branch in source repository containing the revision to use (if fetching from an SCM such as git)')
     parser_upgrade.add_argument('--branch', '-b', default="devtool", help='Name for new development branch to checkout (default "%(default)s")')
     parser_upgrade.add_argument('--no-patch', action="store_true", help='Do not apply patches from the recipe to the new source code')
+    parser_upgrade.add_argument('--no-overrides', '-O', action="store_true", help='Do not create branches for other override configurations')
     group = parser_upgrade.add_mutually_exclusive_group()
     group.add_argument('--same-dir', '-s', help='Build in same directory as source', action="store_true")
     group.add_argument('--no-same-dir', help='Force build in a separate build directory', action="store_true")