A couple of devtool fixes

new file mode 100644
index 0000000..8f5bc86
@@ -0,0 +1,165 @@
+# Development tool - source extraction helper class
+#
+# NOTE: this class is intended for use by devtool and should not be
+# inherited manually.
+#
+# Copyright (C) 2014-2017 Intel Corporation
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+
+DEVTOOL_TEMPDIR ?= ""
+DEVTOOL_PATCH_SRCDIR = "${DEVTOOL_TEMPDIR}/patchworkdir"
+
+
+python() {
+    tempdir = d.getVar('DEVTOOL_TEMPDIR')
+
+    if not tempdir:
+        bb.fatal('devtool-source class is for internal use by devtool only')
+
+    # Make a subdir so we guard against WORKDIR==S
+    workdir = os.path.join(tempdir, 'workdir')
+    d.setVar('WORKDIR', workdir)
+    if not d.getVar('S').startswith(workdir):
+        # Usually a shared workdir recipe (kernel, gcc)
+        # Try to set a reasonable default
+        if bb.data.inherits_class('kernel', d):
+            d.setVar('S', '${WORKDIR}/source')
+        else:
+            d.setVar('S', '${WORKDIR}/%s' % os.path.basename(d.getVar('S')))
+    if bb.data.inherits_class('kernel', d):
+        # We don't want to move the source to STAGING_KERNEL_DIR here
+        d.setVar('STAGING_KERNEL_DIR', '${S}')
+
+    d.setVar('STAMPS_DIR', os.path.join(tempdir, 'stamps'))
+    d.setVar('T', os.path.join(tempdir, 'temp'))
+
+    # Hook in pre/postfuncs
+    is_kernel_yocto = bb.data.inherits_class('kernel-yocto', d)
+    if is_kernel_yocto:
+        unpacktask = 'do_kernel_checkout'
+        d.appendVarFlag('do_configure', 'postfuncs', ' devtool_post_configure')
+    else:
+        unpacktask = 'do_unpack'
+    d.appendVarFlag(unpacktask, 'postfuncs', ' devtool_post_unpack')
+    d.prependVarFlag('do_patch', 'prefuncs', ' devtool_pre_patch')
+    d.appendVarFlag('do_patch', 'postfuncs', ' devtool_post_patch')
+
+    # NOTE: in order for the patch stuff to be fully functional,
+    # PATCHTOOL and PATCH_COMMIT_FUNCTIONS need to be set; we can't
+    # do that here because we can't guarantee the order of the anonymous
+    # functions, so it gets done in the bbappend we create.
+}
+
+
+python devtool_post_unpack() {
+    import oe.recipeutils
+    import shutil
+    sys.path.insert(0, os.path.join(d.getVar('COREBASE'), 'scripts', 'lib'))
+    import scriptutils
+    from devtool import setup_git_repo
+
+    tempdir = d.getVar('DEVTOOL_TEMPDIR')
+    workdir = d.getVar('WORKDIR')
+    srcsubdir = d.getVar('S')
+
+    def _move_file(src, dst):
+        """Move a file. Creates all the directory components of destination path."""
+        dst_d = os.path.dirname(dst)
+        if dst_d:
+            bb.utils.mkdirhier(dst_d)
+        shutil.move(src, dst)
+
+    def _ls_tree(directory):
+        """Recursive listing of files in a directory"""
+        ret = []
+        for root, dirs, files in os.walk(directory):
+            ret.extend([os.path.relpath(os.path.join(root, fname), directory) for
+                        fname in files])
+        return ret
+
+    # Move local source files into separate subdir
+    recipe_patches = [os.path.basename(patch) for patch in
+                        oe.recipeutils.get_recipe_patches(d)]
+    local_files = oe.recipeutils.get_recipe_local_files(d)
+
+    # Ignore local files with subdir={BP}
+    srcabspath = os.path.abspath(srcsubdir)
+    local_files = [fname for fname in local_files if
+                    os.path.exists(os.path.join(workdir, fname)) and
+                    (srcabspath == workdir or not
+                    os.path.join(workdir, fname).startswith(srcabspath +
+                        os.sep))]
+    if local_files:
+        for fname in local_files:
+            _move_file(os.path.join(workdir, fname),
+                        os.path.join(tempdir, 'oe-local-files', fname))
+        with open(os.path.join(tempdir, 'oe-local-files', '.gitignore'),
+                    'w') as f:
+            f.write('# Ignore local files, by default. Remove this file '
+                    'if you want to commit the directory to Git\n*\n')
+
+    if srcsubdir == workdir:
+        # Find non-patch non-local sources that were "unpacked" to srctree
+        # directory
+        src_files = [fname for fname in _ls_tree(workdir) if
+                        os.path.basename(fname) not in recipe_patches]
+        srcsubdir = d.getVar('DEVTOOL_PATCH_SRCDIR')
+        # Move source files to S
+        for path in src_files:
+            _move_file(os.path.join(workdir, path),
+                        os.path.join(srcsubdir, path))
+    elif os.path.dirname(srcsubdir) != workdir:
+        # Handle if S is set to a subdirectory of the source
+        srcsubdir = os.path.join(workdir, os.path.relpath(srcsubdir, workdir).split(os.sep)[0])
+
+    scriptutils.git_convert_standalone_clone(srcsubdir)
+
+    # Make sure that srcsubdir exists
+    bb.utils.mkdirhier(srcsubdir)
+    if not os.listdir(srcsubdir):
+        bb.warn("No source unpacked to S - either the %s recipe "
+                "doesn't use any source or the correct source "
+                "directory could not be determined" % d.getVar('PN'))
+
+    devbranch = d.getVar('DEVTOOL_DEVBRANCH')
+    setup_git_repo(srcsubdir, d.getVar('PV'), devbranch, d=d)
+
+    (stdout, _) = bb.process.run('git rev-parse HEAD', cwd=srcsubdir)
+    initial_rev = stdout.rstrip()
+    with open(os.path.join(tempdir, 'initial_rev'), 'w') as f:
+        f.write(initial_rev)
+
+    with open(os.path.join(tempdir, 'srcsubdir'), 'w') as f:
+        f.write(srcsubdir)
+}
+
+python devtool_pre_patch() {
+    if d.getVar('S') == d.getVar('WORKDIR'):
+        d.setVar('S', '${DEVTOOL_PATCH_SRCDIR}')
+}
+
+python devtool_post_patch() {
+    tempdir = d.getVar('DEVTOOL_TEMPDIR')
+    with open(os.path.join(tempdir, 'srcsubdir'), 'r') as f:
+        srcsubdir = f.read()
+    bb.process.run('git tag -f devtool-patched', cwd=srcsubdir)
+}
+
+python devtool_post_configure() {
+    import shutil
+    tempdir = d.getVar('DEVTOOL_TEMPDIR')
+    shutil.copy2(os.path.join(d.getVar('B'), '.config'), tempdir)
+}

index 6af0d38..2a54993 100644
@@ -1015,6 +1015,11 @@ python sstate_eventhandler2() {
     d = e.data
     stamps = e.stamps.values()
     removeworkdir = (d.getVar("SSTATE_PRUNE_OBSOLETEWORKDIR", False) == "1")
+    preservestampfile = d.expand('${SSTATE_MANIFESTS}/preserve-stamps')
+    preservestamps = []
+    if os.path.exists(preservestampfile):
+        with open(preservestampfile, 'r') as f:
+            preservestamps = f.readlines()
     seen = []
     for a in d.getVar("SSTATE_ARCHS").split():
         toremove = []

@@ -1025,7 +1030,7 @@ python sstate_eventhandler2() {
             lines = f.readlines()
             for l in lines:
                 (stamp, manifest, workdir) = l.split()
-                if stamp not in stamps:
+                if stamp not in stamps and stamp not in preservestamps:
                     toremove.append(l)
                     if stamp not in seen:
                         bb.debug(2, "Stamp %s is not reachable, removing related manifests" % stamp)

@@ -1047,4 +1052,6 @@ python sstate_eventhandler2() {
         with open(i, "w") as f:
             for l in lines:
                 f.write(l)
+    if preservestamps:
+        os.remove(preservestampfile)
 }

index c17131a..165dcba 100644
@@ -1073,7 +1073,7 @@ class DevtoolTests(DevtoolBase):
 
     @OETestID(1628)
     def test_devtool_update_recipe_local_files_subdir(self):
-        # Try devtool extract on a recipe that has a file with subdir= set in
+        # Try devtool update-recipe on a recipe that has a file with subdir= set in
         # SRC_URI such that it overwrites a file that was in an archive that
         # was also in SRC_URI
         # First, modify the recipe

@@ -1103,10 +1103,10 @@ class DevtoolTests(DevtoolBase):
         tempdir = tempfile.mkdtemp(prefix='devtoolqa')
         # Try devtool extract
         self.track_for_cleanup(tempdir)
+        self.track_for_cleanup(self.workspacedir)
+        self.add_command_to_tearDown('bitbake-layers remove-layer */workspace')
         result = runCmd('devtool extract matchbox-terminal %s' % tempdir)
         self.assertExists(os.path.join(tempdir, 'Makefile.am'), 'Extracted source could not be found')
-        # devtool extract shouldn't create the workspace
-        self.assertNotExists(self.workspacedir)
         self._check_src_repo(tempdir)
 
     @OETestID(1379)

@@ -1114,10 +1114,10 @@ class DevtoolTests(DevtoolBase):
         tempdir = tempfile.mkdtemp(prefix='devtoolqa')
         # Try devtool extract
         self.track_for_cleanup(tempdir)
+        self.track_for_cleanup(self.workspacedir)
+        self.add_command_to_tearDown('bitbake-layers remove-layer */workspace')
         result = runCmd('devtool extract virtual/make %s' % tempdir)
         self.assertExists(os.path.join(tempdir, 'Makefile.am'), 'Extracted source could not be found')
-        # devtool extract shouldn't create the workspace
-        self.assertNotExists(self.workspacedir)
         self._check_src_repo(tempdir)
 
     @OETestID(1168)

273 lines not on this page · raw message

index 41bd34e..39d0bd7 100644
@@ -418,7 +418,7 @@ def upgrade(args, config, basepath, workspace):
 
         rf = None
         try:
-            rev1 = standard._extract_source(srctree, False, 'devtool-orig', False, rd, tinfoil)
+            rev1 = standard._extract_source(srctree, False, 'devtool-orig', False, config, rd, tinfoil)
             rev2, md5, sha256, srcbranch = _extract_new_source(args.version, srctree, args.no_patch,
                                                     args.srcrev, args.srcbranch, args.branch, args.keep_temp,
                                                     tinfoil, rd)

index 6620445..3620995 100644
@@ -346,6 +346,9 @@ python copy_buildsystem () {
             # the sig computed from the metadata.
             f.write('SIGGEN_LOCKEDSIGS_TASKSIG_CHECK = "warn"\n\n')
 
+            # We want to be able to set this without a full reparse
+            f.write('BB_HASHCONFIG_WHITELIST_append = " SIGGEN_UNLOCKED_RECIPES"\n\n')
+
             # Set up whitelist for run on install
             f.write('BB_SETSCENE_ENFORCE_WHITELIST = "%:* *:do_shared_workdir *:do_rm_work wic-tools:* *:do_addto_recipe_sysroot"\n\n')
 

index c9ad9dd..5292f18 100755
@@ -130,25 +130,6 @@ def read_workspace():
                                      'recipefile': recipefile}
                     logger.debug('Found recipe %s' % workspace[pn])
 
-def create_unlockedsigs():
-    """ This function will make unlocked-sigs.inc match the recipes in the
-    workspace. This runs on every run of devtool, but it lets us ensure
-    the unlocked items are in sync with the workspace. """
-
-    confdir = os.path.join(basepath, 'conf')
-    unlockedsigs = os.path.join(confdir, 'unlocked-sigs.inc')
-    bb.utils.mkdirhier(confdir)
-    with open(os.path.join(confdir, 'unlocked-sigs.inc'), 'w') as f:
-        f.write("# DO NOT MODIFY! YOUR CHANGES WILL BE LOST.\n" +
-                "# This layer was created by the OpenEmbedded devtool" +
-                " utility in order to\n" +
-                "# contain recipes that are unlocked.\n")
-
-        f.write('SIGGEN_UNLOCKED_RECIPES += "\\\n')
-        for pn in workspace:
-            f.write('    ' + pn)
-        f.write('"')
-
 def create_workspace(args, config, basepath, workspace):
     if args.layerpath:
         workspacedir = os.path.abspath(args.layerpath)

@@ -332,7 +313,6 @@ def main():
 
     if not getattr(args, 'no_workspace', False):
         read_workspace()
-        create_unlockedsigs()
 
     try:
         ret = args.func(args, config, basepath, workspace)

index 14170cb..94e3d7d 100644
@@ -297,3 +297,43 @@ def replace_from_file(path, old, new):
         except ValueError:
             pass
     write_file(path, "\n".join(new_contents))
+
+
+def update_unlockedsigs(basepath, workspace, fixed_setup, extra=None):
+    """ This function will make unlocked-sigs.inc match the recipes in the
+    workspace plus any extras we want unlocked. """
+
+    if not fixed_setup:
+        # Only need to write this out within the eSDK
+        return
+
+    if not extra:
+        extra = []
+
+    confdir = os.path.join(basepath, 'conf')
+    unlockedsigs = os.path.join(confdir, 'unlocked-sigs.inc')
+
+    # Get current unlocked list if any
+    values = {}
+    def get_unlockedsigs_varfunc(varname, origvalue, op, newlines):
+        values[varname] = origvalue
+        return origvalue, None, 0, True
+    if os.path.exists(unlockedsigs):
+        with open(unlockedsigs, 'r') as f:
+            bb.utils.edit_metadata(f, ['SIGGEN_UNLOCKED_RECIPES'], get_unlockedsigs_varfunc)
+    unlocked = sorted(values.get('SIGGEN_UNLOCKED_RECIPES', []))
+
+    # If the new list is different to the current list, write it out
+    newunlocked = sorted(list(workspace.keys()) + extra)
+    if unlocked != newunlocked:
+        bb.utils.mkdirhier(confdir)
+        with open(unlockedsigs, 'w') as f:
+            f.write("# DO NOT MODIFY! YOUR CHANGES WILL BE LOST.\n" +
+                    "# This layer was created by the OpenEmbedded devtool" +
+                    " utility in order to\n" +
+                    "# contain recipes that are unlocked.\n")
+
+            f.write('SIGGEN_UNLOCKED_RECIPES += "\\\n')
+            for pn in newunlocked:
+                f.write('    ' + pn)
+            f.write('"')

index dfa37b1..7587c38 100644
@@ -30,7 +30,7 @@ import errno
 import glob
 import filecmp
 from collections import OrderedDict
-from devtool import exec_build_env_command, setup_tinfoil, check_workspace_recipe, use_external_build, setup_git_repo, recipe_to_append, get_bbclassextend_targets, DevtoolError
+from devtool import exec_build_env_command, setup_tinfoil, check_workspace_recipe, use_external_build, setup_git_repo, recipe_to_append, get_bbclassextend_targets, update_unlockedsigs, DevtoolError
 from devtool import parse_recipe
 
 logger = logging.getLogger('devtool')

@@ -387,7 +387,7 @@ def extract(args, config, basepath, workspace):
             return 1
 
         srctree = os.path.abspath(args.srctree)
-        initial_rev = _extract_source(srctree, args.keep_temp, args.branch, False, config, rd, tinfoil)
+        initial_rev = _extract_source(srctree, args.keep_temp, args.branch, False, config, basepath, workspace, args.fixed_setup, rd, tinfoil)
         logger.info('Source tree extracted to %s' % srctree)
 
         if initial_rev:

@@ -411,7 +411,7 @@ def sync(args, config, basepath, workspace):
             return 1
 
         srctree = os.path.abspath(args.srctree)
-        initial_rev = _extract_source(srctree, args.keep_temp, args.branch, True, config, rd, tinfoil)
+        initial_rev = _extract_source(srctree, args.keep_temp, args.branch, True, config, basepath, workspace, args.fixed_setup, rd, tinfoil)
         logger.info('Source tree %s synchronized' % srctree)
 
         if initial_rev:

@@ -422,7 +422,7 @@ def sync(args, config, basepath, workspace):
         tinfoil.shutdown()
 
 
-def _extract_source(srctree, keep_temp, devbranch, sync, config, d, tinfoil):
+def _extract_source(srctree, keep_temp, devbranch, sync, config, basepath, workspace, fixed_setup, d, tinfoil):
     """Extract sources of a recipe"""
     import oe.recipeutils
     import oe.patch

@@ -691,7 +691,7 @@ def modify(args, config, basepath, workspace):
         initial_rev = None
         commits = []
         if not args.no_extract:
-            initial_rev = _extract_source(srctree, args.keep_temp, args.branch, False, config, rd, tinfoil)
+            initial_rev = _extract_source(srctree, args.keep_temp, args.branch, False, config, basepath, workspace, args.fixed_setup, rd, tinfoil)
             if not initial_rev:
                 return 1
             logger.info('Source tree extracted to %s' % srctree)

@@ -753,6 +753,8 @@ def modify(args, config, basepath, workspace):
                 for commit in commits:
                     f.write('# commit: %s\n' % commit)
 
+        update_unlockedsigs(basepath, workspace, args.fixed_setup, [pn])
+
         _add_md5(config, pn, appendfile)
 
         logger.info('Recipe %s now set up to build from %s' % (pn, srctree))

@@ -1760,7 +1762,7 @@ def register_commands(subparsers, context):
     group.add_argument('--no-same-dir', help='Force build in a separate build directory', action="store_true")
     parser_modify.add_argument('--branch', '-b', default="devtool", help='Name for development branch to checkout (when not using -n/--no-extract) (default "%(default)s")')
     parser_modify.add_argument('--keep-temp', help='Keep temporary directory (for debugging)', action="store_true")
-    parser_modify.set_defaults(func=modify)
+    parser_modify.set_defaults(func=modify, fixed_setup=context.fixed_setup)
 
     parser_extract = subparsers.add_parser('extract', help='Extract the source for an existing recipe',
                                        description='Extracts the source for an existing recipe',

@@ -1769,7 +1771,7 @@ def register_commands(subparsers, context):
     parser_extract.add_argument('srctree', help='Path to where to extract the source tree')
     parser_extract.add_argument('--branch', '-b', default="devtool", help='Name for development branch to checkout (default "%(default)s")')
     parser_extract.add_argument('--keep-temp', action="store_true", help='Keep temporary directory (for debugging)')
-    parser_extract.set_defaults(func=extract)
+    parser_extract.set_defaults(func=extract, fixed_setup=context.fixed_setup)
 
     parser_sync = subparsers.add_parser('sync', help='Synchronize the source tree for an existing recipe',
                                        description='Synchronize the previously extracted source tree for an existing recipe',

@@ -1779,7 +1781,7 @@ def register_commands(subparsers, context):
     parser_sync.add_argument('srctree', help='Path to the source tree')
     parser_sync.add_argument('--branch', '-b', default="devtool", help='Name for development branch to checkout')
     parser_sync.add_argument('--keep-temp', action="store_true", help='Keep temporary directory (for debugging)')
-    parser_sync.set_defaults(func=sync)
+    parser_sync.set_defaults(func=sync, fixed_setup=context.fixed_setup)
 
     parser_rename = subparsers.add_parser('rename', help='Rename a recipe file in the workspace',
                                        description='Renames the recipe file for a recipe in the workspace, changing the name or version part or both, ensuring that all references within the workspace are updated at the same time. Only works when the recipe file itself is in the workspace, e.g. after devtool add. Particularly useful when devtool add did not automatically determine the correct name.',

index 39d0bd7..f1b3ff0 100644
@@ -33,7 +33,7 @@ sys.path = sys.path + [devtool_path]
 
 import oe.recipeutils
 from devtool import standard
-from devtool import exec_build_env_command, setup_tinfoil, DevtoolError, parse_recipe, use_external_build
+from devtool import exec_build_env_command, setup_tinfoil, DevtoolError, parse_recipe, use_external_build, update_unlockedsigs
 
 logger = logging.getLogger('devtool')
 

@@ -418,7 +418,7 @@ def upgrade(args, config, basepath, workspace):
 
         rf = None
         try:
-            rev1 = standard._extract_source(srctree, False, 'devtool-orig', False, config, rd, tinfoil)
+            rev1 = standard._extract_source(srctree, False, 'devtool-orig', False, config, basepath, workspace, args.fixed_setup, rd, tinfoil)
             rev2, md5, sha256, srcbranch = _extract_new_source(args.version, srctree, args.no_patch,
                                                     args.srcrev, args.srcbranch, args.branch, args.keep_temp,
                                                     tinfoil, rd)

@@ -432,6 +432,9 @@ def upgrade(args, config, basepath, workspace):
         af = _write_append(rf, srctree, args.same_dir, args.no_same_dir, rev2,
                         copied, config.workspace_path, rd)
         standard._add_md5(config, pn, af)
+
+        update_unlockedsigs(basepath, workspace, [pn], args.fixed_setup)
+
         logger.info('Upgraded source extracted to %s' % srctree)
         logger.info('New recipe is %s' % rf)
     finally:

@@ -457,4 +460,4 @@ def register_commands(subparsers, context):
     group.add_argument('--same-dir', '-s', help='Build in same directory as source', action="store_true")
     group.add_argument('--no-same-dir', help='Force build in a separate build directory', action="store_true")
     parser_upgrade.add_argument('--keep-temp', action="store_true", help='Keep temporary directory (for debugging)')
-    parser_upgrade.set_defaults(func=upgrade)
+    parser_upgrade.set_defaults(func=upgrade, fixed_setup=context.fixed_setup)