devtool / recipetool improvements

index d12cc2e..590dfef 100644
@@ -150,7 +150,7 @@ def add(args, config, basepath, workspace):
             stdout, _ = exec_build_env_command(config.init_path, basepath, 'recipetool --color=%s create -o %s "%s" %s' % (color, tempdir, source, extracmdopts))
         except bb.process.ExecutionError as e:
             if e.exitcode == 15:
-                raise DevtoolError('Unable to auto-determine name from source tree, please specify it with -N/--name')
+                raise DevtoolError('Unable to auto-determine recipe name from source tree, please specify it on the command line')
             else:
                 raise DevtoolError('Command \'%s\' failed:\n%s' % (e.command, e.stdout))
 

index a8f5e97..18eddb7 100644
@@ -24,6 +24,7 @@ import tempfile
 import logging
 import argparse
 import subprocess
+import scriptutils
 from devtool import exec_build_env_command, setup_tinfoil, check_workspace_recipe, DevtoolError
 from devtool import parse_recipe
 

@@ -48,17 +49,7 @@ def edit_recipe(args, config, basepath, workspace):
             raise DevtoolError("Recipe file for %s is not under the workspace" %
                                args.recipename)
 
-    editor = os.environ.get('EDITOR', None)
-    if not editor:
-        raise DevtoolError("EDITOR environment variable not set")
-
-    import subprocess
-    try:
-        subprocess.check_call('%s "%s"' % (editor, recipefile), shell=True)
-    except subprocess.CalledProcessError as e:
-        return e.returncode
-
-    return 0
+    return scriptutils.run_editor(recipefile)
 
 
 def configure_help(args, config, basepath, workspace):

index 5625a8e..bdf0693 100644
@@ -27,6 +27,7 @@ import os
 import re
 import subprocess
 import sys
+import scriptutils
 
 
 logger = logging.getLogger('recipetool')

@@ -96,12 +97,7 @@ def newappend(args):
             return 1
 
     if args.edit:
-        editor = os.getenv('VISUAL', os.getenv('EDITOR', 'vi'))
-        try:
-            return subprocess.check_call([editor, append_path, recipe_path])
-        except OSError as exc:
-            logger.error("Execution of editor '%s' failed: %s", editor, exc)
-            return 1
+        return scriptutils.run_editor([append_path, recipe_path])
     else:
         print(append_path)
 

index 69e76d8..aef19d3 100644
@@ -20,6 +20,7 @@ import os
 import logging
 import glob
 import argparse
+import subprocess
 
 def logger_create(name):
     logger = logging.getLogger(name)

@@ -101,3 +102,17 @@ def fetch_uri(d, uri, destdir, srcrev=None):
         os.chdir(olddir)
     return ret
 
+def run_editor(fn):
+    if isinstance(fn, basestring):
+        params = '"%s"' % fn
+    else:
+        params = ''
+        for fnitem in fn:
+            params += ' "%s"' % fnitem
+
+    editor = os.getenv('VISUAL', os.getenv('EDITOR', 'vi'))
+    try:
+        return subprocess.check_call('%s %s' % (editor, params), shell=True)
+    except OSError as exc:
+        logger.error("Execution of editor '%s' failed: %s", editor, exc)
+        return 1

index 590dfef..bbbe426 100644
@@ -190,7 +190,7 @@ def add(args, config, basepath, workspace):
             shutil.move(recipes[0], recipefile)
         else:
             raise DevtoolError('Command \'%s\' did not create any recipe file:\n%s' % (e.command, e.stdout))
-        attic_recipe = os.path.join(config.workspace_path, 'attic', os.path.basename(recipefile))
+        attic_recipe = os.path.join(config.workspace_path, 'attic', recipename, os.path.basename(recipefile))
         if os.path.exists(attic_recipe):
             logger.warn('A modified recipe from a previous invocation exists in %s - you may wish to move this over the top of the new recipe if you had changes in it that you want to continue with' % attic_recipe)
     finally:

@@ -645,7 +645,7 @@ def _check_preserve(config, recipename):
     import bb.utils
     origfile = os.path.join(config.workspace_path, '.devtool_md5')
     newfile = os.path.join(config.workspace_path, '.devtool_md5_new')
-    preservepath = os.path.join(config.workspace_path, 'attic')
+    preservepath = os.path.join(config.workspace_path, 'attic', recipename)
     with open(origfile, 'r') as f:
         with open(newfile, 'w') as tf:
             for line in f.readlines():

@@ -1256,7 +1256,7 @@ def reset(args, config, basepath, workspace):
     for pn in recipes:
         _check_preserve(config, pn)
 
-        preservepath = os.path.join(config.workspace_path, 'attic', pn)
+        preservepath = os.path.join(config.workspace_path, 'attic', pn, pn)
         def preservedir(origdir):
             if os.path.exists(origdir):
                 for root, dirs, files in os.walk(origdir):

@@ -1265,7 +1265,7 @@ def reset(args, config, basepath, workspace):
                         _move_file(os.path.join(origdir, fn),
                                    os.path.join(preservepath, fn))
                     for dn in dirs:
-                        os.rmdir(os.path.join(root, dn))
+                        preservedir(os.path.join(root, dn))
                 os.rmdir(origdir)
 
         preservedir(os.path.join(config.workspace_path, 'recipes', pn))

index bbbe426..804c127 100644
@@ -1034,6 +1034,7 @@ def _update_recipe_srcrev(args, srctree, rd, config_data):
                     'changes')
 
     _remove_source_files(args, remove_files, destpath)
+    return True
 
 def _update_recipe_patch(args, config, workspace, srctree, rd, config_data):
     """Implement the 'patch' mode of update-recipe"""

@@ -1135,10 +1136,12 @@ def _update_recipe_patch(args, config, workspace, srctree, rd, config_data):
             elif not updatefiles:
                 # Neither patches nor recipe were updated
                 logger.info('No patches or files need updating')
+                return False
     finally:
         shutil.rmtree(tempdir)
 
     _remove_source_files(args, remove_files, destpath)
+    return True
 
 def _guess_recipe_update_mode(srctree, rdata):
     """Guess the recipe update mode to use"""

@@ -1187,15 +1190,16 @@ def update_recipe(args, config, basepath, workspace):
         mode = args.mode
 
     if mode == 'srcrev':
-        _update_recipe_srcrev(args, srctree, rd, tinfoil.config_data)
+        updated = _update_recipe_srcrev(args, srctree, rd, tinfoil.config_data)
     elif mode == 'patch':
-        _update_recipe_patch(args, config, workspace, srctree, rd, tinfoil.config_data)
+        updated = _update_recipe_patch(args, config, workspace, srctree, rd, tinfoil.config_data)
     else:
         raise DevtoolError('update_recipe: invalid mode %s' % mode)
 
-    rf = rd.getVar('FILE', True)
-    if rf.startswith(config.workspace_path):
-        logger.warn('Recipe file %s has been updated but is inside the workspace - you will need to move it (and any associated files next to it) out to the desired layer before using "devtool reset" in order to keep any changes' % rf)
+    if updated:
+        rf = rd.getVar('FILE', True)
+        if rf.startswith(config.workspace_path):
+            logger.warn('Recipe file %s has been updated but is inside the workspace - you will need to move it (and any associated files next to it) out to the desired layer before using "devtool reset" in order to keep any changes' % rf)
 
     return 0
 

index 2d57da0b..ed8f3a1 100755
@@ -271,10 +271,18 @@ def main():
 
     subparsers = parser.add_subparsers(dest="subparser_name", title='subcommands', metavar='<subcommand>')
 
+    subparsers.add_subparser_group('sdk', 'SDK maintenance', -2)
+    subparsers.add_subparser_group('advanced', 'Advanced', -1)
+    subparsers.add_subparser_group('starting', 'Beginning work on a recipe', 100)
+    subparsers.add_subparser_group('info', 'Getting information')
+    subparsers.add_subparser_group('working', 'Working on a recipe in the workspace')
+    subparsers.add_subparser_group('testbuild', 'Testing changes on target')
+
     if not context.fixed_setup:
         parser_create_workspace = subparsers.add_parser('create-workspace',
                                                         help='Set up workspace in an alternative location',
-                                                        description='Sets up a new workspace. NOTE: other devtool subcommands will create a workspace automatically as needed, so you only need to use %(prog)s if you want to specify where the workspace should be located.')
+                                                        description='Sets up a new workspace. NOTE: other devtool subcommands will create a workspace automatically as needed, so you only need to use %(prog)s if you want to specify where the workspace should be located.',
+                                                        group='advanced')
         parser_create_workspace.add_argument('layerpath', nargs='?', help='Path in which the workspace layer should be created')
         parser_create_workspace.add_argument('--create-only', action="store_true", help='Only create the workspace layer, do not alter configuration')
         parser_create_workspace.set_defaults(func=create_workspace, no_workspace=True)

83 lines not on this page · raw message

index 48c3a11..ff764fa 100644
@@ -109,7 +109,8 @@ def register_commands(subparsers, context):
     parser = subparsers.add_parser('build-image',
                                    help='Build image including workspace recipe packages',
                                    description='Builds an image, extending it to include '
-                                   'packages from recipes in the workspace')
+                                   'packages from recipes in the workspace',
+                                   group='testbuild', order=-10)
     parser.add_argument('imagename', help='Image recipe to build', nargs='?')
     parser.add_argument('-p', '--add-packages', help='Instead of adding packages for the '
                         'entire workspace, specify packages to be added to the image '

index b10a6a9..48f6fe1 100644
@@ -79,7 +79,8 @@ def build(args, config, basepath, workspace):
 def register_commands(subparsers, context):
     """Register devtool subcommands from this plugin"""
     parser_build = subparsers.add_parser('build', help='Build a recipe',
-                                         description='Builds the specified recipe using bitbake (up to and including %s)' % ', '.join(_get_build_tasks(context.config)))
+                                         description='Builds the specified recipe using bitbake (up to and including %s)' % ', '.join(_get_build_tasks(context.config)),
+                                         group='working')
     parser_build.add_argument('recipename', help='Recipe to build')
     parser_build.add_argument('-s', '--disable-parallel-make', action="store_true", help='Disable make parallelism')
     parser_build.set_defaults(func=build)

index c90c6b1..0236c53 100644
@@ -131,7 +131,9 @@ def undeploy(args, config, basepath, workspace):
 
 def register_commands(subparsers, context):
     """Register devtool subcommands from the deploy plugin"""
-    parser_deploy = subparsers.add_parser('deploy-target', help='Deploy recipe output files to live target machine')
+    parser_deploy = subparsers.add_parser('deploy-target',
+                                          help='Deploy recipe output files to live target machine',
+                                          group='testbuild')
     parser_deploy.add_argument('recipename', help='Recipe to deploy')
     parser_deploy.add_argument('target', help='Live target machine running an ssh server: user@hostname[:destdir]')
     parser_deploy.add_argument('-c', '--no-host-check', help='Disable ssh host key checking', action='store_true')

@@ -139,7 +141,9 @@ def register_commands(subparsers, context):
     parser_deploy.add_argument('-n', '--dry-run', help='List files to be deployed only', action='store_true')
     parser_deploy.set_defaults(func=deploy)
 
-    parser_undeploy = subparsers.add_parser('undeploy-target', help='Undeploy recipe output files in live target machine')
+    parser_undeploy = subparsers.add_parser('undeploy-target',
+                                            help='Undeploy recipe output files in live target machine',
+                                            group='testbuild')
     parser_undeploy.add_argument('recipename', help='Recipe to undeploy')
     parser_undeploy.add_argument('target', help='Live target machine running an ssh server: user@hostname')
     parser_undeploy.add_argument('-c', '--no-host-check', help='Disable ssh host key checking', action='store_true')

index a296fce..afb5809 100644
@@ -54,6 +54,9 @@ def package(args, config, basepath, workspace):
 def register_commands(subparsers, context):
     """Register devtool subcommands from the package plugin"""
     if context.fixed_setup:
-        parser_package = subparsers.add_parser('package', help='Build packages for a recipe', description='Builds packages for a recipe\'s output files')
+        parser_package = subparsers.add_parser('package',
+                                               help='Build packages for a recipe',
+                                               description='Builds packages for a recipe\'s output files',
+                                               group='testbuild', order=-5)
         parser_package.add_argument('recipename', help='Recipe to package')
         parser_package.set_defaults(func=package)

index 5282afb..daee7fb 100644
@@ -57,7 +57,8 @@ def register_commands(subparsers, context):
     """Register devtool subcommands from this plugin"""
     if context.fixed_setup:
         parser_runqemu = subparsers.add_parser('runqemu', help='Run QEMU on the specified image',
-                                               description='Runs QEMU to boot the specified image')
+                                               description='Runs QEMU to boot the specified image',
+                                               group='testbuild', order=-20)
         parser_runqemu.add_argument('imagename', help='Name of built image to boot within QEMU', nargs='?')
         parser_runqemu.add_argument('args', help='Any remaining arguments are passed to the runqemu script (pass --help after imagename to see what these are)',
                                     nargs=argparse.REMAINDER)

index 12de942..f6c5434 100644
@@ -296,10 +296,16 @@ def sdk_install(args, config, basepath, workspace):
 def register_commands(subparsers, context):
     """Register devtool subcommands from the sdk plugin"""
     if context.fixed_setup:
-        parser_sdk = subparsers.add_parser('sdk-update', help='Update SDK components from a nominated location')
+        parser_sdk = subparsers.add_parser('sdk-update',
+                                           help='Update SDK components from a nominated location',
+                                           group='sdk')
         parser_sdk.add_argument('updateserver', help='The update server to fetch latest SDK components from', nargs='?')
         parser_sdk.add_argument('--skip-prepare', action="store_true", help='Skip re-preparing the build system after updating (for debugging only)')
         parser_sdk.set_defaults(func=sdk_update)
-        parser_sdk_install = subparsers.add_parser('sdk-install', help='Install additional SDK components', description='Installs additional recipe development files into the SDK. (You can use "devtool search" to find available recipes.)')
+
+        parser_sdk_install = subparsers.add_parser('sdk-install',
+                                                   help='Install additional SDK components',
+                                                   description='Installs additional recipe development files into the SDK. (You can use "devtool search" to find available recipes.)',
+                                                   group='sdk')
         parser_sdk_install.add_argument('recipename', help='Name of the recipe to install the development artifacts for', nargs='+')
         parser_sdk_install.set_defaults(func=sdk_install)

index 2ea4462..b44bed7 100644
@@ -82,6 +82,7 @@ def search(args, config, basepath, workspace):
 def register_commands(subparsers, context):
     """Register devtool subcommands from this plugin"""
     parser_search = subparsers.add_parser('search', help='Search available recipes',
-                                            description='Searches for available target recipes. Matches on recipe name, package name, description and installed files, and prints the recipe name on match.')
+                                            description='Searches for available target recipes. Matches on recipe name, package name, description and installed files, and prints the recipe name on match.',
+                                            group='info')
     parser_search.add_argument('keyword', help='Keyword to search for (regular expression syntax allowed)')
     parser_search.set_defaults(func=search, no_workspace=True)

index 804c127..084039a 100644
@@ -1303,7 +1303,8 @@ def register_commands(subparsers, context):
 
     defsrctree = get_default_srctree(context.config)
     parser_add = subparsers.add_parser('add', help='Add a new recipe',
-                                       description='Adds a new recipe to the workspace to build a specified source tree. Can optionally fetch a remote URI and unpack it to create the source tree.')
+                                       description='Adds a new recipe to the workspace to build a specified source tree. Can optionally fetch a remote URI and unpack it to create the source tree.',
+                                       group='starting', order=100)
     parser_add.add_argument('recipename', nargs='?', help='Name for new recipe to add (just name - no version, path or extension). If not specified, will attempt to auto-detect it.')
     parser_add.add_argument('srctree', nargs='?', help='Path to external source tree. If not specified, a subdirectory of %s will be used.' % defsrctree)
     parser_add.add_argument('fetchuri', nargs='?', help='Fetch the specified URI and extract it to create the source tree')

@@ -1319,7 +1320,8 @@ def register_commands(subparsers, context):
     parser_add.set_defaults(func=add)
 
     parser_modify = subparsers.add_parser('modify', help='Modify the source for an existing recipe',
-                                       description='Enables modifying the source for an existing recipe. You can either provide your own pre-prepared source tree, or specify -x/--extract to extract the source being fetched by the recipe.')
+                                       description='Enables modifying the source for an existing recipe. You can either provide your own pre-prepared source tree, or specify -x/--extract to extract the source being fetched by the recipe.',
+                                       group='starting', order=90)
     parser_modify.add_argument('recipename', help='Name of existing recipe to edit (just name - no version, path or extension)')
     parser_modify.add_argument('srctree', nargs='?', help='Path to external source tree. If not specified, a subdirectory of %s will be used.' % defsrctree)
     parser_modify.add_argument('--wildcard', '-w', action="store_true", help='Use wildcard for unversioned bbappend')

@@ -1333,7 +1335,8 @@ def register_commands(subparsers, context):
     parser_modify.set_defaults(func=modify)
 
     parser_extract = subparsers.add_parser('extract', help='Extract the source for an existing recipe',
-                                       description='Extracts the source for an existing recipe')
+                                       description='Extracts the source for an existing recipe',
+                                       group='advanced')
     parser_extract.add_argument('recipename', help='Name of recipe to extract the source for')
     parser_extract.add_argument('srctree', help='Path to where to extract the source tree')
     parser_extract.add_argument('--branch', '-b', default="devtool", help='Name for development branch to checkout (default "%(default)s")')

@@ -1342,7 +1345,8 @@ def register_commands(subparsers, context):
 
     parser_sync = subparsers.add_parser('sync', help='Synchronize the source tree for an existing recipe',
                                        description='Synchronize the previously extracted source tree for an existing recipe',
-                                       formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+                                       formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+                                       group='advanced')
     parser_sync.add_argument('recipename', help='Name of recipe to sync the source for')
     parser_sync.add_argument('srctree', help='Path to the source tree')
     parser_sync.add_argument('--branch', '-b', default="devtool", help='Name for development branch to checkout')

@@ -1350,7 +1354,8 @@ def register_commands(subparsers, context):
     parser_sync.set_defaults(func=sync)
 
     parser_update_recipe = subparsers.add_parser('update-recipe', help='Apply changes from external source tree to recipe',
-                                       description='Applies changes from external source tree to a recipe (updating/adding/removing patches as necessary, or by updating SRCREV). Note that these changes need to have been committed to the git repository in order to be recognised.')
+                                       description='Applies changes from external source tree to a recipe (updating/adding/removing patches as necessary, or by updating SRCREV). Note that these changes need to have been committed to the git repository in order to be recognised.',
+                                       group='working', order=-90)
     parser_update_recipe.add_argument('recipename', help='Name of recipe to update')
     parser_update_recipe.add_argument('--mode', '-m', choices=['patch', 'srcrev', 'auto'], default='auto', help='Update mode (where %(metavar)s is %(choices)s; default is %(default)s)', metavar='MODE')
     parser_update_recipe.add_argument('--initial-rev', help='Override starting revision for patches')

@@ -1360,11 +1365,13 @@ def register_commands(subparsers, context):
     parser_update_recipe.set_defaults(func=update_recipe)
 
     parser_status = subparsers.add_parser('status', help='Show workspace status',
-                                          description='Lists recipes currently in your workspace and the paths to their respective external source trees')
+                                          description='Lists recipes currently in your workspace and the paths to their respective external source trees',
+                                          group='info', order=100)
     parser_status.set_defaults(func=status)
 
     parser_reset = subparsers.add_parser('reset', help='Remove a recipe from your workspace',
-                                         description='Removes the specified recipe from your workspace (resetting its state)')
+                                         description='Removes the specified recipe from your workspace (resetting its state)',
+                                         group='working', order=-100)
     parser_reset.add_argument('recipename', nargs='?', help='Recipe to reset')
     parser_reset.add_argument('--all', '-a', action="store_true", help='Reset all recipes (clear workspace)')
     parser_reset.add_argument('--no-clean', '-n', action="store_true", help='Don\'t clean the sysroot to remove recipe output')

index e2be38e..0e53c82 100644
@@ -339,7 +339,8 @@ def upgrade(args, config, basepath, workspace):
 def register_commands(subparsers, context):
     """Register devtool subcommands from this plugin"""
     parser_upgrade = subparsers.add_parser('upgrade', help='Upgrade an existing recipe',
-                                           description='Upgrades an existing recipe to a new upstream version. Puts the upgraded recipe file into the workspace along with any associated files, and extracts the source tree to a specified location (in case patches need rebasing or adding to as a result of the upgrade).')
+                                           description='Upgrades an existing recipe to a new upstream version. Puts the upgraded recipe file into the workspace along with any associated files, and extracts the source tree to a specified location (in case patches need rebasing or adding to as a result of the upgrade).',
+                                           group='starting')
     parser_upgrade.add_argument('recipename', help='Name of recipe to upgrade (just name - no version, path or extension)')
     parser_upgrade.add_argument('srctree', help='Path to where to extract the source tree')
     parser_upgrade.add_argument('--version', '-V', help='Version to upgrade to (PV)')

index 18eddb7..905d6d2 100644
@@ -214,7 +214,8 @@ The ./configure %s output for %s follows.
 def register_commands(subparsers, context):
     """Register devtool subcommands from this plugin"""
     parser_edit_recipe = subparsers.add_parser('edit-recipe', help='Edit a recipe file in your workspace',
-                                         description='Runs the default editor (as specified by the EDITOR variable) on the specified recipe. Note that the recipe file itself must be in the workspace (i.e. as a result of "devtool add" or "devtool upgrade"); you can override this with the -a/--any-recipe option.')
+                                         description='Runs the default editor (as specified by the EDITOR variable) on the specified recipe. Note that the recipe file itself must be in the workspace (i.e. as a result of "devtool add" or "devtool upgrade"); you can override this with the -a/--any-recipe option.',
+                                         group='working')
     parser_edit_recipe.add_argument('recipename', help='Recipe to edit')
     parser_edit_recipe.add_argument('--any-recipe', '-a', action="store_true", help='Edit any recipe, not just where the recipe file itself is in the workspace')
     parser_edit_recipe.set_defaults(func=edit_recipe)

@@ -223,7 +224,8 @@ def register_commands(subparsers, context):
     # gets the order wrong - recipename must come before --arg
     parser_configure_help = subparsers.add_parser('configure-help', help='Get help on configure script options',
                                          usage='devtool configure-help [options] recipename [--arg ...]',
-                                         description='Displays the help for the configure script for the specified recipe (i.e. runs ./configure --help) prefaced by a header describing the current options being specified. Output is piped through less (or whatever PAGER is set to, if set) for easy browsing.')
+                                         description='Displays the help for the configure script for the specified recipe (i.e. runs ./configure --help) prefaced by a header describing the current options being specified. Output is piped through less (or whatever PAGER is set to, if set) for easy browsing.',
+                                         group='working')
     parser_configure_help.add_argument('recipename', help='Recipe to show configure help for')
     parser_configure_help.add_argument('-p', '--no-pager', help='Disable paged output', action="store_true")
     parser_configure_help.add_argument('-n', '--no-header', help='Disable explanatory header text', action="store_true")

index 744cfe3..bf3ebad 100644
@@ -51,6 +51,10 @@ class ArgumentSubParser(ArgumentParser):
         if 'order' in kwargs:
             self._order = kwargs.pop('order')
         super(ArgumentSubParser, self).__init__(*args, **kwargs)
+        for agroup in self._action_groups:
+            if agroup.title == 'optional arguments':
+                agroup.title = 'options'
+                break
 
     def parse_known_args(self, args=None, namespace=None):
         # This works around argparse not handling optional positional arguments being

index 0236c53..d742ed3 100644
@@ -133,6 +133,7 @@ def register_commands(subparsers, context):
     """Register devtool subcommands from the deploy plugin"""
     parser_deploy = subparsers.add_parser('deploy-target',
                                           help='Deploy recipe output files to live target machine',
+                                          description='Deploys a recipe\'s build output (i.e. the output of the do_install task) to a live target machine over ssh. Note: this only deploys the recipe itself and not any runtime dependencies, so it is assumed that those have been installed on the target beforehand.',
                                           group='testbuild')
     parser_deploy.add_argument('recipename', help='Recipe to deploy')
     parser_deploy.add_argument('target', help='Live target machine running an ssh server: user@hostname[:destdir]')

@@ -143,6 +144,7 @@ def register_commands(subparsers, context):
 
     parser_undeploy = subparsers.add_parser('undeploy-target',
                                             help='Undeploy recipe output files in live target machine',
+                                            description='Un-deploys recipe output files previously deployed to a live target machine by devtool deploy-target.',
                                             group='testbuild')
     parser_undeploy.add_argument('recipename', help='Recipe to undeploy')
     parser_undeploy.add_argument('target', help='Live target machine running an ssh server: user@hostname')

index f6c5434..fbf2e79 100644
@@ -95,8 +95,6 @@ def sdk_update(args, config, basepath, workspace):
     updateserver = args.updateserver
     if not updateserver:
         updateserver = config.get('SDK', 'updateserver', '')
-    if not updateserver:
-        raise DevtoolError("Update server not specified in config file, you must specify it on the command line")
     logger.debug("updateserver: %s" % updateserver)
 
     # Make sure we are using sdk-update from within SDK

@@ -297,9 +295,14 @@ def register_commands(subparsers, context):
     """Register devtool subcommands from the sdk plugin"""
     if context.fixed_setup:
         parser_sdk = subparsers.add_parser('sdk-update',
-                                           help='Update SDK components from a nominated location',
+                                           help='Update SDK components',
+                                           description='Updates installed SDK components from a remote server',
                                            group='sdk')
-        parser_sdk.add_argument('updateserver', help='The update server to fetch latest SDK components from', nargs='?')
+        updateserver = context.config.get('SDK', 'updateserver', '')
+        if updateserver:
+            parser_sdk.add_argument('updateserver', help='The update server to fetch latest SDK components from (default %s)' % updateserver, nargs='?')
+        else:
+            parser_sdk.add_argument('updateserver', help='The update server to fetch latest SDK components from')
         parser_sdk.add_argument('--skip-prepare', action="store_true", help='Skip re-preparing the build system after updating (for debugging only)')
         parser_sdk.set_defaults(func=sdk_update)
 

index d742ed3..d2a314b 100644
@@ -1,6 +1,6 @@
 # Development tool - deploy/undeploy command plugin
 #
-# Copyright (C) 2014-2015 Intel Corporation
+# Copyright (C) 2014-2016 Intel Corporation
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 2 as

@@ -19,10 +19,57 @@
 import os
 import subprocess
 import logging
+import tempfile
+import shutil
 from devtool import exec_fakeroot, setup_tinfoil, check_workspace_recipe, DevtoolError
 
 logger = logging.getLogger('devtool')
 
+deploylist_path = '/.devtool'
+
+def _prepare_remote_script(deploy, verbose=False):
+    """
+    Prepare a shell script for running on the target to
+    deploy/undeploy files. We have to be careful what we put in this
+    script - only commands that are likely to be available on the
+    target are suitable (the target might be constrained, e.g. using
+    busybox rather than bash with coreutils).
+    """
+    lines = []
+    lines.append('#!/bin/sh')
+    lines.append('set -e')
+    lines.append('manifest="%s/$1.list"' % deploylist_path)
+    lines.append('if [ -f $manifest ] ; then')
+    # Read manifest in reverse and delete files / remove empty dirs
+    lines.append('    sed \'1!G;h;$!d\' $manifest | while read file')
+    lines.append('    do')
+    lines.append('        if [ -d $file ] ; then')
+    lines.append('            rmdir $file > /dev/null 2>&1 || true')
+    lines.append('        else')
+    lines.append('            rm $file')
+    lines.append('        fi')
+    lines.append('    done')
+    lines.append('    rm $manifest')
+    if not deploy:
+        # May as well remove all traces
+        lines.append('    rmdir `dirname $manifest` > /dev/null 2>&1 || true')
+    lines.append('fi')
+
+    if deploy:
+        lines.append('mkdir -p `dirname $manifest`')
+        lines.append('mkdir -p $2')
+        if verbose:
+            lines.append('    tar xv -C $2 -f - | tee $manifest')
+        else:
+            lines.append('    tar xv -C $2 -f - > $manifest')
+        lines.append('sed -i "s!^./!$2!" $manifest')
+    # Delete the script itself
+    lines.append('rm $0')
+    lines.append('')
+
+    return '\n'.join(lines)
+
+
 def deploy(args, config, basepath, workspace):
     """Entry point for the devtool 'deploy' subcommand"""
     import re

@@ -36,9 +83,8 @@ def deploy(args, config, basepath, workspace):
         destdir = '/'
     else:
         args.target = host
-
-    deploy_dir = os.path.join(basepath, 'target_deploy', args.target)
-    deploy_file = os.path.join(deploy_dir, args.recipename + '.list')
+    if not destdir.endswith('/'):
+        destdir += '/'
 
     tinfoil = setup_tinfoil(basepath=basepath)
     try:

@@ -59,74 +105,90 @@ def deploy(args, config, basepath, workspace):
                 print('  %s' % os.path.join(destdir, os.path.relpath(root, recipe_outdir), fn))
         return 0
 
-    if os.path.exists(deploy_file):
-        if undeploy(args, config, basepath, workspace):
-            # Error already shown
-            return 1
 
     extraoptions = ''
     if args.no_host_check:
         extraoptions += '-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no'
-    if args.show_status:
-        tarextractopts = 'xv'
-    else:
-        tarextractopts = 'x'
+    if not args.show_status:
         extraoptions += ' -q'
-    # We cannot use scp here, because it doesn't preserve symlinks
-    ret = exec_fakeroot(rd, 'tar cf - . | ssh %s %s \'tar %s -C %s -f -\'' % (extraoptions, args.target, tarextractopts, destdir), cwd=recipe_outdir, shell=True)
+
+    # In order to delete previously deployed files and have the manifest file on
+    # the target, we write out a shell script and then copy it to the target
+    # so we can then run it (piping tar output to it).
+    # (We cannot use scp here, because it doesn't preserve symlinks.)
+    tmpdir = tempfile.mkdtemp(prefix='devtool')
+    try:
+        tmpscript = '/tmp/devtool_deploy.sh'
+        shellscript = _prepare_remote_script(deploy=True, verbose=args.show_status)
+        # Write out the script to a file
+        with open(os.path.join(tmpdir, os.path.basename(tmpscript)), 'w') as f:
+            f.write(shellscript)
+        # Copy it to the target
+        ret = subprocess.call("scp %s %s/* %s:%s" % (extraoptions, tmpdir, args.target, os.path.dirname(tmpscript)), shell=True)
+        if ret != 0:
+            raise DevtoolError('Failed to copy script to %s - rerun with -s to '
+                            'get a complete error message' % args.target)
+    finally:
+        shutil.rmtree(tmpdir)
+
+    # Now run the script
+    ret = exec_fakeroot(rd, 'tar cf - . | ssh %s %s \'sh %s %s %s\'' % (extraoptions, args.target, tmpscript, args.recipename, destdir), cwd=recipe_outdir, shell=True)
     if ret != 0:
         raise DevtoolError('Deploy failed - rerun with -s to get a complete '
                            'error message')
 
     logger.info('Successfully deployed %s' % recipe_outdir)
 
-    if not os.path.exists(deploy_dir):
-        os.makedirs(deploy_dir)
-
     files_list = []
     for root, _, files in os.walk(recipe_outdir):
         for filename in files:
             filename = os.path.relpath(os.path.join(root, filename), recipe_outdir)
             files_list.append(os.path.join(destdir, filename))
 
-    with open(deploy_file, 'w') as fobj:
-        fobj.write('\n'.join(files_list))
-
     return 0
 
 def undeploy(args, config, basepath, workspace):
     """Entry point for the devtool 'undeploy' subcommand"""
-    deploy_file = os.path.join(basepath, 'target_deploy', args.target, args.recipename + '.list')
-    if not os.path.exists(deploy_file):
-        raise DevtoolError('%s has not been deployed' % args.recipename)
-
-    if args.dry_run:
-        print('Previously deployed files to be un-deployed for %s on target %s:' % (args.recipename, args.target))
-        with open(deploy_file, 'r') as f:
-            for line in f:
-                print('  %s' % line.rstrip())
-        return 0
-
     extraoptions = ''
     if args.no_host_check:
         extraoptions += '-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no'
     if not args.show_status:
         extraoptions += ' -q'
 
-    ret = subprocess.call("scp %s %s %s:/tmp" % (extraoptions, deploy_file, args.target), shell=True)
-    if ret != 0:
-        raise DevtoolError('Failed to copy file list to %s - rerun with -s to '
-                           'get a complete error message' % args.target)
+    args.target = args.target.split(':')[0]
 
-    ret = subprocess.call("ssh %s %s 'xargs -n1 rm -f </tmp/%s'" % (extraoptions, args.target, os.path.basename(deploy_file)), shell=True)
-    if ret == 0:
-        logger.info('Successfully undeployed %s' % args.recipename)
-        os.remove(deploy_file)
-    else:
+    if args.dry_run:
+        listfile = os.path.join(deploylist_path, '%s.list' % args.recipename)
+        print('Previously deployed files to be un-deployed for %s on target %s:' % (args.recipename, args.target))
+        ret = subprocess.call('ssh %s %s \'[ -f %s ] && cat %s || true\'' % (extraoptions, args.target, listfile, listfile), shell=True)
+        if ret != 0:
+            raise DevtoolError('Undeploy failed - rerun with -s to get a complete '
+                               'error message')
+        return 0
+
+    tmpdir = tempfile.mkdtemp(prefix='devtool')
+    try:
+        tmpscript = '/tmp/devtool_undeploy.sh'
+        shellscript = _prepare_remote_script(deploy=False)
+        # Write out the script to a file
+        with open(os.path.join(tmpdir, os.path.basename(tmpscript)), 'w') as f:
+            f.write(shellscript)
+        # Copy it to the target
+        ret = subprocess.call("scp %s %s/* %s:%s" % (extraoptions, tmpdir, args.target, os.path.dirname(tmpscript)), shell=True)
+        if ret != 0:
+            raise DevtoolError('Failed to copy script to %s - rerun with -s to '
+                                'get a complete error message' % args.target)
+    finally:
+        shutil.rmtree(tmpdir)
+
+    # Now run the script
+    ret = subprocess.call('ssh %s %s \'sh %s %s\'' % (extraoptions, args.target, tmpscript, args.recipename), shell=True)
+    if ret != 0:
         raise DevtoolError('Undeploy failed - rerun with -s to get a complete '
                            'error message')
 
-    return ret
+    logger.info('Successfully undeployed %s' % args.recipename)
+    return 0
 
 
 def register_commands(subparsers, context):

index d2a314b..d54f6ba 100644
@@ -21,13 +21,14 @@ import subprocess
 import logging
 import tempfile
 import shutil
+import argparse_oe
 from devtool import exec_fakeroot, setup_tinfoil, check_workspace_recipe, DevtoolError
 
 logger = logging.getLogger('devtool')
 
 deploylist_path = '/.devtool'
 
-def _prepare_remote_script(deploy, verbose=False):
+def _prepare_remote_script(deploy, verbose=False, dryrun=False, undeployall=False):
     """
     Prepare a shell script for running on the target to
     deploy/undeploy files. We have to be careful what we put in this

@@ -38,19 +39,33 @@ def _prepare_remote_script(deploy, verbose=False):
     lines = []
     lines.append('#!/bin/sh')
     lines.append('set -e')
+    if undeployall:
+        # Yes, I know this is crude - but it does work
+        lines.append('for entry in %s/*.list; do' % deploylist_path)
+        lines.append('[ ! -f $entry ] && exit')
+        lines.append('set `basename $entry | sed "s/.list//"`')
+    if dryrun:
+        if not deploy:
+            lines.append('echo "Previously deployed files for $1:"')
     lines.append('manifest="%s/$1.list"' % deploylist_path)
     lines.append('if [ -f $manifest ] ; then')
     # Read manifest in reverse and delete files / remove empty dirs
     lines.append('    sed \'1!G;h;$!d\' $manifest | while read file')
     lines.append('    do')
-    lines.append('        if [ -d $file ] ; then')
-    lines.append('            rmdir $file > /dev/null 2>&1 || true')
-    lines.append('        else')
-    lines.append('            rm $file')
-    lines.append('        fi')
+    if dryrun:
+        lines.append('        if [ ! -d $file ] ; then')
+        lines.append('            echo $file')
+        lines.append('        fi')
+    else:
+        lines.append('        if [ -d $file ] ; then')
+        lines.append('            rmdir $file > /dev/null 2>&1 || true')
+        lines.append('        else')
+        lines.append('            rm $file')
+        lines.append('        fi')
     lines.append('    done')
-    lines.append('    rm $manifest')
-    if not deploy:
+    if not dryrun:
+        lines.append('    rm $manifest')
+    if not deploy and not dryrun:
         # May as well remove all traces
         lines.append('    rmdir `dirname $manifest` > /dev/null 2>&1 || true')
     lines.append('fi')

@@ -63,6 +78,12 @@ def _prepare_remote_script(deploy, verbose=False):
         else:
             lines.append('    tar xv -C $2 -f - > $manifest')
         lines.append('sed -i "s!^./!$2!" $manifest')
+
+    if undeployall:
+        if not dryrun:
+            lines.append('echo "NOTE: Successfully undeployed $1"')
+        lines.append('done')
+
     # Delete the script itself
     lines.append('rm $0')
     lines.append('')

@@ -149,6 +170,11 @@ def deploy(args, config, basepath, workspace):
 
 def undeploy(args, config, basepath, workspace):
     """Entry point for the devtool 'undeploy' subcommand"""
+    if args.all and args.recipename:
+        raise argparse_oe.ArgumentUsageError('Cannot specify -a/--all with a recipe name', 'undeploy-target')
+    elif not args.recipename and not args.all:
+        raise argparse_oe.ArgumentUsageError('If you don\'t specify a recipe, you must specify -a/--all', 'undeploy-target')
+
     extraoptions = ''
     if args.no_host_check:
         extraoptions += '-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no'

@@ -157,19 +183,10 @@ def undeploy(args, config, basepath, workspace):
 
     args.target = args.target.split(':')[0]
 
-    if args.dry_run:
-        listfile = os.path.join(deploylist_path, '%s.list' % args.recipename)
-        print('Previously deployed files to be un-deployed for %s on target %s:' % (args.recipename, args.target))
-        ret = subprocess.call('ssh %s %s \'[ -f %s ] && cat %s || true\'' % (extraoptions, args.target, listfile, listfile), shell=True)
-        if ret != 0:
-            raise DevtoolError('Undeploy failed - rerun with -s to get a complete '
-                               'error message')
-        return 0
-
     tmpdir = tempfile.mkdtemp(prefix='devtool')
     try:
         tmpscript = '/tmp/devtool_undeploy.sh'
-        shellscript = _prepare_remote_script(deploy=False)
+        shellscript = _prepare_remote_script(deploy=False, dryrun=args.dry_run, undeployall=args.all)
         # Write out the script to a file
         with open(os.path.join(tmpdir, os.path.basename(tmpscript)), 'w') as f:
             f.write(shellscript)

@@ -187,7 +204,8 @@ def undeploy(args, config, basepath, workspace):
         raise DevtoolError('Undeploy failed - rerun with -s to get a complete '
                            'error message')
 
-    logger.info('Successfully undeployed %s' % args.recipename)
+    if not args.all and not args.dry_run:
+        logger.info('Successfully undeployed %s' % args.recipename)
     return 0
 
 

@@ -208,9 +226,10 @@ def register_commands(subparsers, context):
                                             help='Undeploy recipe output files in live target machine',
                                             description='Un-deploys recipe output files previously deployed to a live target machine by devtool deploy-target.',
                                             group='testbuild')
-    parser_undeploy.add_argument('recipename', help='Recipe to undeploy')
+    parser_undeploy.add_argument('recipename', help='Recipe to undeploy (if not using -a/--all)', nargs='?')
     parser_undeploy.add_argument('target', help='Live target machine running an ssh server: user@hostname')
     parser_undeploy.add_argument('-c', '--no-host-check', help='Disable ssh host key checking', action='store_true')
     parser_undeploy.add_argument('-s', '--show-status', help='Show progress/status output', action='store_true')
+    parser_undeploy.add_argument('-a', '--all', help='Undeploy all recipes deployed on the target', action='store_true')
     parser_undeploy.add_argument('-n', '--dry-run', help='List files to be undeployed only', action='store_true')
     parser_undeploy.set_defaults(func=undeploy)

index d54f6ba..66644cc 100644
@@ -28,7 +28,7 @@ logger = logging.getLogger('devtool')
 
 deploylist_path = '/.devtool'
 
-def _prepare_remote_script(deploy, verbose=False, dryrun=False, undeployall=False):
+def _prepare_remote_script(deploy, verbose=False, dryrun=False, undeployall=False, nopreserve=False, nocheckspace=False):
     """
     Prepare a shell script for running on the target to
     deploy/undeploy files. We have to be careful what we put in this

@@ -48,6 +48,7 @@ def _prepare_remote_script(deploy, verbose=False, dryrun=False, undeployall=Fals
         if not deploy:
             lines.append('echo "Previously deployed files for $1:"')
     lines.append('manifest="%s/$1.list"' % deploylist_path)
+    lines.append('preservedir="%s/$1.preserve"' % deploylist_path)
     lines.append('if [ -f $manifest ] ; then')
     # Read manifest in reverse and delete files / remove empty dirs
     lines.append('    sed \'1!G;h;$!d\' $manifest | while read file')

@@ -58,7 +59,10 @@ def _prepare_remote_script(deploy, verbose=False, dryrun=False, undeployall=Fals
         lines.append('        fi')
     else:
         lines.append('        if [ -d $file ] ; then')
-        lines.append('            rmdir $file > /dev/null 2>&1 || true')
+        # Avoid deleting a preserved directory in case it has special perms
+        lines.append('            if [ ! -d $preservedir/$file ] ; then')
+        lines.append('                rmdir $file > /dev/null 2>&1 || true')
+        lines.append('            fi')
         lines.append('        else')
         lines.append('            rm $file')
         lines.append('        fi')

@@ -71,6 +75,39 @@ def _prepare_remote_script(deploy, verbose=False, dryrun=False, undeployall=Fals
     lines.append('fi')
 
     if deploy:
+        if not nocheckspace:
+            # Check for available space
+            # FIXME This doesn't take into account files spread across multiple
+            # partitions, but doing that is non-trivial
+            # Find the part of the destination path that exists
+            lines.append('checkpath="$2"')
+            lines.append('while [ "$checkpath" != "/" ] && [ ! -e $checkpath ]')
+            lines.append('do')
+            lines.append('    checkpath=`dirname "$checkpath"`')
+            lines.append('done')
+            lines.append('freespace=`df -P $checkpath | sed "1d" | awk \'{ print $4 }\'`')
+            # First line of the file is the total space
+            lines.append('total=`head -n1 $3`')
+            lines.append('if [ $total -gt $freespace ] ; then')
+            lines.append('    echo "ERROR: insufficient space on target (available ${freespace}, needed ${total})"')
+            lines.append('    exit 1')
+            lines.append('fi')
+        if not nopreserve:
+            # Preserve any files that exist. Note that this will add to the
+            # preserved list with successive deployments if the list of files
+            # deployed changes, but because we've deleted any previously
+            # deployed files at this point it will never preserve anything
+            # that was deployed, only files that existed prior to any deploying
+            # (which makes the most sense)
+            lines.append('cat $3 | sed "1d" | while read file fsize')
+            lines.append('do')
+            lines.append('    if [ -e $file ] ; then')
+            lines.append('    dest="$preservedir/$file"')
+            lines.append('    mkdir -p `dirname $dest`')
+            lines.append('    mv $file $dest')
+            lines.append('    fi')
+            lines.append('done')
+            lines.append('rm $3')
         lines.append('mkdir -p `dirname $manifest`')
         lines.append('mkdir -p $2')
         if verbose:

@@ -78,6 +115,14 @@ def _prepare_remote_script(deploy, verbose=False, dryrun=False, undeployall=Fals
         else:
             lines.append('    tar xv -C $2 -f - > $manifest')
         lines.append('sed -i "s!^./!$2!" $manifest')
+    elif not dryrun:
+        # Put any preserved files back
+        lines.append('if [ -d $preservedir ] ; then')
+        lines.append('    cd $preservedir')
+        lines.append('    find . -type f -exec mv {} /{} \;')
+        lines.append('    cd /')
+        lines.append('    rm -rf $preservedir')
+        lines.append('fi')
 
     if undeployall:
         if not dryrun:

@@ -94,6 +139,7 @@ def _prepare_remote_script(deploy, verbose=False, dryrun=False, undeployall=Fals
 def deploy(args, config, basepath, workspace):
     """Entry point for the devtool 'deploy' subcommand"""
     import re
+    import math
     import oe.recipeutils
 
     check_workspace_recipe(workspace, args.recipename, checksrc=False)

@@ -119,11 +165,23 @@ def deploy(args, config, basepath, workspace):
                            'recipe? If so, the install step has not installed '
                            'any files.' % args.recipename)
 
+    filelist = []
+    ftotalsize = 0
+    for root, _, files in os.walk(recipe_outdir):
+        for fn in files:
+            # Get the size in kiB (since we'll be comparing it to the output of du -k)
+            # MUST use lstat() here not stat() or getfilesize() since we don't want to
+            # dereference symlinks
+            fsize = int(math.ceil(float(os.lstat(os.path.join(root, fn)).st_size)/1024))
+            ftotalsize += fsize
+            # The path as it would appear on the target
+            fpath = os.path.join(destdir, os.path.relpath(root, recipe_outdir), fn)
+            filelist.append((fpath, fsize))
+
     if args.dry_run:
         print('Files to be deployed for %s on target %s:' % (args.recipename, args.target))
-        for root, _, files in os.walk(recipe_outdir):
-            for fn in files:
-                print('  %s' % os.path.join(destdir, os.path.relpath(root, recipe_outdir), fn))
+        for item, _ in filelist:
+            print('  %s' % item)
         return 0
 
 

@@ -140,11 +198,20 @@ def deploy(args, config, basepath, workspace):
     tmpdir = tempfile.mkdtemp(prefix='devtool')
     try:
         tmpscript = '/tmp/devtool_deploy.sh'
-        shellscript = _prepare_remote_script(deploy=True, verbose=args.show_status)
+        tmpfilelist = os.path.join(os.path.dirname(tmpscript), 'devtool_deploy.list')
+        shellscript = _prepare_remote_script(deploy=True,
+                                             verbose=args.show_status,
+                                             nopreserve=args.no_preserve,
+                                             nocheckspace=args.no_check_space)
         # Write out the script to a file
         with open(os.path.join(tmpdir, os.path.basename(tmpscript)), 'w') as f:
             f.write(shellscript)
-        # Copy it to the target
+        # Write out the file list
+        with open(os.path.join(tmpdir, os.path.basename(tmpfilelist)), 'w') as f:
+            f.write('%d\n' % ftotalsize)
+            for fpath, fsize in filelist:
+                f.write('%s %d\n' % (fpath, fsize))
+        # Copy them to the target
         ret = subprocess.call("scp %s %s/* %s:%s" % (extraoptions, tmpdir, args.target, os.path.dirname(tmpscript)), shell=True)
         if ret != 0:
             raise DevtoolError('Failed to copy script to %s - rerun with -s to '

@@ -153,7 +220,7 @@ def deploy(args, config, basepath, workspace):
         shutil.rmtree(tmpdir)
 
     # Now run the script
-    ret = exec_fakeroot(rd, 'tar cf - . | ssh %s %s \'sh %s %s %s\'' % (extraoptions, args.target, tmpscript, args.recipename, destdir), cwd=recipe_outdir, shell=True)
+    ret = exec_fakeroot(rd, 'tar cf - . | ssh %s %s \'sh %s %s %s %s\'' % (extraoptions, args.target, tmpscript, args.recipename, destdir, tmpfilelist), cwd=recipe_outdir, shell=True)
     if ret != 0:
         raise DevtoolError('Deploy failed - rerun with -s to get a complete '
                            'error message')

@@ -213,13 +280,15 @@ def register_commands(subparsers, context):
     """Register devtool subcommands from the deploy plugin"""
     parser_deploy = subparsers.add_parser('deploy-target',
                                           help='Deploy recipe output files to live target machine',
-                                          description='Deploys a recipe\'s build output (i.e. the output of the do_install task) to a live target machine over ssh. Note: this only deploys the recipe itself and not any runtime dependencies, so it is assumed that those have been installed on the target beforehand.',
+                                          description='Deploys a recipe\'s build output (i.e. the output of the do_install task) to a live target machine over ssh. By default, any existing files will be preserved instead of being overwritten and will be restored if you run devtool undeploy-target. Note: this only deploys the recipe itself and not any runtime dependencies, so it is assumed that those have been installed on the target beforehand.',
                                           group='testbuild')
     parser_deploy.add_argument('recipename', help='Recipe to deploy')
     parser_deploy.add_argument('target', help='Live target machine running an ssh server: user@hostname[:destdir]')
     parser_deploy.add_argument('-c', '--no-host-check', help='Disable ssh host key checking', action='store_true')
     parser_deploy.add_argument('-s', '--show-status', help='Show progress/status output', action='store_true')
     parser_deploy.add_argument('-n', '--dry-run', help='List files to be deployed only', action='store_true')
+    parser_deploy.add_argument('-p', '--no-preserve', help='Do not preserve existing files', action='store_true')
+    parser_deploy.add_argument('--no-check-space', help='Do not check for available space before deploying', action='store_true')
     parser_deploy.set_defaults(func=deploy)
 
     parser_undeploy = subparsers.add_parser('undeploy-target',

index 084039a..daf18c9 100644
@@ -1320,7 +1320,7 @@ def register_commands(subparsers, context):
     parser_add.set_defaults(func=add)
 
     parser_modify = subparsers.add_parser('modify', help='Modify the source for an existing recipe',
-                                       description='Enables modifying the source for an existing recipe. You can either provide your own pre-prepared source tree, or specify -x/--extract to extract the source being fetched by the recipe.',
+                                       description='Sets up the build environment to modify the source for an existing recipe. The default behaviour is to extract the source being fetched by the recipe into a git tree so you can work on it; alternatively if you already have your own pre-prepared source tree you can specify -n/--no-extract.',
                                        group='starting', order=90)
     parser_modify.add_argument('recipename', help='Name of existing recipe to edit (just name - no version, path or extension)')
     parser_modify.add_argument('srctree', nargs='?', help='Path to external source tree. If not specified, a subdirectory of %s will be used.' % defsrctree)

@@ -1331,7 +1331,7 @@ def register_commands(subparsers, context):
     group = parser_modify.add_mutually_exclusive_group()
     group.add_argument('--same-dir', '-s', help='Build in same directory as source', action="store_true")
     group.add_argument('--no-same-dir', help='Force build in a separate build directory', action="store_true")
-    parser_modify.add_argument('--branch', '-b', default="devtool", help='Name for development branch to checkout (only when using -x) (default "%(default)s")')
+    parser_modify.add_argument('--branch', '-b', default="devtool", help='Name for development branch to checkout (when not using -n/--no-extract) (default "%(default)s")')
     parser_modify.set_defaults(func=modify)
 
     parser_extract = subparsers.add_parser('extract', help='Extract the source for an existing recipe',

index 4d11e04..1a06cac 100644
@@ -22,6 +22,12 @@ from recipetool.create import RecipeHandler, validate_pv
 logger = logging.getLogger('recipetool')
 
 tinfoil = None
+plugins = None
+
+def plugin_init(pluginlist):
+    # Take a reference to the list so we can use it later
+    global plugins
+    plugins = pluginlist
 
 def tinfoil_init(instance):
     global tinfoil

@@ -48,6 +54,13 @@ class CmakeRecipeHandler(RecipeHandler):
 
     @staticmethod
     def extract_cmake_deps(outlines, srctree, extravalues, cmakelistsfile=None):
+        # Find all plugins that want to register handlers
+        logger.debug('Loading cmake handlers')
+        handlers = []
+        for plugin in plugins:
+            if hasattr(plugin, 'register_cmake_handlers'):
+                plugin.register_cmake_handlers(handlers)
+
         values = {}
         inherits = []
 

@@ -152,6 +165,9 @@ class CmakeRecipeHandler(RecipeHandler):
             with open(fn, 'r') as f:
                 for line in f:
                     line = line.strip()
+                    for handler in handlers:
+                        if handler.process_line(srctree, fn, line, libdeps, pcdeps, deps, outlines, inherits, values):
+                            continue
                     res = include_re.match(line)
                     if res:
                         includefn = bb.utils.which(':'.join(searchpaths), res.group(1))

@@ -194,7 +210,15 @@ class CmakeRecipeHandler(RecipeHandler):
                     if res:
                         origpkg = res.group(1)
                         pkg = interpret_value(origpkg.lower())
-                        if pkg == 'gettext':
+                        found = False
+                        for handler in handlers:
+                            if handler.process_findpackage(srctree, fn, pkg, deps, outlines, inherits, values):
+                                logger.debug('Mapped CMake package %s via handler %s' % (pkg, handler.__class__.__name__))
+                                found = True
+                                break
+                        if found:
+                            continue
+                        elif pkg == 'gettext':
                             inherits.append('gettext')
                         elif pkg == 'perl':
                             inherits.append('perlnative')

@@ -207,6 +231,7 @@ class CmakeRecipeHandler(RecipeHandler):
                         else:
                             dep = cmake_pkgmap.get(pkg, None)
                             if dep:
+                                logger.debug('Mapped CMake package %s to recipe %s via internal list' % (pkg, dep))
                                 deps.append(dep)
                             elif dep is None:
                                 unmappedpkgs.append(origpkg)

@@ -236,11 +261,39 @@ class CmakeRecipeHandler(RecipeHandler):
 
         RecipeHandler.handle_depends(libdeps, pcdeps, deps, outlines, values, tinfoil.config_data)
 
+        for handler in handlers:
+            handler.post_process(srctree, libdeps, pcdeps, deps, outlines, inherits, values)
+
         if inherits:
             values['inherit'] = ' '.join(list(set(inherits)))
 
         return values
 
+
+class CmakeExtensionHandler(object):
+    '''Base class for CMake extension handlers'''
+    def process_line(self, srctree, fn, line, libdeps, pcdeps, deps, outlines, inherits, values):
+        '''
+        Handle a line parsed out of an CMake file.
+        Return True if you've completely handled the passed in line, otherwise return False.
+        '''
+        return False
+
+    def process_findpackage(self, srctree, fn, pkg, deps, outlines, inherits, values):
+        '''
+        Handle a find_package package parsed out of a CMake file.
+        Return True if you've completely handled the passed in package, otherwise return False.
+        '''
+        return False
+
+    def post_process(self, srctree, fn, pkg, deps, outlines, inherits, values):
+        '''
+        Apply any desired post-processing on the output
+        '''
+        return
+
+
+
 class SconsRecipeHandler(RecipeHandler):
     def process(self, srctree, classes, lines_before, lines_after, handled, extravalues):
         if 'buildsystem' in handled:

@@ -255,6 +308,7 @@ class SconsRecipeHandler(RecipeHandler):
             return True
         return False
 
+
 class QmakeRecipeHandler(RecipeHandler):
     def process(self, srctree, classes, lines_before, lines_after, handled, extravalues):
         if 'buildsystem' in handled:

@@ -266,6 +320,7 @@ class QmakeRecipeHandler(RecipeHandler):
             return True
         return False
 
+
 class AutotoolsRecipeHandler(RecipeHandler):
     def process(self, srctree, classes, lines_before, lines_after, handled, extravalues):
         if 'buildsystem' in handled:

@@ -322,6 +377,13 @@ class AutotoolsRecipeHandler(RecipeHandler):
     def extract_autotools_deps(outlines, srctree, extravalues=None, acfile=None):
         import shlex
 
+        # Find all plugins that want to register handlers
+        logger.debug('Loading autotools handlers')
+        handlers = []
+        for plugin in plugins:
+            if hasattr(plugin, 'register_autotools_handlers'):
+                plugin.register_autotools_handlers(handlers)
+
         values = {}
         inherits = []
 

@@ -384,6 +446,9 @@ class AutotoolsRecipeHandler(RecipeHandler):
         unmapped = []
 
         def process_macro(keyword, value):
+            for handler in handlers:
+                if handler.process_macro(srctree, keyword, value, process_value, libdeps, pcdeps, deps, outlines, inherits, values):
+                    return
             if keyword == 'PKG_CHECK_MODULES':
                 res = pkg_re.search(value)
                 if res:

@@ -409,6 +474,9 @@ class AutotoolsRecipeHandler(RecipeHandler):
                 if res:
                     for prog in shlex.split(res.group(1)):
                         prog = prog.split()[0]
+                        for handler in handlers:
+                            if handler.process_prog(srctree, keyword, value, prog, deps, outlines, inherits, values):
+                                return
                         progclass = progclassmap.get(prog, None)
                         if progclass:
                             inherits.append(progclass)

@@ -537,6 +605,10 @@ class AutotoolsRecipeHandler(RecipeHandler):
                     'AM_INIT_AUTOMAKE',
                     'define(',
                     ]
+
+        for handler in handlers:
+            handler.extend_keywords(keywords)
+
         for srcfile in srcfiles:
             nesting = 0
             in_keyword = ''

@@ -581,12 +653,44 @@ class AutotoolsRecipeHandler(RecipeHandler):
 
         RecipeHandler.handle_depends(libdeps, pcdeps, deps, outlines, values, tinfoil.config_data)
 
+        for handler in handlers:
+            handler.post_process(srctree, libdeps, pcdeps, deps, outlines, inherits, values)
+
         if inherits:
             values['inherit'] = ' '.join(list(set(inherits)))
 
         return values
 
 
+class AutotoolsExtensionHandler(object):
+    '''Base class for Autotools extension handlers'''
+    def process_macro(self, srctree, keyword, value, process_value, libdeps, pcdeps, deps, outlines, inherits, values):
+        '''
+        Handle a macro parsed out of an autotools file. Note that if you want this to be called
+        for any macro other than the ones AutotoolsRecipeHandler already looks for, you'll need
+        to add it to the keywords list in extend_keywords().
+        Return True if you've completely handled the passed in macro, otherwise return False.
+        '''
+        return False
+
+    def extend_keywords(self, keywords):
+        '''Adds keywords to be recognised by the parser (so that you get a call to process_macro)'''
+        return
+
+    def process_prog(self, srctree, keyword, value, prog, deps, outlines, inherits, values):
+        '''
+        Handle an AC_PATH_PROG, AC_CHECK_PROG etc. line
+        Return True if you've completely handled the passed in macro, otherwise return False.
+        '''
+        return False
+
+    def post_process(self, srctree, fn, pkg, deps, outlines, inherits, values):
+        '''
+        Apply any desired post-processing on the output
+        '''
+        return
+
+
 class MakefileRecipeHandler(RecipeHandler):
     def process(self, srctree, classes, lines_before, lines_after, handled, extravalues):
         if 'buildsystem' in handled:

@@ -703,11 +807,12 @@ class SpecFileRecipeHandler(RecipeHandler):
                 break
 
 def register_recipe_handlers(handlers):
-    # These are in a specific order so that the right one is detected first
-    handlers.append(CmakeRecipeHandler())
-    handlers.append(AutotoolsRecipeHandler())
-    handlers.append(SconsRecipeHandler())
-    handlers.append(QmakeRecipeHandler())
-    handlers.append(MakefileRecipeHandler())
+    # Set priorities with some gaps so that other plugins can insert
+    # their own handlers (so avoid changing these numbers)
+    handlers.append((CmakeRecipeHandler(), 50))
+    handlers.append((AutotoolsRecipeHandler(), 40))
+    handlers.append((SconsRecipeHandler(), 30))
+    handlers.append((QmakeRecipeHandler(), 20))
+    handlers.append((MakefileRecipeHandler(), 10))
     handlers.append((VersionFileRecipeHandler(), -1))
     handlers.append((SpecFileRecipeHandler(), -1))

index 4d65c96..c382330 100644
@@ -716,4 +716,4 @@ def has_non_literals(value):
 
 def register_recipe_handlers(handlers):
     # We need to make sure this is ahead of the makefile fallback handler
-    handlers.insert(0, PythonRecipeHandler())
+    handlers.append((PythonRecipeHandler(), 70))

index 3e4bab8..7560cdf 100644
@@ -43,6 +43,7 @@ def tinfoil_init(instance):
 class RecipeHandler(object):
     recipelibmap = {}
     recipeheadermap = {}
+    recipecmakefilemap = {}
 
     @staticmethod
     def load_libmap(d):

@@ -90,15 +91,18 @@ class RecipeHandler(object):
         RecipeHandler.recipelibmap['GLESv2'] = 'virtual/libgles2'
 
     @staticmethod
-    def load_headermap(d):
-        '''Build up lib headerfile->recipe mapping'''
+    def load_devel_filemap(d):
+        '''Build up development file->recipe mapping'''
         if RecipeHandler.recipeheadermap:
             return
+        pkgdata_dir = d.getVar('PKGDATA_DIR', True)
         includedir = d.getVar('includedir', True)
+        cmakedir = os.path.join(d.getVar('libdir', True), 'cmake')
         for pkg in glob.glob(os.path.join(pkgdata_dir, 'runtime', '*-dev')):
             with open(os.path.join(pkgdata_dir, 'runtime', pkg)) as f:
                 pn = None
                 headers = []
+                cmakefiles = []
                 for line in f:
                     if line.startswith('PN:'):
                         pn = line.split(':', 1)[-1].strip()

@@ -108,9 +112,14 @@ class RecipeHandler(object):
                         for fullpth in sorted(dictval):
                             if fullpth.startswith(includedir) and fullpth.endswith('.h'):
                                 headers.append(os.path.relpath(fullpth, includedir))
+                            elif fullpth.startswith(cmakedir) and fullpth.endswith('.cmake'):
+                                cmakefiles.append(os.path.relpath(fullpth, cmakedir))
                 if pn and headers:
                     for header in headers:
                         RecipeHandler.recipeheadermap[header] = pn
+                if pn and cmakefiles:
+                    for fn in cmakefiles:
+                        RecipeHandler.recipecmakefilemap[fn] = pn
 
     @staticmethod
     def checkfiles(path, speclist, recursive=False):

@@ -172,7 +181,7 @@ class RecipeHandler(object):
                 deps.append(recipe)
             elif recipe is None:
                 if header:
-                    RecipeHandler.load_headermap(d)
+                    RecipeHandler.load_devel_filemap(d)
                     recipe = RecipeHandler.recipeheadermap.get(header, None)
                     if recipe:
                         deps.append(recipe)

index 1a06cac..43dcca3 100644
@@ -17,6 +17,7 @@
 
 import re
 import logging
+import glob
 from recipetool.create import RecipeHandler, validate_pv
 
 logger = logging.getLogger('recipetool')

@@ -156,6 +157,16 @@ class CmakeRecipeHandler(RecipeHandler):
         subdir_re = re.compile('add_subdirectory\s*\(\s*([^)\s]*)\s*([^)\s]*)\s*\)', re.IGNORECASE)
         dep_re = re.compile('([^ ><=]+)( *[<>=]+ *[^ ><=]+)?')
 
+        def find_cmake_package(pkg):
+            RecipeHandler.load_devel_filemap(tinfoil.config_data)
+            for fn, pn in RecipeHandler.recipecmakefilemap.iteritems():
+                splitname = fn.split('/')
+                if len(splitname) > 1:
+                    if splitname[0].lower().startswith(pkg.lower()):
+                        if splitname[1] == '%s-config.cmake' % pkg.lower() or splitname[1] == '%sConfig.cmake' % pkg or splitname[1] == 'Find%s.cmake' % pkg:
+                            return pn
+            return None
+
         def interpret_value(value):
             return value.strip('"')
 

@@ -209,7 +220,7 @@ class CmakeRecipeHandler(RecipeHandler):
                     res = findpackage_re.match(line)
                     if res:
                         origpkg = res.group(1)
-                        pkg = interpret_value(origpkg.lower())
+                        pkg = interpret_value(origpkg)
                         found = False
                         for handler in handlers:
                             if handler.process_findpackage(srctree, fn, pkg, deps, outlines, inherits, values):

@@ -218,23 +229,29 @@ class CmakeRecipeHandler(RecipeHandler):
                                 break
                         if found:
                             continue
-                        elif pkg == 'gettext':
+                        elif pkg == 'Gettext':
                             inherits.append('gettext')
-                        elif pkg == 'perl':
+                        elif pkg == 'Perl':
                             inherits.append('perlnative')
-                        elif pkg == 'pkgconfig':
+                        elif pkg == 'PkgConfig':
                             inherits.append('pkgconfig')
-                        elif pkg == 'pythoninterp':
+                        elif pkg == 'PythonInterp':
                             inherits.append('pythonnative')
-                        elif pkg == 'pythonlibs':
+                        elif pkg == 'PythonLibs':
                             inherits.append('python-dir')
                         else:
-                            dep = cmake_pkgmap.get(pkg, None)
+                            # Try to map via looking at installed CMake packages in pkgdata
+                            dep = find_cmake_package(pkg)
                             if dep:
-                                logger.debug('Mapped CMake package %s to recipe %s via internal list' % (pkg, dep))
+                                logger.debug('Mapped CMake package %s to recipe %s via pkgdata' % (pkg, dep))
                                 deps.append(dep)
-                            elif dep is None:
-                                unmappedpkgs.append(origpkg)
+                            else:
+                                dep = cmake_pkgmap.get(pkg.lower(), None)
+                                if dep:
+                                    logger.debug('Mapped CMake package %s to recipe %s via internal list' % (pkg, dep))
+                                    deps.append(dep)
+                                elif dep is None:
+                                    unmappedpkgs.append(origpkg)
                         continue
                     res = checklib_re.match(line)
                     if res:

@@ -257,7 +274,7 @@ class CmakeRecipeHandler(RecipeHandler):
         parse_cmake_file(srcfiles[0])
 
         if unmappedpkgs:
-            outlines.append('# NOTE: unable to map the following CMake package dependencies: %s' % ' '.join(unmappedpkgs))
+            outlines.append('# NOTE: unable to map the following CMake package dependencies: %s' % ' '.join(list(set(unmappedpkgs))))
 
         RecipeHandler.handle_depends(libdeps, pcdeps, deps, outlines, values, tinfoil.config_data)