devtool/recipetool improvements

index 4b35fc9..8cf92ca 100755
@@ -35,6 +35,7 @@ import fnmatch
 scripts_lib_path = os.path.abspath(os.path.join(os.path.dirname(os.path.realpath(__file__)), '..', 'lib'))
 sys.path.insert(0, scripts_lib_path)
 import scriptutils
+import argparse_oe
 logger = scriptutils.logger_create('devtool-stress')
 
 def select_recipes(args):

@@ -204,8 +205,8 @@ def stress_modify(args):
 
 
 def main():
-    parser = argparse.ArgumentParser(description="devtool stress tester",
-                                     epilog="Use %(prog)s <subcommand> --help to get help on a specific command")
+    parser = argparse_oe.ArgumentParser(description="devtool stress tester",
+                                        epilog="Use %(prog)s <subcommand> --help to get help on a specific command")
     parser.add_argument('-d', '--debug', help='Enable debug output', action='store_true')
     parser.add_argument('-r', '--resume-from', help='Resume from specified recipe', metavar='PN')
     parser.add_argument('-o', '--only', help='Only test specified recipes (comma-separated without spaces, wildcards allowed)', metavar='PNLIST')

index 9d3287c..955495e 100755
@@ -37,6 +37,7 @@ lib_path = scripts_path + '/lib'
 sys.path = sys.path + [lib_path]
 from devtool import DevtoolError, setup_tinfoil
 import scriptutils
+import argparse_oe
 logger = scriptutils.logger_create('devtool')
 
 plugins = []

@@ -185,9 +186,9 @@ def main():
             break
         pth = os.path.dirname(pth)
 
-    parser = argparse.ArgumentParser(description="OpenEmbedded development tool",
-                                     add_help=False,
-                                     epilog="Use %(prog)s <subcommand> --help to get help on a specific command")
+    parser = argparse_oe.ArgumentParser(description="OpenEmbedded development tool",
+                                        add_help=False,
+                                        epilog="Use %(prog)s <subcommand> --help to get help on a specific command")
     parser.add_argument('--basepath', help='Base directory of SDK / build directory')
     parser.add_argument('--bbpath', help='Explicitly specify the BBPATH, rather than getting it from the metadata')
     parser.add_argument('-d', '--debug', help='Enable debug output', action='store_true')

new file mode 100644
index 0000000..c2fee6d
@@ -0,0 +1,11 @@
+import sys
+import argparse
+
+class ArgumentParser(argparse.ArgumentParser):
+    """Our own version of argparse's ArgumentParser"""
+
+    def error(self, message):
+        sys.stderr.write('ERROR: %s\n' % message)
+        self.print_help()
+        sys.exit(2)
+

index 3366882..4dd7ef2 100644
@@ -19,6 +19,7 @@ import sys
 import os
 import logging
 import glob
+import argparse
 
 def logger_create(name):
     logger = logging.getLogger(name)

index afdceaa..8e22e02 100755
@@ -33,6 +33,7 @@ scripts_path = os.path.dirname(os.path.realpath(__file__))
 lib_path = scripts_path + '/lib'
 sys.path = sys.path + [lib_path]
 import scriptutils
+import argparse_oe
 logger = scriptutils.logger_create('pkgdatautil')
 
 def tinfoil_init():

@@ -417,8 +418,8 @@ def find_path(args):
 
 
 def main():
-    parser = argparse.ArgumentParser(description="OpenEmbedded pkgdata tool - queries the pkgdata files written out during do_package",
-                                     epilog="Use %(prog)s <subcommand> --help to get help on a specific command")
+    parser = argparse_oe.ArgumentParser(description="OpenEmbedded pkgdata tool - queries the pkgdata files written out during do_package",
+                                        epilog="Use %(prog)s <subcommand> --help to get help on a specific command")
     parser.add_argument('-d', '--debug', help='Enable debug output', action='store_true')
     parser.add_argument('-p', '--pkgdata-dir', help='Path to pkgdata directory (determined automatically if not specified)')
     subparsers = parser.add_subparsers(title='subcommands', metavar='<subcommand>')

index c4c35bd..c8c79c2 100755
@@ -24,6 +24,7 @@ scripts_path = os.path.dirname(os.path.realpath(__file__))
 lib_path = scripts_path + '/lib'
 sys.path = sys.path + [lib_path]
 import scriptutils
+import argparse_oe
 logger = scriptutils.logger_create('sdktool')
 
 def mkdir(d):

@@ -113,8 +114,8 @@ def publish(args):
 
 
 def main():
-    parser = argparse.ArgumentParser(description="OpenEmbedded development tool",
-                                     epilog="Use %(prog)s <subcommand> --help to get help on a specific command")
+    parser = argparse_oe.ArgumentParser(description="OpenEmbedded development tool",
+                                        epilog="Use %(prog)s <subcommand> --help to get help on a specific command")
     parser.add_argument('-d', '--debug', help='Enable debug output', action='store_true')
     parser.add_argument('-q', '--quiet', help='Print only errors', action='store_true')
 

index bc50b2a..f989e87 100755
@@ -36,6 +36,7 @@ sys.path.insert(0, os.path.dirname(os.path.realpath(__file__)) + '/lib')
 import scriptpath
 scriptpath.add_bitbake_lib_path()
 scriptpath.add_oe_lib_path()
+import argparse_oe
 
 import oeqa.selftest
 import oeqa.utils.ftools as ftools

@@ -65,7 +66,7 @@ log = logger_create()
 
 def get_args_parser():
     description = "Script that runs unit tests agains bitbake and other Yocto related tools. The goal is to validate tools functionality and metadata integrity. Refer to https://wiki.yoctoproject.org/wiki/Oe-selftest for more information."
-    parser = argparse.ArgumentParser(description=description)
+    parser = argparse_oe.ArgumentParser(description=description)
     group = parser.add_mutually_exclusive_group(required=True)
     group.add_argument('--run-tests', required=False, action='store', nargs='*', dest="run_tests", default=None, help='Select what tests to run (modules, classes or test methods). Format should be: <module>.<class>.<test_method>')
     group.add_argument('--run-all-tests', required=False, action="store_true", dest="run_all_tests", default=False, help='Run all (unhidden) tests')

index 791a66a..1198cc2 100755
@@ -27,6 +27,7 @@ scripts_path = os.path.dirname(os.path.realpath(__file__))
 lib_path = scripts_path + '/lib'
 sys.path = sys.path + [lib_path]
 import scriptutils
+import argparse_oe
 logger = scriptutils.logger_create('recipetool')
 
 plugins = []

@@ -45,9 +46,9 @@ def main():
         logger.error("This script can only be run after initialising the build environment (e.g. by using oe-init-build-env)")
         sys.exit(1)
 
-    parser = argparse.ArgumentParser(description="OpenEmbedded recipe tool",
-                                     add_help=False,
-                                     epilog="Use %(prog)s <subcommand> --help to get help on a specific command")
+    parser = argparse_oe.ArgumentParser(description="OpenEmbedded recipe tool",
+                                        add_help=False,
+                                        epilog="Use %(prog)s <subcommand> --help to get help on a specific command")
     parser.add_argument('-d', '--debug', help='Enable debug output', action='store_true')
     parser.add_argument('-q', '--quiet', help='Print only errors', action='store_true')
     parser.add_argument('--color', choices=['auto', 'always', 'never'], default='auto', help='Colorize output (where %(metavar)s is %(choices)s)', metavar='COLOR')

index 1a1b965..a29feff 100755
@@ -15,6 +15,10 @@ import subprocess
 import argparse
 import logging
 
+scripts_lib_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'lib')
+sys.path.insert(0, scripts_lib_path)
+import argparse_oe
+
 version = "0.3"
 
 log = logging.getLogger("send-error-report")

@@ -143,7 +147,7 @@ def send_data(data, args):
 
 
 if __name__ == '__main__':
-    arg_parse = argparse.ArgumentParser(description="This scripts will send an error report to your specified error-report-web server.")
+    arg_parse = argparse_oe.ArgumentParser(description="This scripts will send an error report to your specified error-report-web server.")
 
     arg_parse.add_argument("error_file",
                            help="Generated error report file location",

index f3a44eb..97d03d7 100755
@@ -38,6 +38,7 @@ lib_path = scripts_path + '/lib'
 sys.path = sys.path + [lib_path]
 
 import scriptpath
+import argparse_oe
 
 # Add meta/lib to sys.path
 scriptpath.add_oe_lib_path()

@@ -82,7 +83,7 @@ log = logger_create()
 # Define and return the arguments parser for the script
 def get_args_parser():
     description = "This script is used to run automated runtime tests using remotely published image files. You should prepare the build environment just like building local images and running the tests."
-    parser = argparse.ArgumentParser(description=description)
+    parser = argparse_oe.ArgumentParser(description=description)
     parser.add_argument('--image-types', required=True, action="store", nargs='*', dest="image_types", default=None, help='The image types to test(ex: core-image-minimal).')
     parser.add_argument('--repo-link', required=True, action="store", type=str, dest="repo_link", default=None, help='The link to the remote images repository.')
     parser.add_argument('--required-packages', required=False, action="store", nargs='*', dest="required_packages", default=None, help='Required packages for the tests. They will be built before the testing begins.')

index 2d75046..7987fbb 100644
@@ -108,6 +108,11 @@ def create_recipe(args):
             # Assume the archive contains the directory structure verbatim
             # so we need to extract to a subdirectory
             fetchuri += ';subdir=%s' % os.path.splitext(os.path.basename(urlparse.urlsplit(fetchuri).path))[0]
+        git_re = re.compile('(https?)://([^;]+\.git)(;.*)?')
+        res = git_re.match(fetchuri)
+        if res:
+            # Need to switch the URI around so that the git fetcher is used
+            fetchuri = 'git://%s;protocol=%s%s' % (res.group(2), res.group(1), res.group(3) or '')
         srcuri = fetchuri
         rev_re = re.compile(';rev=([^;]+)')
         res = rev_re.search(srcuri)

index 7987fbb..f98f525 100644
@@ -122,7 +122,11 @@ def create_recipe(args):
         tempsrc = tempfile.mkdtemp(prefix='recipetool-')
         srctree = tempsrc
         logger.info('Fetching %s...' % srcuri)
-        checksums = scriptutils.fetch_uri(tinfoil.config_data, fetchuri, srctree, srcrev)
+        try:
+            checksums = scriptutils.fetch_uri(tinfoil.config_data, fetchuri, srctree, srcrev)
+        except bb.fetch2.FetchError:
+            # Error already printed
+            sys.exit(1)
         dirlist = os.listdir(srctree)
         if 'git.indirectionsymlink' in dirlist:
             dirlist.remove('git.indirectionsymlink')

index ed14a53..931ef3b 100644
@@ -279,7 +279,7 @@ class MakefileRecipeHandler(RecipeHandler):
 
             installtarget = True
             try:
-                stdout, stderr = bb.process.run('make -qn install', cwd=srctree, shell=True)
+                stdout, stderr = bb.process.run('make -n install', cwd=srctree, shell=True)
             except bb.process.ExecutionError as e:
                 if e.exitcode != 1:
                     installtarget = False

index e0af2a0..266c423 100644
@@ -278,7 +278,10 @@ class PythonRecipeHandler(RecipeHandler):
             for k in sorted(bbinfo):
                 v = bbinfo[k]
                 mdinfo.append('{} = "{}"'.format(k, v))
-            lines_before[src_uri_line-1:src_uri_line-1] = mdinfo
+            if src_uri_line:
+                lines_before[src_uri_line-1:src_uri_line-1] = mdinfo
+            else:
+                lines_before.extend(mdinfo)
 
         mapped_deps, unmapped_deps = self.scan_setup_python_deps(srctree, setup_info, setup_non_literals)
 

index 4dd7ef2..69e76d8 100644
@@ -86,6 +86,10 @@ def fetch_uri(d, uri, destdir, srcrev=None):
             ud = fetcher.ud[u]
             ud.ignore_checksums = True
         fetcher.download()
+        for u in fetcher.ud:
+            ud = fetcher.ud[u]
+            if ud.localpath.rstrip(os.sep) == localdata.getVar('DL_DIR', True).rstrip(os.sep):
+                raise Exception('Local path is download directory - please check that the URI "%s" is correct' % uri)
         fetcher.unpack(destdir)
         for u in fetcher.ud:
             ud = fetcher.ud[u]

index f98f525..3d5a373 100644
@@ -130,10 +130,17 @@ def create_recipe(args):
         dirlist = os.listdir(srctree)
         if 'git.indirectionsymlink' in dirlist:
             dirlist.remove('git.indirectionsymlink')
-        if len(dirlist) == 1 and os.path.isdir(os.path.join(srctree, dirlist[0])):
-            # We unpacked a single directory, so we should use that
-            srcsubdir = dirlist[0]
-            srctree = os.path.join(srctree, srcsubdir)
+        if len(dirlist) == 1:
+            singleitem = os.path.join(srctree, dirlist[0])
+            if os.path.isdir(singleitem):
+                # We unpacked a single directory, so we should use that
+                srcsubdir = dirlist[0]
+                srctree = os.path.join(srctree, srcsubdir)
+            else:
+                with open(singleitem, 'r') as f:
+                    if '<html' in f.read(100).lower():
+                        logger.error('Fetching "%s" returned a single HTML page - check the URL is correct and functional' % fetchuri)
+                        sys.exit(1)
     else:
         # Assume we're pointing to an existing source tree
         if args.extract_to:

index 3d5a373..5c249ab 100644
@@ -251,10 +251,22 @@ def create_recipe(args):
         lines_after.append('')
 
     # Find all plugins that want to register handlers
-    handlers = []
+    logger.debug('Loading recipe handlers')
+    raw_handlers = []
     for plugin in plugins:
         if hasattr(plugin, 'register_recipe_handlers'):
-            plugin.register_recipe_handlers(handlers)
+            plugin.register_recipe_handlers(raw_handlers)
+    # Sort handlers by priority
+    handlers = []
+    for i, handler in enumerate(raw_handlers):
+        if isinstance(handler, tuple):
+            handlers.append((handler[0], handler[1], i))
+        else:
+            handlers.append((handler, 0, i))
+    handlers.sort(key=lambda item: (item[1], -item[2]), reverse=True)
+    for handler, priority, _ in handlers:
+        logger.debug('Handler: %s (priority %d)' % (handler.__class__.__name__, priority))
+    handlers = [item[0] for item in handlers]
 
     # Apply the handlers
     classes = []

index 0a44ae7..8faea93 100644
@@ -279,7 +279,7 @@ class DevtoolTests(DevtoolBase):
         self.assertIn('_git.bb', recipefile, 'Recipe file incorrectly named')
         checkvars = {}
         checkvars['S'] = '${WORKDIR}/git'
-        checkvars['PV'] = '1.0+git${SRCPV}'
+        checkvars['PV'] = '1.11+git${SRCPV}'
         checkvars['SRC_URI'] = url
         checkvars['SRCREV'] = '${AUTOREV}'
         self._test_recipe_contents(recipefile, checkvars, [])

index b1f1d2ab..bbfce7c 100644
@@ -395,7 +395,7 @@ class RecipetoolTests(RecipetoolBase):
         checkvars['LICENSE'] = 'LGPLv2.1'
         checkvars['LIC_FILES_CHKSUM'] = 'file://COPYING;md5=7fbc338309ac38fefcd64b04bb903e34'
         checkvars['S'] = '${WORKDIR}/git'
-        checkvars['PV'] = '1.0+git${SRCPV}'
+        checkvars['PV'] = '1.11+git${SRCPV}'
         checkvars['SRC_URI'] = srcuri
         checkvars['DEPENDS'] = 'libpng pango libx11 libxext jpeg'
         inherits = ['autotools', 'pkgconfig']

189 lines not on this page · raw message

376 lines not on this page · raw message

index 266c423..4d65c96 100644
@@ -159,7 +159,7 @@ class PythonRecipeHandler(RecipeHandler):
     def __init__(self):
         pass
 
-    def process(self, srctree, classes, lines_before, lines_after, handled):
+    def process(self, srctree, classes, lines_before, lines_after, handled, extravalues):
         if 'buildsystem' in handled:
             return False
 

index 8faea93..7af82df 100644
@@ -232,15 +232,16 @@ class DevtoolTests(DevtoolBase):
         self.assertIn(srcdir, result.output)
         # Check recipe
         recipefile = get_bb_var('FILE', testrecipe)
-        self.assertIn('%s.bb' % testrecipe, recipefile, 'Recipe file incorrectly named')
+        self.assertIn('%s_%s.bb' % (testrecipe, testver), recipefile, 'Recipe file incorrectly named')
         checkvars = {}
-        checkvars['S'] = '${WORKDIR}/MarkupSafe-%s' % testver
-        checkvars['SRC_URI'] = url
+        checkvars['S'] = '${WORKDIR}/MarkupSafe-${PV}'
+        checkvars['SRC_URI'] = url.replace(testver, '${PV}')
         self._test_recipe_contents(recipefile, checkvars, [])
         # Try with version specified
         result = runCmd('devtool reset -n %s' % testrecipe)
         shutil.rmtree(srcdir)
-        result = runCmd('devtool add %s %s -f %s -V %s' % (testrecipe, srcdir, url, testver))
+        fakever = '1.9'
+        result = runCmd('devtool add %s %s -f %s -V %s' % (testrecipe, srcdir, url, fakever))
         self.assertTrue(os.path.isfile(os.path.join(srcdir, 'setup.py')), 'Unable to find setup.py in source directory')
         # Test devtool status
         result = runCmd('devtool status')

@@ -248,10 +249,10 @@ class DevtoolTests(DevtoolBase):
         self.assertIn(srcdir, result.output)
         # Check recipe
         recipefile = get_bb_var('FILE', testrecipe)
-        self.assertIn('%s_%s.bb' % (testrecipe, testver), recipefile, 'Recipe file incorrectly named')
+        self.assertIn('%s_%s.bb' % (testrecipe, fakever), recipefile, 'Recipe file incorrectly named')
         checkvars = {}
-        checkvars['S'] = '${WORKDIR}/MarkupSafe-${PV}'
-        checkvars['SRC_URI'] = url.replace(testver, '${PV}')
+        checkvars['S'] = '${WORKDIR}/MarkupSafe-%s' % testver
+        checkvars['SRC_URI'] = url
         self._test_recipe_contents(recipefile, checkvars, [])
 
     @testcase(1161)

index 6c7b9fd..4887604 100644
@@ -86,6 +86,27 @@ def validate_pv(pv):
         return False
     return True
 
+def determine_from_filename(srcfile):
+    """Determine name and version from a filename"""
+    part = ''
+    if '.tar.' in srcfile:
+        namepart = srcfile.split('.tar.')[0]
+    else:
+        namepart = os.path.splitext(srcfile)[0]
+    splitval = namepart.rsplit('_', 1)
+    if len(splitval) == 1:
+        splitval = namepart.rsplit('-', 1)
+    pn = splitval[0].replace('_', '-')
+    if len(splitval) > 1:
+        if splitval[1][0] in '0123456789':
+            pv = splitval[1]
+        else:
+            pn = '-'.join(splitval).replace('_', '-')
+            pv = None
+    else:
+        pv = None
+    return (pn, pv)
+
 def supports_srcrev(uri):
     localdata = bb.data.createCopy(tinfoil.config_data)
     # This is a bit sad, but if you don't have this set there can be some

@@ -234,6 +255,17 @@ def create_recipe(args):
     else:
         realpv = None
 
+    if srcuri and not realpv or not pn:
+        parseres = urlparse.urlparse(srcuri)
+        if parseres.path:
+            srcfile = os.path.basename(parseres.path)
+            name_pn, name_pv = determine_from_filename(srcfile)
+            logger.debug('Determined from filename: name = "%s", version = "%s"' % (name_pn, name_pv))
+            if name_pn and not pn:
+                pn = name_pn
+            if name_pv and not realpv:
+                realpv = name_pv
+
     if not srcuri:
         lines_before.append('# No information for SRC_URI yet (only an external source tree was specified)')
     lines_before.append('SRC_URI = "%s"' % srcuri)

index a5e81f3..c11de9d 100644
@@ -1140,6 +1140,44 @@ def reset(args, config, basepath, workspace):
     return 0
 
 
+def edit_recipe(args, config, basepath, workspace):
+    """Entry point for the devtool 'edit-recipe' subcommand"""
+    if args.any_recipe:
+        tinfoil = setup_tinfoil(config_only=False, basepath=basepath)
+        try:
+            rd = parse_recipe(config, tinfoil, args.recipename, True)
+            if not rd:
+                return 1
+            recipefile = rd.getVar('FILE', True)
+        finally:
+            tinfoil.shutdown()
+    else:
+        check_workspace_recipe(workspace, args.recipename)
+        bbappend = workspace[args.recipename]['bbappend']
+        bbfile = os.path.basename(bbappend).replace('.bbappend', '.bb').replace('%', '*')
+        recipefile = glob.glob(os.path.join(config.workspace_path,
+                                            'recipes',
+                                            args.recipename,
+                                            bbfile))
+        if recipefile:
+            recipefile = recipefile[0]
+        else:
+            raise DevtoolError("Recipe file for %s is not under the workspace" %
+                               args.recipename)
+
+    editor = os.environ.get('EDITOR', None)
+    if not editor:
+        raise DevtoolError("EDITOR environment variable not set")
+
+    import subprocess
+    try:
+        subprocess.check_call('%s "%s"' % (editor, recipefile), shell=True)
+    except subprocess.CalledProcessError as e:
+        return e.returncode
+
+    return 0
+
+
 def register_commands(subparsers, context):
     """Register devtool subcommands from this plugin"""
     parser_add = subparsers.add_parser('add', help='Add a new recipe',

@@ -1204,3 +1242,9 @@ def register_commands(subparsers, context):
     parser_reset.add_argument('--all', '-a', action="store_true", help='Reset all recipes (clear workspace)')
     parser_reset.add_argument('--no-clean', '-n', action="store_true", help='Don\'t clean the sysroot to remove recipe output')
     parser_reset.set_defaults(func=reset)
+
+    parser_edit_recipe = subparsers.add_parser('edit-recipe', help='Edit a recipe file in your workspace',
+                                         description='Runs the default editor (as specified by the EDITOR variable) on the specified recipe. Note that the recipe file itself must be in the workspace (i.e. as a result of "devtool add" or "devtool upgrade"); you can override this with the -a/--any-recipe option.')
+    parser_edit_recipe.add_argument('recipename', help='Recipe to edit')
+    parser_edit_recipe.add_argument('--any-recipe', '-a', action="store_true", help='Edit any recipe, not just where the recipe file itself is in the workspace')
+    parser_edit_recipe.set_defaults(func=edit_recipe)

index c11de9d..57175a4 100644
@@ -1181,7 +1181,7 @@ def edit_recipe(args, config, basepath, workspace):
 def register_commands(subparsers, context):
     """Register devtool subcommands from this plugin"""
     parser_add = subparsers.add_parser('add', help='Add a new recipe',
-                                       description='Adds a new recipe to the workspace to build a specified source tree')
+                                       description='Adds a new recipe to the workspace to build a specified source tree. Can optionally fetch a remote URI and unpack it to create the source tree.')
     parser_add.add_argument('recipename', help='Name for new recipe to add (just name - no version, path or extension)')
     parser_add.add_argument('srctree', help='Path to external source tree')
     group = parser_add.add_mutually_exclusive_group()

@@ -1190,7 +1190,7 @@ def register_commands(subparsers, context):
     parser_add.add_argument('--fetch', '-f', help='Fetch the specified URI and extract it to create the source tree', metavar='URI')
     parser_add.add_argument('--version', '-V', help='Version to use within recipe (PV)')
     parser_add.add_argument('--no-git', '-g', help='If -f/--fetch is specified, do not set up source tree as a git repository', action="store_true")
-    parser_add.add_argument('--binary', '-b', help='Treat the source tree as something that should be installed verbatim (no compilation, same directory structure)', action='store_true')
+    parser_add.add_argument('--binary', '-b', help='Treat the source tree as something that should be installed verbatim (no compilation, same directory structure). Useful with binary packages e.g. RPMs.', action='store_true')
     parser_add.set_defaults(func=add)
 
     parser_modify = subparsers.add_parser('modify', help='Modify the source for an existing recipe',

index e617d60..7f16e17 100644
@@ -22,6 +22,7 @@ import os
 import sys
 import subprocess
 import logging
+import re
 
 logger = logging.getLogger('devtool')
 

@@ -199,3 +200,17 @@ def setup_git_repo(repodir, version, devbranch, basetag='devtool-base'):
 
     bb.process.run('git checkout -b %s' % devbranch, cwd=repodir)
     bb.process.run('git tag -f %s' % basetag, cwd=repodir)
+
+def recipe_to_append(recipefile, config, wildcard=False):
+    """
+    Convert a recipe file to a bbappend file path within the workspace.
+    NOTE: if the bbappend already exists, you should be using
+    workspace[args.recipename]['bbappend'] instead of calling this
+    function.
+    """
+    appendname = os.path.splitext(os.path.basename(recipefile))[0]
+    if wildcard:
+        appendname = re.sub(r'_.*', '_%', appendname)
+    appendpath = os.path.join(config.workspace_path, 'appends')
+    appendfile = os.path.join(appendpath, appendname + '.bbappend')
+    return appendfile

index 57175a4..0103d93 100644
@@ -26,7 +26,7 @@ import argparse
 import scriptutils
 import errno
 from collections import OrderedDict
-from devtool import exec_build_env_command, setup_tinfoil, check_workspace_recipe, use_external_build, setup_git_repo, DevtoolError
+from devtool import exec_build_env_command, setup_tinfoil, check_workspace_recipe, use_external_build, setup_git_repo, recipe_to_append, DevtoolError
 from devtool import parse_recipe
 
 logger = logging.getLogger('devtool')

@@ -64,10 +64,6 @@ def add(args, config, basepath, workspace):
         raise DevtoolError("Specified source tree %s could not be found" %
                            srctree)
 
-    appendpath = os.path.join(config.workspace_path, 'appends')
-    if not os.path.exists(appendpath):
-        os.makedirs(appendpath)
-
     recipedir = os.path.join(config.workspace_path, 'recipes', args.recipename)
     bb.utils.mkdirhier(recipedir)
     rfv = None

@@ -121,7 +117,8 @@ def add(args, config, basepath, workspace):
     if not rd:
         return 1
 
-    appendfile = os.path.join(appendpath, '%s.bbappend' % bp)
+    appendfile = recipe_to_append(recipefile, config)
+    bb.utils.mkdirhier(os.path.dirname(appendfile))
     with open(appendfile, 'w') as f:
         f.write('inherit externalsrc\n')
         f.write('EXTERNALSRC = "%s"\n' % srctree)

@@ -584,11 +581,7 @@ def modify(args, config, basepath, workspace):
                            pn)
 
     recipefile = rd.getVar('FILE', True)
-    appendname = os.path.splitext(os.path.basename(recipefile))[0]
-    if args.wildcard:
-        appendname = re.sub(r'_.*', '_%', appendname)
-    appendpath = os.path.join(config.workspace_path, 'appends')
-    appendfile = os.path.join(appendpath, appendname + '.bbappend')
+    appendfile = recipe_to_append(recipefile, config, args.wildcard)
     if os.path.exists(appendfile):
         raise DevtoolError("Another variant of recipe %s is already in your "
                            "workspace (only one variant of a recipe can "

@@ -632,8 +625,7 @@ def modify(args, config, basepath, workspace):
         srcsubdir = os.path.relpath(s, workdir).split(os.sep, 1)[1]
         srctree = os.path.join(srctree, srcsubdir)
 
-    if not os.path.exists(appendpath):
-        os.makedirs(appendpath)
+    bb.utils.mkdirhier(os.path.dirname(appendfile))
     with open(appendfile, 'w') as f:
         f.write('FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:"\n')
         # Local files can be modified/tracked in separate subdir under srctree

index 0103d93..195da1a 100644
@@ -908,14 +908,13 @@ def _update_recipe_srcrev(args, srctree, rd, config_data):
 
     _remove_source_files(args, remove_files, destpath)
 
-def _update_recipe_patch(args, config, srctree, rd, config_data):
+def _update_recipe_patch(args, config, workspace, srctree, rd, config_data):
     """Implement the 'patch' mode of update-recipe"""
     import bb
     import oe.recipeutils
 
     recipefile = rd.getVar('FILE', True)
-    append = os.path.join(config.workspace_path, 'appends', '%s.bbappend' %
-                          os.path.splitext(os.path.basename(recipefile))[0])
+    append = workspace[args.recipename]['bbappend']
     if not os.path.exists(append):
         raise DevtoolError('unable to find workspace bbappend for recipe %s' %
                            args.recipename)

@@ -1063,7 +1062,7 @@ def update_recipe(args, config, basepath, workspace):
     if mode == 'srcrev':
         _update_recipe_srcrev(args, srctree, rd, tinfoil.config_data)
     elif mode == 'patch':
-        _update_recipe_patch(args, config, srctree, rd, tinfoil.config_data)
+        _update_recipe_patch(args, config, workspace, srctree, rd, tinfoil.config_data)
     else:
         raise DevtoolError('update_recipe: invalid mode %s' % mode)
 

index c2fee6d..ec7eb8d 100644
@@ -9,3 +9,42 @@ class ArgumentParser(argparse.ArgumentParser):
         self.print_help()
         sys.exit(2)
 
+    def add_subparsers(self, *args, **kwargs):
+        ret = super(ArgumentParser, self).add_subparsers(*args, **kwargs)
+        ret._parser_class = ArgumentSubParser
+        return ret
+
+class ArgumentSubParser(ArgumentParser):
+    def parse_known_args(self, args=None, namespace=None):
+        # This works around argparse not handling optional positional arguments being
+        # intermixed with other options. A pretty horrible hack, but we're not left
+        # with much choice given that the bug in argparse exists and it's difficult
+        # to subclass.
+        # Borrowed from http://stackoverflow.com/questions/20165843/argparse-how-to-handle-variable-number-of-arguments-nargs
+        # with an extra workaround (in format_help() below) for the positional
+        # arguments disappearing from the --help output, as well as structural tweaks.
+        # Originally simplified from http://bugs.python.org/file30204/test_intermixed.py
+        positionals = self._get_positional_actions()
+        for action in positionals:
+            # deactivate positionals
+            action.save_nargs = action.nargs
+            action.nargs = 0
+
+        namespace, remaining_args = super(ArgumentSubParser, self).parse_known_args(args, namespace)
+        for action in positionals:
+            # remove the empty positional values from namespace
+            if hasattr(namespace, action.dest):
+                delattr(namespace, action.dest)
+        for action in positionals:
+            action.nargs = action.save_nargs
+        # parse positionals
+        namespace, extras = super(ArgumentSubParser, self).parse_known_args(remaining_args, namespace)
+        return namespace, extras
+
+    def format_help(self):
+        # Quick, restore the positionals!
+        positionals = self._get_positional_actions()
+        for action in positionals:
+            if hasattr(action, 'save_nargs'):
+                action.nargs = action.save_nargs
+        return super(ArgumentParser, self).format_help()

index 195da1a..b129db3 100644
@@ -49,7 +49,12 @@ def add(args, config, basepath, workspace):
     if '/' in args.recipename:
         raise DevtoolError('"/" is not a valid character in recipe names')
 
-    srctree = os.path.abspath(args.srctree)
+    if args.srctree:
+        srctree = os.path.abspath(args.srctree)
+    else:
+        srctree = get_default_srctree(config, args.recipename)
+        logger.info('Using default source tree path %s' % srctree)
+
     if os.path.exists(srctree):
         if args.fetch:
             if not os.path.isdir(srctree):

@@ -61,8 +66,16 @@ def add(args, config, basepath, workspace):
                                    "it already exists and is non-empty" %
                                    srctree)
     elif not args.fetch:
-        raise DevtoolError("Specified source tree %s could not be found" %
-                           srctree)
+        if args.srctree:
+            raise DevtoolError("Specified source tree %s could not be found" %
+                               args.srctree)
+        else:
+            raise DevtoolError("No source tree exists at default path %s - "
+                               "either create and populate this directory, "
+                               "or specify a path to a source tree, or use "
+                               "the -f/--fetch option to fetch source code "
+                               "and extract it to the source directory" %
+                               srctree)
 
     recipedir = os.path.join(config.workspace_path, 'recipes', args.recipename)
     bb.utils.mkdirhier(recipedir)

@@ -1168,13 +1181,22 @@ def edit_recipe(args, config, basepath, workspace):
 
     return 0
 
+def get_default_srctree(config, recipename=''):
+    """Get the default srctree path"""
+    srctreeparent = config.get('General', 'default_source_parent_dir', config.workspace_path)
+    if recipename:
+        return os.path.join(srctreeparent, 'sources', recipename)
+    else:
+        return os.path.join(srctreeparent, 'sources')
 
 def register_commands(subparsers, context):
     """Register devtool subcommands from this plugin"""
+
+    defsrctree = get_default_srctree(context.config)
     parser_add = subparsers.add_parser('add', help='Add a new recipe',
                                        description='Adds a new recipe to the workspace to build a specified source tree. Can optionally fetch a remote URI and unpack it to create the source tree.')
     parser_add.add_argument('recipename', help='Name for new recipe to add (just name - no version, path or extension)')
-    parser_add.add_argument('srctree', help='Path to external source tree')
+    parser_add.add_argument('srctree', nargs='?', help='Path to external source tree. If not specified, a subdirectory of %s will be used.' % defsrctree)
     group = parser_add.add_mutually_exclusive_group()
     group.add_argument('--same-dir', '-s', help='Build in same directory as source', action="store_true")
     group.add_argument('--no-same-dir', help='Force build in a separate build directory', action="store_true")

index b129db3..6cdd44c 100644
@@ -37,25 +37,39 @@ def add(args, config, basepath, workspace):
     import bb
     import oe.recipeutils
 
-    if args.recipename in workspace:
-        raise DevtoolError("recipe %s is already in your workspace" %
-                            args.recipename)
+    if args.recipename and not args.srctree:
+        # These are positional arguments, but because we're nice, allow
+        # specifying source tree without name if we can detect that that
+        # is what the user meant
+        if os.sep in args.recipename:
+            args.srctree = args.recipename
+            args.recipename = None
+        elif os.path.isdir(args.recipename):
+            logger.warn('Ambiguous argument %s - assuming you mean it to be the recipe name')
 
-    reason = oe.recipeutils.validate_pn(args.recipename)
-    if reason:
-        raise DevtoolError(reason)
+    if args.recipename:
+        if args.recipename in workspace:
+            raise DevtoolError("recipe %s is already in your workspace" %
+                               args.recipename)
+        reason = oe.recipeutils.validate_pn(args.recipename)
+        if reason:
+            raise DevtoolError(reason)
 
-    # FIXME this ought to be in validate_pn but we're using that in other contexts
-    if '/' in args.recipename:
-        raise DevtoolError('"/" is not a valid character in recipe names')
+        # FIXME this ought to be in validate_pn but we're using that in other contexts
+        if '/' in args.recipename:
+            raise DevtoolError('"/" is not a valid character in recipe names')
 
     if args.srctree:
         srctree = os.path.abspath(args.srctree)
+        srctreeparent = None
+        tmpsrcdir = None
     else:
-        srctree = get_default_srctree(config, args.recipename)
-        logger.info('Using default source tree path %s' % srctree)
+        srctree = None
+        srctreeparent = get_default_srctree(config)
+        bb.utils.mkdirhier(srctreeparent)
+        tmpsrcdir = tempfile.mkdtemp(prefix='devtoolsrc', dir=srctreeparent)
 
-    if os.path.exists(srctree):
+    if srctree and os.path.exists(srctree):
         if args.fetch:
             if not os.path.isdir(srctree):
                 raise DevtoolError("Cannot fetch into source tree path %s as "

@@ -69,33 +83,21 @@ def add(args, config, basepath, workspace):
         if args.srctree:
             raise DevtoolError("Specified source tree %s could not be found" %
                                args.srctree)
-        else:
+        elif srctree:
             raise DevtoolError("No source tree exists at default path %s - "
                                "either create and populate this directory, "
                                "or specify a path to a source tree, or use "
                                "the -f/--fetch option to fetch source code "
                                "and extract it to the source directory" %
                                srctree)
+        else:
+            raise DevtoolError("You must either specify a source tree "
+                               "or -f to fetch source")
 
-    recipedir = os.path.join(config.workspace_path, 'recipes', args.recipename)
-    bb.utils.mkdirhier(recipedir)
-    rfv = None
     if args.version:
         if '_' in args.version or ' ' in args.version:
             raise DevtoolError('Invalid version string "%s"' % args.version)
-        rfv = args.version
-    if args.fetch:
-        if args.fetch.startswith('git://'):
-            rfv = 'git'
-        elif args.fetch.startswith('svn://'):
-            rfv = 'svn'
-        elif args.fetch.startswith('hg://'):
-            rfv = 'hg'
-    if rfv:
-        bp = "%s_%s" % (args.recipename, rfv)
-    else:
-        bp = args.recipename
-    recipefile = os.path.join(recipedir, "%s.bb" % bp)
+
     if args.color == 'auto' and sys.stdout.isatty():
         color = 'always'
     else:

@@ -103,19 +105,71 @@ def add(args, config, basepath, workspace):
     extracmdopts = ''
     if args.fetch:
         source = args.fetch
-        extracmdopts = '-x %s' % srctree
+        if srctree:
+            extracmdopts += ' -x %s' % srctree
+        else:
+            extracmdopts += ' -x %s' % tmpsrcdir
     else:
         source = srctree
+    if args.recipename:
+        extracmdopts += ' -N %s' % args.recipename
     if args.version:
         extracmdopts += ' -V %s' % args.version
     if args.binary:
         extracmdopts += ' -b'
+
+    tempdir = tempfile.mkdtemp(prefix='devtool')
     try:
-        stdout, _ = exec_build_env_command(config.init_path, basepath, 'recipetool --color=%s create -o %s "%s" %s' % (color, recipefile, source, extracmdopts))
-    except bb.process.ExecutionError as e:
-        raise DevtoolError('Command \'%s\' failed:\n%s' % (e.command, e.stdout))
+        try:
+            stdout, _ = exec_build_env_command(config.init_path, basepath, 'recipetool --color=%s create -o %s "%s" %s' % (color, tempdir, source, extracmdopts))
+        except bb.process.ExecutionError as e:
+            if e.exitcode == 15:
+                raise DevtoolError('Unable to auto-determine name from source tree, please specify it with -N/--name')
+            else:
+                raise DevtoolError('Command \'%s\' failed:\n%s' % (e.command, e.stdout))
+
+        recipes = glob.glob(os.path.join(tempdir, '*.bb'))
+        if recipes:
+            recipename = os.path.splitext(os.path.basename(recipes[0]))[0].split('_')[0]
+            if recipename in workspace:
+                raise DevtoolError('A recipe with the same name as the one being created (%s) already exists in your workspace' % recipename)
+            recipedir = os.path.join(config.workspace_path, 'recipes', recipename)
+            bb.utils.mkdirhier(recipedir)
+            recipefile = os.path.join(recipedir, os.path.basename(recipes[0]))
+            appendfile = recipe_to_append(recipefile, config)
+            if os.path.exists(appendfile):
+                # This shouldn't be possible, but just in case
+                raise DevtoolError('A recipe with the same name as the one being created already exists in your workspace')
+            if os.path.exists(recipefile):
+                raise DevtoolError('A recipe file %s already exists in your workspace; this shouldn\'t be there - please delete it before continuing' % recipefile)
+            if tmpsrcdir:
+                srctree = os.path.join(srctreeparent, recipename)
+                if os.path.exists(tmpsrcdir):
+                    if os.path.exists(srctree):
+                        if os.path.isdir(srctree):
+                            try:
+                                os.rmdir(srctree)
+                            except OSError as e:
+                                if e.errno == errno.ENOTEMPTY:
+                                    raise DevtoolError('Source tree path %s already exists and is not empty' % srctree)
+                                else:
+                                    raise
+                        else:
+                            raise DevtoolError('Source tree path %s already exists and is not a directory' % srctree)
+                    logger.info('Using default source tree path %s' % srctree)
+                    shutil.move(tmpsrcdir, srctree)
+                else:
+                    raise DevtoolError('Couldn\'t find source tree created by recipetool')
+            bb.utils.mkdirhier(recipedir)
+            shutil.move(recipes[0], recipefile)
+        else:
+            raise DevtoolError('Command \'%s\' did not create any recipe file:\n%s' % (e.command, e.stdout))
+    finally:
+        if tmpsrcdir and os.path.exists(tmpsrcdir):
+            shutil.rmtree(tmpsrcdir)
+        shutil.rmtree(tempdir)
 
-    _add_md5(config, args.recipename, recipefile)
+    _add_md5(config, recipename, recipefile)
 
     if args.fetch and not args.no_git:
         setup_git_repo(srctree, args.version, 'devtool')

@@ -130,7 +184,6 @@ def add(args, config, basepath, workspace):
     if not rd:
         return 1
 
-    appendfile = recipe_to_append(recipefile, config)
     bb.utils.mkdirhier(os.path.dirname(appendfile))
     with open(appendfile, 'w') as f:
         f.write('inherit externalsrc\n')

@@ -148,7 +201,7 @@ def add(args, config, basepath, workspace):
             f.write('    rm -f ${D}/singletask.lock\n')
             f.write('}\n')
 
-    _add_md5(config, args.recipename, appendfile)
+    _add_md5(config, recipename, appendfile)
 
     logger.info('Recipe %s has been automatically created; further editing may be required to make it fully functional' % recipefile)
 

@@ -1195,7 +1248,7 @@ def register_commands(subparsers, context):
     defsrctree = get_default_srctree(context.config)
     parser_add = subparsers.add_parser('add', help='Add a new recipe',
                                        description='Adds a new recipe to the workspace to build a specified source tree. Can optionally fetch a remote URI and unpack it to create the source tree.')
-    parser_add.add_argument('recipename', help='Name for new recipe to add (just name - no version, path or extension)')
+    parser_add.add_argument('recipename', nargs='?', help='Name for new recipe to add (just name - no version, path or extension). If not specified, will attempt to auto-detect it.')
     parser_add.add_argument('srctree', nargs='?', help='Path to external source tree. If not specified, a subdirectory of %s will be used.' % defsrctree)
     group = parser_add.add_mutually_exclusive_group()
     group.add_argument('--same-dir', '-s', help='Build in same directory as source', action="store_true")

index 955495e..93ba58e 100755
@@ -280,6 +280,8 @@ def main():
         if str(err):
             logger.error(str(err))
         ret = 1
+    except argparse_oe.ArgumentUsageError as ae:
+        parser.error_subcommand(ae.message, ae.subcommand)
 
     return ret
 

index ec7eb8d..fd86692 100644
@@ -1,6 +1,12 @@
 import sys
 import argparse
 
+class ArgumentUsageError(Exception):
+    """Exception class you can raise (and catch) in order to show the help"""
+    def __init__(self, message, subcommand=None):
+        self.message = message
+        self.subcommand = subcommand
+
 class ArgumentParser(argparse.ArgumentParser):
     """Our own version of argparse's ArgumentParser"""
 

@@ -9,6 +15,16 @@ class ArgumentParser(argparse.ArgumentParser):
         self.print_help()
         sys.exit(2)
 
+    def error_subcommand(self, message, subcommand):
+        if subcommand:
+            for action in self._actions:
+                if isinstance(action, argparse._SubParsersAction):
+                    for choice, subparser in action.choices.items():
+                        if choice == subcommand:
+                            subparser.error(message)
+                            return
+        self.error(message)
+
     def add_subparsers(self, *args, **kwargs):
         ret = super(ArgumentParser, self).add_subparsers(*args, **kwargs)
         ret._parser_class = ArgumentSubParser

106 lines not on this page · raw message

index 93ba58e..bda05e1 100755
@@ -152,6 +152,10 @@ def _create_workspace(workspacedir, config, basepath):
             f.write('\nIf you no longer need to use devtool you can remove the path to this\n')
             f.write('workspace layer from your conf/bblayers.conf file (and then delete the\n')
             f.write('layer, if you wish).\n')
+            f.write('\nNote that by default, if devtool fetches and unpacks source code, it\n')
+            f.write('will place it in a subdirectory of a "sources" subdirectory of the\n')
+            f.write('layer. If you prefer it to be elsewhere you can specify the source\n')
+            f.write('tree path on the command line.\n')
 
 def _enable_workspace_layer(workspacedir, config, basepath):
     """Ensure the workspace layer is in bblayers.conf"""

index e249661..c710c16 100644
@@ -644,10 +644,15 @@ def modify(args, config, basepath, workspace):
         raise DevtoolError("recipe %s is already in your workspace" %
                            args.recipename)
 
-    if not args.extract and not os.path.isdir(args.srctree):
+    if args.srctree:
+        srctree = os.path.abspath(args.srctree)
+    else:
+        srctree = get_default_srctree(config, args.recipename)
+
+    if not args.extract and not os.path.isdir(srctree):
         raise DevtoolError("directory %s does not exist or not a directory "
                            "(specify -x to extract source from recipe)" %
-                           args.srctree)
+                           srctree)
     if args.extract:
         tinfoil = _prep_extract_operation(config, basepath, args.recipename)
         if not tinfoil:

@@ -679,29 +684,28 @@ def modify(args, config, basepath, workspace):
 
     initial_rev = None
     commits = []
-    srctree = os.path.abspath(args.srctree)
     if args.extract:
-        initial_rev = _extract_source(args.srctree, False, args.branch, False, rd)
+        initial_rev = _extract_source(srctree, False, args.branch, False, rd)
         if not initial_rev:
             return 1
         logger.info('Source tree extracted to %s' % srctree)
         # Get list of commits since this revision
-        (stdout, _) = bb.process.run('git rev-list --reverse %s..HEAD' % initial_rev, cwd=args.srctree)
+        (stdout, _) = bb.process.run('git rev-list --reverse %s..HEAD' % initial_rev, cwd=srctree)
         commits = stdout.split()
     else:
-        if os.path.exists(os.path.join(args.srctree, '.git')):
+        if os.path.exists(os.path.join(srctree, '.git')):
             # Check if it's a tree previously extracted by us
             try:
-                (stdout, _) = bb.process.run('git branch --contains devtool-base', cwd=args.srctree)
+                (stdout, _) = bb.process.run('git branch --contains devtool-base', cwd=srctree)
             except bb.process.ExecutionError:
                 stdout = ''
             for line in stdout.splitlines():
                 if line.startswith('*'):
-                    (stdout, _) = bb.process.run('git rev-parse devtool-base', cwd=args.srctree)
+                    (stdout, _) = bb.process.run('git rev-parse devtool-base', cwd=srctree)
                     initial_rev = stdout.rstrip()
             if not initial_rev:
                 # Otherwise, just grab the head revision
-                (stdout, _) = bb.process.run('git rev-parse HEAD', cwd=args.srctree)
+                (stdout, _) = bb.process.run('git rev-parse HEAD', cwd=srctree)
                 initial_rev = stdout.rstrip()
 
     # Check that recipe isn't using a shared workdir

@@ -1282,9 +1286,9 @@ def register_commands(subparsers, context):
     parser_add.set_defaults(func=add)
 
     parser_modify = subparsers.add_parser('modify', help='Modify the source for an existing recipe',
-                                       description='Enables modifying the source for an existing recipe')
+                                       description='Enables modifying the source for an existing recipe. You can either provide your own pre-prepared source tree, or specify -x/--extract to extract the source being fetched by the recipe.')
     parser_modify.add_argument('recipename', help='Name of existing recipe to edit (just name - no version, path or extension)')
-    parser_modify.add_argument('srctree', help='Path to external source tree')
+    parser_modify.add_argument('srctree', nargs='?', help='Path to external source tree. If not specified, a subdirectory of %s will be used.' % defsrctree)
     parser_modify.add_argument('--wildcard', '-w', action="store_true", help='Use wildcard for unversioned bbappend')
     parser_modify.add_argument('--extract', '-x', action="store_true", help='Extract source as well')
     group = parser_modify.add_mutually_exclusive_group()

index c710c16..ed9e793 100644
@@ -26,6 +26,7 @@ import argparse
 import argparse_oe
 import scriptutils
 import errno
+import glob
 from collections import OrderedDict
 from devtool import exec_build_env_command, setup_tinfoil, check_workspace_recipe, use_external_build, setup_git_repo, recipe_to_append, DevtoolError
 from devtool import parse_recipe

@@ -1168,7 +1169,16 @@ def status(args, config, basepath, workspace):
     """Entry point for the devtool 'status' subcommand"""
     if workspace:
         for recipe, value in workspace.iteritems():
-            print("%s: %s" % (recipe, value['srctree']))
+            bbfile = os.path.basename(value['bbappend']).replace('.bbappend', '.bb').replace('%', '*')
+            recipefile = glob.glob(os.path.join(config.workspace_path,
+                                                'recipes',
+                                                recipe,
+                                                bbfile))
+            if recipefile:
+                recipestr = ' (%s)' % recipefile[0]
+            else:
+                recipestr = ''
+            print("%s: %s%s" % (recipe, value['srctree'], recipestr))
     else:
         logger.info('No recipes currently in your workspace - you can use "devtool modify" to work on an existing recipe or "devtool add" to add a new one')
     return 0

index ed9e793..a9c4cf8 100644
@@ -1229,6 +1229,17 @@ def reset(args, config, basepath, workspace):
         # We don't automatically create this dir next to appends, but the user can
         preservedir(os.path.join(config.workspace_path, 'appends', pn))
 
+        srctree = workspace[pn]['srctree']
+        if os.path.isdir(srctree):
+            if os.listdir(srctree):
+                # We don't want to risk wiping out any work in progress
+                logger.info('Leaving source tree %s as-is; if you no '
+                            'longer need it then please delete it manually'
+                            % srctree)
+            else:
+                # This is unlikely, but if it's empty we can just remove it
+                os.rmdir(srctree)
+
     return 0
 
 

index a9c4cf8..fbc49a9 100644
@@ -346,7 +346,7 @@ def sync(args, config, basepath, workspace):
 
     srctree = os.path.abspath(args.srctree)
     initial_rev = _extract_source(srctree, args.keep_temp, args.branch, True, rd)
-    logger.info('Source tree extracted to %s' % srctree)
+    logger.info('Source tree %s synchronized' % srctree)
 
     if initial_rev:
         return 0

@@ -1326,11 +1326,11 @@ def register_commands(subparsers, context):
     parser_extract.add_argument('--keep-temp', action="store_true", help='Keep temporary directory (for debugging)')
     parser_extract.set_defaults(func=extract, no_workspace=True)
 
-    parser_sync = subparsers.add_parser('sync', help='Synchronize the source for an existing recipe',
-                                       description='Synchronize the source for an existing recipe',
+    parser_sync = subparsers.add_parser('sync', help='Synchronize the source tree for an existing recipe',
+                                       description='Synchronize the previously extracted source tree for an existing recipe',
                                        formatter_class=argparse.ArgumentDefaultsHelpFormatter)
-    parser_sync.add_argument('recipename', help='Name for recipe to sync the source for')
-    parser_sync.add_argument('srctree', help='Path to where to sync the source tree')
+    parser_sync.add_argument('recipename', help='Name of recipe to sync the source for')
+    parser_sync.add_argument('srctree', help='Path to the source tree')
     parser_sync.add_argument('--branch', '-b', default="devtool", help='Name for development branch to checkout')
     parser_sync.add_argument('--keep-temp', action="store_true", help='Keep temporary directory (for debugging)')
     parser_sync.set_defaults(func=sync)

index 0aff59e..1674aba 100644
@@ -127,6 +127,7 @@ class AutotoolsRecipeHandler(RecipeHandler):
                                 extravalues['PN'] = pn
 
         if autoconf:
+            lines_before.append('')
             lines_before.append('# NOTE: if this software is not capable of being built in a separate build directory')
             lines_before.append('# from the source, you should replace autotools with autotools-brokensep in the')
             lines_before.append('# inherit line')

@@ -150,11 +151,14 @@ class AutotoolsRecipeHandler(RecipeHandler):
         # FIXME this mapping is very thin
         progmap = {'flex': 'flex-native',
                 'bison': 'bison-native',
-                'm4': 'm4-native'}
+                'm4': 'm4-native',
+                'tar': 'tar-native',
+                'ar': 'binutils-native'}
         progclassmap = {'gconftool-2': 'gconf',
                 'pkg-config': 'pkgconfig'}
 
-        ignoredeps = ['gcc-runtime', 'glibc', 'uclibc']
+        ignoredeps = ['gcc-runtime', 'glibc', 'uclibc', 'tar-native', 'binutils-native']
+        ignorelibs = ['socket']
 
         pkg_re = re.compile('PKG_CHECK_MODULES\(\[?[a-zA-Z0-9]*\]?, \[?([^,\]]*)[),].*')
         lib_re = re.compile('AC_CHECK_LIB\(\[?([a-zA-Z0-9]*)\]?, .*')

@@ -245,8 +249,12 @@ class AutotoolsRecipeHandler(RecipeHandler):
                     if res:
                         pcdeps.extend([x[0] for x in res])
                 inherits.append('pkgconfig')
-            elif keyword == 'AM_GNU_GETTEXT':
+            elif keyword in ('AM_GNU_GETTEXT', 'AM_GLIB_GNU_GETTEXT', 'GETTEXT_PACKAGE'):
                 inherits.append('gettext')
+            elif keyword in ('AC_PROG_INTLTOOL', 'IT_PROG_INTLTOOL'):
+                deps.append('intltool-native')
+            elif keyword == 'AM_PATH_GLIB_2_0':
+                deps.append('glib-2.0')
             elif keyword == 'AC_CHECK_PROG' or keyword == 'AC_PATH_PROG':
                 res = progs_re.search(value)
                 if res:

@@ -266,13 +274,16 @@ class AutotoolsRecipeHandler(RecipeHandler):
                 res = lib_re.search(value)
                 if res:
                     lib = res.group(1)
-                    libdep = recipelibmap.get(lib, None)
-                    if libdep:
-                        deps.append(libdep)
+                    if lib in ignorelibs:
+                        logger.debug('Ignoring library dependency %s' % lib)
                     else:
-                        if libdep is None:
-                            if not lib.startswith('$'):
-                                unmappedlibs.append(lib)
+                        libdep = recipelibmap.get(lib, None)
+                        if libdep:
+                            deps.append(libdep)
+                        else:
+                            if libdep is None:
+                                if not lib.startswith('$'):
+                                    unmappedlibs.append(lib)
             elif keyword == 'AC_PATH_X':
                 deps.append('libx11')
             elif keyword == 'AC_INIT':

@@ -303,6 +314,11 @@ class AutotoolsRecipeHandler(RecipeHandler):
 
         keywords = ['PKG_CHECK_MODULES',
                     'AM_GNU_GETTEXT',
+                    'AM_GLIB_GNU_GETTEXT',
+                    'GETTEXT_PACKAGE',
+                    'AC_PROG_INTLTOOL',
+                    'IT_PROG_INTLTOOL',
+                    'AM_PATH_GLIB_2_0',
                     'AC_CHECK_PROG',
                     'AC_PATH_PROG',
                     'AC_CHECK_LIB',

@@ -351,10 +367,10 @@ class AutotoolsRecipeHandler(RecipeHandler):
                         extravalues[k] = v.strip('"\'').rstrip('()')
 
         if unmapped:
-            outlines.append('# NOTE: the following prog dependencies are unknown, ignoring: %s' % ' '.join(unmapped))
+            outlines.append('# NOTE: the following prog dependencies are unknown, ignoring: %s' % ' '.join(list(set(unmapped))))
 
         if unmappedlibs:
-            outlines.append('# NOTE: the following library dependencies are unknown, ignoring: %s' % ' '.join(unmappedlibs))
+            outlines.append('# NOTE: the following library dependencies are unknown, ignoring: %s' % ' '.join(list(set(unmappedlibs))))
             outlines.append('#       (this is based on recipes that have previously been built and packaged)')
 
         recipemap = read_pkgconfig_provides(tinfoil.config_data)

index 4887604..873b7dd 100644
@@ -404,6 +404,8 @@ def create_recipe(args):
             # to just remove it first
             os.rmdir(args.extract_to)
         shutil.move(srctree, args.extract_to)
+        if tempsrc == srctree:
+            tempsrc = None
         logger.info('Source extracted to %s' % args.extract_to)
 
     if outfile == '-':

index 7af82df..41e1b4b 100644
@@ -15,18 +15,43 @@ class DevtoolBase(oeSelfTest):
 
     def _test_recipe_contents(self, recipefile, checkvars, checkinherits):
         with open(recipefile, 'r') as f:
+            invar = None
+            invalue = None
             for line in f:
-                if '=' in line:
+                var = None
+                if invar:
+                    value = line.strip().strip('"')
+                    if value.endswith('\\'):
+                        invalue += ' ' + value[:-1].strip()
+                        continue
+                    else:
+                        invalue += ' ' + value.strip()
+                        var = invar
+                        value = invalue
+                        invar = None
+                elif '=' in line:
                     splitline = line.split('=', 1)
                     var = splitline[0].rstrip()
                     value = splitline[1].strip().strip('"')
-                    if var in checkvars:
-                        needvalue = checkvars.pop(var)
-                        self.assertEqual(value, needvalue, 'values for %s do not match' % var)
-                if line.startswith('inherit '):
+                    if value.endswith('\\'):
+                        invalue = value[:-1].strip()
+                        invar = var
+                        continue
+                elif line.startswith('inherit '):
                     inherits = line.split()[1:]
 
-        self.assertEqual(checkvars, {}, 'Some variables not found: %s' % checkvars)
+                if var and var in checkvars:
+                    needvalue = checkvars.pop(var)
+                    if needvalue is None:
+                        self.fail('Variable %s should not appear in recipe')
+                    self.assertEqual(value, needvalue, 'values for %s do not match' % var)
+
+
+        missingvars = {}
+        for var, value in checkvars.iteritems():
+            if value is not None:
+                missingvars[var] = value
+        self.assertEqual(missingvars, {}, 'Some expected variables not found in recipe: %s' % checkvars)
 
         for inherit in checkinherits:
             self.assertIn(inherit, inherits, 'Missing inherit of %s' % inherit)

@@ -304,6 +329,32 @@ class DevtoolTests(DevtoolBase):
         checkvars['SRCREV'] = checkrev
         self._test_recipe_contents(recipefile, checkvars, [])
 
+    def test_devtool_add_fetch_simple(self):
+        # Fetch source from a remote URL, auto-detecting name
+        tempdir = tempfile.mkdtemp(prefix='devtoolqa')
+        self.track_for_cleanup(tempdir)
+        testver = '1.6.0'
+        url = 'http://www.ivarch.com/programs/sources/pv-%s.tar.bz2' % testver
+        testrecipe = 'pv'
+        srcdir = os.path.join(self.workspacedir, 'sources', testrecipe)
+        # Test devtool add
+        self.track_for_cleanup(self.workspacedir)
+        self.add_command_to_tearDown('bitbake-layers remove-layer */workspace')
+        result = runCmd('devtool add %s' % url)
+        self.assertTrue(os.path.exists(os.path.join(self.workspacedir, 'conf', 'layer.conf')), 'Workspace directory not created. %s' % result.output)
+        self.assertTrue(os.path.isfile(os.path.join(srcdir, 'configure')), 'Unable to find configure script in source directory')
+        # Test devtool status
+        result = runCmd('devtool status')
+        self.assertIn(testrecipe, result.output)
+        self.assertIn(srcdir, result.output)
+        # Check recipe
+        recipefile = get_bb_var('FILE', testrecipe)
+        self.assertIn('%s_%s.bb' % (testrecipe, testver), recipefile, 'Recipe file incorrectly named')
+        checkvars = {}
+        checkvars['S'] = None
+        checkvars['SRC_URI'] = url.replace(testver, '${PV}')
+        self._test_recipe_contents(recipefile, checkvars, [])
+
     @testcase(1164)
     def test_devtool_modify(self):
         # Clean up anything in the workdir/sysroot/sstate cache

index bbfce7c..34e383f 100644
@@ -401,6 +401,25 @@ class RecipetoolTests(RecipetoolBase):
         inherits = ['autotools', 'pkgconfig']
         self._test_recipe_contents(recipefile, checkvars, inherits)
 
+    def test_recipetool_create_simple(self):
+        # Try adding a recipe
+        temprecipe = os.path.join(self.tempdir, 'recipe')
+        os.makedirs(temprecipe)
+        pv = '1.7.3.0'
+        srcuri = 'http://www.dest-unreach.org/socat/download/socat-%s.tar.bz2' % pv
+        result = runCmd('recipetool create %s -o %s' % (srcuri, temprecipe))
+        dirlist = os.listdir(temprecipe)
+        if len(dirlist) < 1 or not os.path.isfile(os.path.join(temprecipe, dirlist[0])):
+            self.fail('recipetool did not create recipe file; output:\n%s' % result.output)
+        self.assertEqual(dirlist[0], 'socat_%s.bb' % pv, 'Recipe file incorrectly named')
+        checkvars = {}
+        checkvars['LICENSE'] = 'Unknown GPLv2'
+        checkvars['LIC_FILES_CHKSUM'] = 'file://COPYING.OpenSSL;md5=5c9bccc77f67a8328ef4ebaf468116f4 file://COPYING;md5=b234ee4d69f5fce4486a80fdaf4a4263'
+        # We don't check DEPENDS since they are variable for this recipe depending on what's in the sysroot
+        checkvars['S'] = None
+        checkvars['SRC_URI'] = srcuri.replace(pv, '${PV}')
+        inherits = ['autotools']
+        self._test_recipe_contents(os.path.join(temprecipe, dirlist[0]), checkvars, inherits)
 
 class RecipetoolAppendsrcBase(RecipetoolBase):
     def _try_recipetool_appendsrcfile(self, testrecipe, newfile, destfile, options, expectedlines, expectedfiles):

index 41e1b4b..955379b 100644
@@ -93,6 +93,8 @@ class DevtoolBase(oeSelfTest):
         filelist = []
         for line in output.splitlines():
             splitline = line.split()
+            if len(splitline) < 8:
+                self.fail('_process_ls_output: invalid output line: %s' % line)
             # Remove trailing . on perms
             splitline[0] = splitline[0].rstrip('.')
             # Remove leading . on paths