devtool: enable adding remote source

index 1c71b24..1d5bfd9 100644
@@ -1,6 +1,6 @@
 # Recipe creation tool - create command plugin
 #
-# Copyright (C) 2014 Intel Corporation
+# Copyright (C) 2014-2015 Intel Corporation
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 2 as

@@ -110,11 +110,8 @@ def create_recipe(args):
     if '://' in args.source:
         # Fetch a URL
         srcuri = args.source
-        if args.extract_to:
-            srctree = args.extract_to
-        else:
-            tempsrc = tempfile.mkdtemp(prefix='recipetool-')
-            srctree = tempsrc
+        tempsrc = tempfile.mkdtemp(prefix='recipetool-')
+        srctree = tempsrc
         logger.info('Fetching %s...' % srcuri)
         checksums = fetch_source(args.source, srctree)
         dirlist = os.listdir(srctree)

@@ -240,6 +237,10 @@ def create_recipe(args):
         outlines.append('')
     outlines.extend(lines_after)
 
+    if args.extract_to:
+        shutil.move(srctree, args.extract_to)
+        logger.info('Source extracted to %s' % args.extract_to)
+
     if outfile == '-':
         sys.stdout.write('\n'.join(outlines) + '\n')
     else:

index 94b5e0b..eb41f28 100644
@@ -23,6 +23,7 @@ import glob
 import tempfile
 import logging
 import argparse
+import scriptutils
 from devtool import exec_build_env_command, setup_tinfoil
 
 logger = logging.getLogger('devtool')

@@ -236,12 +237,7 @@ def _extract_source(srctree, keep_temp, devbranch, d):
             # Handle if S is set to a subdirectory of the source
             srcsubdir = os.path.join(workdir, os.path.relpath(srcsubdir, workdir).split(os.sep)[0])
 
-        if os.path.exists(os.path.join(srcsubdir, '.git')):
-            alternatesfile = os.path.join(srcsubdir, '.git', 'objects', 'info', 'alternates')
-            if os.path.exists(alternatesfile):
-                # This will have been cloned with -s, so we need to convert it to a full clone
-                bb.process.run('git repack -a', cwd=srcsubdir)
-                os.remove(alternatesfile)
+        scriptutils.git_convert_full_clone(srcsubdir)
 
         patchdir = os.path.join(srcsubdir, 'patches')
         haspatches = False

index 1d5bfd9..15de694 100644
@@ -22,6 +22,7 @@ import glob
 import fnmatch
 import re
 import logging
+import scriptutils
 
 logger = logging.getLogger('recipetool')
 

@@ -238,6 +239,7 @@ def create_recipe(args):
     outlines.extend(lines_after)
 
     if args.extract_to:
+        scriptutils.git_convert_full_clone(srctree)
         shutil.move(srctree, args.extract_to)
         logger.info('Source extracted to %s' % args.extract_to)
 

index e786126..4af6fd5 100644
@@ -58,3 +58,13 @@ def load_plugins(logger, plugins, pluginpath):
             if hasattr(plugin, 'plugin_init'):
                 plugin.plugin_init(plugins)
                 plugins.append(plugin)
+
+def git_convert_full_clone(repodir):
+    """If specified directory is a git repository, ensure it's a full clone"""
+    import bb.process
+    if os.path.exists(os.path.join(repodir, '.git')):
+        alternatesfile = os.path.join(repodir, '.git', 'objects', 'info', 'alternates')
+        if os.path.exists(alternatesfile):
+            # This will have been cloned with -s, so we need to convert it to a full clone
+            bb.process.run('git repack -a', cwd=repodir)
+            os.remove(alternatesfile)

index eb41f28..482c418 100644
@@ -46,6 +46,19 @@ def add(args, config, basepath, workspace):
         return -1
 
     srctree = os.path.abspath(args.srctree)
+    if os.path.exists(srctree):
+        if args.fetch:
+            if not os.path.isdir(srctree):
+                logger.error("Cannot fetch into source tree path %s as it exists and is not a directory" % srctree)
+                return 1
+            elif os.listdir(srctree):
+                logger.error("Cannot fetch into source tree path %s as it already exists and is non-empty" % srctree)
+                return 1
+    else:
+        if not args.fetch:
+            logger.error("Specified source tree %s could not be found" % srctree)
+            return 1
+
     appendpath = os.path.join(config.workspace_path, 'appends')
     if not os.path.exists(appendpath):
         os.makedirs(appendpath)

@@ -64,7 +77,13 @@ def add(args, config, basepath, workspace):
         color = 'always'
     else:
         color = args.color
-    stdout, stderr = exec_build_env_command(config.init_path, basepath, 'recipetool --color=%s create -o %s %s' % (color, recipefile, srctree))
+    extracmdopts = ''
+    if args.fetch:
+        source = args.fetch
+        extracmdopts = '-x %s' % srctree
+    else:
+        source = srctree
+    stdout, stderr = exec_build_env_command(config.init_path, basepath, 'recipetool --color=%s create -o %s "%s" %s' % (color, recipefile, source, extracmdopts))
     logger.info('Recipe %s has been automatically created; further editing may be required to make it fully functional' % recipefile)
 
     _add_md5(config, args.recipename, recipefile)

@@ -670,11 +689,11 @@ def build(args, config, basepath, workspace):
 
 def register_commands(subparsers, context):
     parser_add = subparsers.add_parser('add', help='Add a new recipe',
-                                       description='Adds a new recipe',
-                                       formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+                                       description='Adds a new recipe')
     parser_add.add_argument('recipename', help='Name for new recipe to add')
     parser_add.add_argument('srctree', help='Path to external source tree')
     parser_add.add_argument('--same-dir', '-s', help='Build in same directory as source', action="store_true")
+    parser_add.add_argument('--fetch', '-f', help='Fetch the specified URI and extract it to create the source tree', metavar='URI')
     parser_add.add_argument('--version', '-V', help='Version to use within recipe (PV)')
     parser_add.set_defaults(func=add)
 

index 482c418..16f737e 100644
@@ -65,11 +65,21 @@ def add(args, config, basepath, workspace):
 
     recipedir = os.path.join(config.workspace_path, 'recipes', args.recipename)
     bb.utils.mkdirhier(recipedir)
+    rfv = None
     if args.version:
         if '_' in args.version or ' ' in args.version:
             logger.error('Invalid version string "%s"' % args.version)
             return -1
-        bp = "%s_%s" % (args.recipename, args.version)
+        rfv = args.version
+    if args.fetch:
+        if args.fetch.startswith('git://'):
+            rfv = 'git'
+        elif args.fetch.startswith('svn://'):
+            rfv = 'svn'
+        elif args.fetch.startswith('hg://'):
+            rfv = 'hg'
+    if rfv:
+        bp = "%s_%s" % (args.recipename, rfv)
     else:
         bp = args.recipename
     recipefile = os.path.join(recipedir, "%s.bb" % bp)

@@ -83,6 +93,8 @@ def add(args, config, basepath, workspace):
         extracmdopts = '-x %s' % srctree
     else:
         source = srctree
+    if args.version:
+        extracmdopts += ' -V %s' % args.version
     stdout, stderr = exec_build_env_command(config.init_path, basepath, 'recipetool --color=%s create -o %s "%s" %s' % (color, recipefile, source, extracmdopts))
     logger.info('Recipe %s has been automatically created; further editing may be required to make it fully functional' % recipefile)
 

index 15de694..96062ab 100644
@@ -187,9 +187,17 @@ def create_recipe(args):
         pn = recipefn
         pv = None
 
+    if args.version:
+        pv = args.version
+
+    if pv and pv not in 'git svn hg'.split():
+        realpv = pv
+    else:
+        realpv = None
+
     if srcuri:
-        if pv and pv not in 'git svn hg'.split():
-            srcuri = srcuri.replace(pv, '${PV}')
+        if realpv:
+            srcuri = srcuri.replace(realpv, '${PV}')
     else:
         lines_before.append('# No information for SRC_URI yet (only an external source tree was specified)')
     lines_before.append('SRC_URI = "%s"' % srcuri)

@@ -201,7 +209,7 @@ def create_recipe(args):
     if srcuri and supports_srcrev(srcuri):
         lines_before.append('')
         lines_before.append('# Modify these as desired')
-        lines_before.append('PV = "1.0+git${SRCPV}"')
+        lines_before.append('PV = "%s+git${SRCPV}"' % (realpv or '1.0'))
         lines_before.append('SRCREV = "${AUTOREV}"')
     lines_before.append('')
 

@@ -418,5 +426,6 @@ def register_command(subparsers):
     parser_create.add_argument('-o', '--outfile', help='Specify filename for recipe to create', required=True)
     parser_create.add_argument('-m', '--machine', help='Make recipe machine-specific as opposed to architecture-specific', action='store_true')
     parser_create.add_argument('-x', '--extract-to', metavar='EXTRACTPATH', help='Assuming source is a URL, fetch it and extract it to the directory specified as %(metavar)s')
+    parser_create.add_argument('-V', '--version', help='Version to use within recipe (PV)')
     parser_create.set_defaults(func=create_recipe)
 

index 96062ab..36bd6d9 100644
@@ -58,13 +58,13 @@ class RecipeHandler():
 
 
 
-def fetch_source(uri, destdir):
+def fetch_source(uri, destdir, srcrev):
     import bb.data
     bb.utils.mkdirhier(destdir)
     localdata = bb.data.createCopy(tinfoil.config_data)
     bb.data.update_data(localdata)
     localdata.setVar('BB_STRICT_CHECKSUM', '')
-    localdata.setVar('SRCREV', '${AUTOREV}')
+    localdata.setVar('SRCREV', srcrev)
     ret = (None, None)
     olddir = os.getcwd()
     try:

@@ -88,6 +88,9 @@ def fetch_source(uri, destdir):
 
 def supports_srcrev(uri):
     localdata = bb.data.createCopy(tinfoil.config_data)
+    # This is a bit sad, but if you don't have this set there can be some
+    # odd interactions with the urldata cache which lead to errors
+    localdata.setVar('SRCREV', '${AUTOREV}')
     bb.data.update_data(localdata)
     fetcher = bb.fetch2.Fetch([uri], localdata)
     urldata = fetcher.ud

@@ -108,13 +111,19 @@ def create_recipe(args):
     checksums = (None, None)
     tempsrc = ''
     srcsubdir = ''
+    srcrev = '${AUTOREV}'
     if '://' in args.source:
         # Fetch a URL
         srcuri = args.source
+        rev_re = re.compile(';rev=([^;]+)')
+        res = rev_re.search(srcuri)
+        if res:
+            srcrev = res.group(1)
+            srcuri = rev_re.sub('', srcuri)
         tempsrc = tempfile.mkdtemp(prefix='recipetool-')
         srctree = tempsrc
         logger.info('Fetching %s...' % srcuri)
-        checksums = fetch_source(args.source, srctree)
+        checksums = fetch_source(args.source, srctree, srcrev)
         dirlist = os.listdir(srctree)
         if 'git.indirectionsymlink' in dirlist:
             dirlist.remove('git.indirectionsymlink')

@@ -210,7 +219,7 @@ def create_recipe(args):
         lines_before.append('')
         lines_before.append('# Modify these as desired')
         lines_before.append('PV = "%s+git${SRCPV}"' % (realpv or '1.0'))
-        lines_before.append('SRCREV = "${AUTOREV}"')
+        lines_before.append('SRCREV = "%s"' % srcrev)
     lines_before.append('')
 
     if srcsubdir and pv:

index 6881835..1a506d9 100644
@@ -35,6 +35,24 @@ class DevtoolTests(oeSelfTest):
         self.assertNotIn(tempdir, result.output)
         self.assertIn(workspacedir, result.output)
 
+    def _test_recipe_contents(self, recipefile, checkvars, checkinherits):
+        with open(recipefile, 'r') as f:
+            for line in f:
+                if '=' in line:
+                    splitline = line.split('=', 1)
+                    var = splitline[0].rstrip()
+                    value = splitline[1].strip().strip('"')
+                    if var in checkvars:
+                        needvalue = checkvars.pop(var)
+                        self.assertEqual(value, needvalue, 'values for %s do not match' % var)
+                if line.startswith('inherit '):
+                    inherits = line.split()[1:]
+
+        self.assertEqual(checkvars, {}, 'Some variables not found: %s' % checkvars)
+
+        for inherit in checkinherits:
+            self.assertIn(inherit, inherits, 'Missing inherit of %s' % inherit)
+
     def test_recipetool_create(self):
         # Try adding a recipe
         tempdir = tempfile.mkdtemp(prefix='devtoolqa')

@@ -51,19 +69,7 @@ class DevtoolTests(oeSelfTest):
         checkvars['SRC_URI'] = 'https://fedorahosted.org/releases/l/o/logrotate/logrotate-${PV}.tar.gz'
         checkvars['SRC_URI[md5sum]'] = '99e08503ef24c3e2e3ff74cc5f3be213'
         checkvars['SRC_URI[sha256sum]'] = 'f6ba691f40e30e640efa2752c1f9499a3f9738257660994de70a45fe00d12b64'
-        with open(recipefile, 'r') as f:
-            for line in f:
-                if '=' in line:
-                    splitline = line.split('=', 1)
-                    var = splitline[0].rstrip()
-                    value = splitline[1].strip().strip('"')
-                    if var in checkvars:
-                        needvalue = checkvars.pop(var)
-                        self.assertEqual(value, needvalue)
-                if line.startswith('inherit '):
-                    inherits = line.split()[1:]
-
-        self.assertEqual(checkvars, {}, 'Some variables not found')
+        self._test_recipe_contents(recipefile, checkvars, [])
 
     def test_recipetool_create_git(self):
         # Ensure we have the right data in shlibs/pkgdata

@@ -84,23 +90,8 @@ class DevtoolTests(oeSelfTest):
         checkvars['PV'] = '1.0+git${SRCPV}'
         checkvars['SRC_URI'] = srcuri
         checkvars['DEPENDS'] = 'libpng pango libx11 libxext jpeg'
-        inherits = []
-        with open(recipefile, 'r') as f:
-            for line in f:
-                if '=' in line:
-                    splitline = line.split('=', 1)
-                    var = splitline[0].rstrip()
-                    value = splitline[1].strip().strip('"')
-                    if var in checkvars:
-                        needvalue = checkvars.pop(var)
-                        self.assertEqual(value, needvalue)
-                if line.startswith('inherit '):
-                    inherits = line.split()[1:]
-
-        self.assertEqual(checkvars, {}, 'Some variables not found')
-
-        self.assertIn('autotools', inherits, 'Missing inherit of autotools')
-        self.assertIn('pkgconfig', inherits, 'Missing inherit of pkgconfig')
+        inherits = ['autotools', 'pkgconfig']
+        self._test_recipe_contents(recipefile, checkvars, inherits)
 
     def test_devtool_add(self):
         # Check preconditions

@@ -176,6 +167,103 @@ class DevtoolTests(oeSelfTest):
         self.assertFalse(matches, 'Stamp files exist for recipe libftdi that should have been cleaned')
         self.assertFalse(os.path.isfile(os.path.join(staging_libdir, 'libftdi1.so.2.1.0')), 'libftdi binary still found in STAGING_LIBDIR after cleaning')
 
+    def test_devtool_add_fetch(self):
+        # Check preconditions
+        workspacedir = os.path.join(self.builddir, 'workspace')
+        self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory')
+        # Fetch source
+        tempdir = tempfile.mkdtemp(prefix='devtoolqa')
+        self.track_for_cleanup(tempdir)
+        testver = '0.23'
+        url = 'https://pypi.python.org/packages/source/J/MarkupSafe/MarkupSafe-%s.tar.gz' % testver
+        testrecipe = 'python-markupsafe'
+        srcdir = os.path.join(tempdir, testrecipe)
+        # Test devtool add
+        self.track_for_cleanup(workspacedir)
+        self.add_command_to_tearDown('bitbake -c cleansstate %s' % testrecipe)
+        self.add_command_to_tearDown('bitbake-layers remove-layer */workspace')
+        result = runCmd('devtool add %s %s -f %s' % (testrecipe, srcdir, url))
+        self.assertTrue(os.path.exists(os.path.join(workspacedir, 'conf', 'layer.conf')), 'Workspace directory not created')
+        self.assertTrue(os.path.isfile(os.path.join(srcdir, 'setup.py')), 'Unable to find setup.py in source directory')
+        # Test devtool status
+        result = runCmd('devtool status')
+        self.assertIn(testrecipe, result.output)
+        self.assertIn(srcdir, result.output)
+        # Check recipe
+        recipefile = get_bb_var('FILE', testrecipe)
+        self.assertIn('%s.bb' % testrecipe, recipefile, 'Recipe file incorrectly named')
+        checkvars = {}
+        checkvars['S'] = '${WORKDIR}/MarkupSafe-%s' % testver
+        checkvars['SRC_URI'] = url
+        self._test_recipe_contents(recipefile, checkvars, [])
+        # Try with version specified
+        result = runCmd('devtool reset -n %s' % testrecipe)
+        shutil.rmtree(srcdir)
+        result = runCmd('devtool add %s %s -f %s -V %s' % (testrecipe, srcdir, url, testver))
+        self.assertTrue(os.path.isfile(os.path.join(srcdir, 'setup.py')), 'Unable to find setup.py in source directory')
+        # Test devtool status
+        result = runCmd('devtool status')
+        self.assertIn(testrecipe, result.output)
+        self.assertIn(srcdir, result.output)
+        # Check recipe
+        recipefile = get_bb_var('FILE', testrecipe)
+        self.assertIn('%s_%s.bb' % (testrecipe, testver), recipefile, 'Recipe file incorrectly named')
+        checkvars = {}
+        checkvars['S'] = '${WORKDIR}/MarkupSafe-${PV}'
+        checkvars['SRC_URI'] = url.replace(testver, '${PV}')
+        self._test_recipe_contents(recipefile, checkvars, [])
+
+    def test_devtool_add_fetch_git(self):
+        # Check preconditions
+        workspacedir = os.path.join(self.builddir, 'workspace')
+        self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory')
+        # Fetch source
+        tempdir = tempfile.mkdtemp(prefix='devtoolqa')
+        self.track_for_cleanup(tempdir)
+        url = 'git://git.yoctoproject.org/libmatchbox'
+        checkrev = '462f0652055d89c648ddd54fd7b03f175c2c6973'
+        testrecipe = 'libmatchbox2'
+        srcdir = os.path.join(tempdir, testrecipe)
+        # Test devtool add
+        self.track_for_cleanup(workspacedir)
+        self.add_command_to_tearDown('bitbake -c cleansstate %s' % testrecipe)
+        self.add_command_to_tearDown('bitbake-layers remove-layer */workspace')
+        result = runCmd('devtool add %s %s -f %s' % (testrecipe, srcdir, url))
+        self.assertTrue(os.path.exists(os.path.join(workspacedir, 'conf', 'layer.conf')), 'Workspace directory not created')
+        self.assertTrue(os.path.isfile(os.path.join(srcdir, 'configure.ac')), 'Unable to find configure.ac in source directory')
+        # Test devtool status
+        result = runCmd('devtool status')
+        self.assertIn(testrecipe, result.output)
+        self.assertIn(srcdir, result.output)
+        # Check recipe
+        recipefile = get_bb_var('FILE', testrecipe)
+        self.assertIn('_git.bb', recipefile, 'Recipe file incorrectly named')
+        checkvars = {}
+        checkvars['S'] = '${WORKDIR}/git'
+        checkvars['PV'] = '1.0+git${SRCPV}'
+        checkvars['SRC_URI'] = url
+        checkvars['SRCREV'] = '${AUTOREV}'
+        self._test_recipe_contents(recipefile, checkvars, [])
+        # Try with revision and version specified
+        result = runCmd('devtool reset -n %s' % testrecipe)
+        shutil.rmtree(srcdir)
+        url_rev = '%s;rev=%s' % (url, checkrev)
+        result = runCmd('devtool add %s %s -f "%s" -V 1.5' % (testrecipe, srcdir, url_rev))
+        self.assertTrue(os.path.isfile(os.path.join(srcdir, 'configure.ac')), 'Unable to find configure.ac in source directory')
+        # Test devtool status
+        result = runCmd('devtool status')
+        self.assertIn(testrecipe, result.output)
+        self.assertIn(srcdir, result.output)
+        # Check recipe
+        recipefile = get_bb_var('FILE', testrecipe)
+        self.assertIn('_git.bb', recipefile, 'Recipe file incorrectly named')
+        checkvars = {}
+        checkvars['S'] = '${WORKDIR}/git'
+        checkvars['PV'] = '1.5+git${SRCPV}'
+        checkvars['SRC_URI'] = url
+        checkvars['SRCREV'] = checkrev
+        self._test_recipe_contents(recipefile, checkvars, [])
+
     def test_devtool_modify(self):
         # Check preconditions
         workspacedir = os.path.join(self.builddir, 'workspace')