Fixes/enhancements for devtool/recipetool/externalsrc

index 607861f..9d7ab00 100644
@@ -86,6 +86,9 @@ python () {
         # Ensure compilation happens every time
         d.setVarFlag('do_compile', 'nostamp', '1')
 
+        # We don't want the workdir to go away
+        d.appendVar('RM_WORK_EXCLUDE', ' ' + d.getVar('PN', True))
+
         # If B=S the same builddir is used even for different architectures.
         # Thus, use a shared CONFIGURESTAMPFILE so that change of do_configure
         # task hash is correctly detected if e.g. MACHINE changes. In addition,

index 9d7ab00..b608bd0 100644
@@ -25,6 +25,7 @@
 #
 
 SRCTREECOVEREDTASKS ?= "do_patch do_unpack do_fetch"
+EXTERNALSRC_SYMLINKS ?= "oe-workdir:${WORKDIR} oe-logs:${T}"
 
 python () {
     externalsrc = d.getVar('EXTERNALSRC', True)

@@ -82,6 +83,7 @@ python () {
             bb.build.deltask(task, d)
 
         d.prependVarFlag('do_compile', 'prefuncs', "externalsrc_compile_prefunc ")
+        d.prependVarFlag('do_configure', 'prefuncs', "externalsrc_configure_prefunc ")
 
         # Ensure compilation happens every time
         d.setVarFlag('do_compile', 'nostamp', '1')

@@ -101,6 +103,24 @@ python () {
             d.setVarFlag('do_configure', 'file-checksums', configstamp + ':True')
 }
 
+python externalsrc_configure_prefunc() {
+    # Create desired symlinks
+    symlinks = (d.getVar('EXTERNALSRC_SYMLINKS', True) or '').split()
+    for symlink in symlinks:
+        symsplit = symlink.split(':', 1)
+        lnkfile = os.path.join(d.getVar('S', True), symsplit[0])
+        target = d.expand(symsplit[1])
+        if len(symsplit) > 1:
+            if os.path.islink(lnkfile):
+                # Link already exists, leave it if it points to the right location already
+                if os.readlink(lnkfile) == target:
+                    continue
+            elif os.path.exists(lnkfile):
+                # File/dir exists with same name as link, just leave it alone
+                continue
+            os.symlink(target, lnkfile)
+}
+
 python externalsrc_compile_prefunc() {
     # Make it obvious that this is happening, since forgetting about it could lead to much confusion
     bb.plain('NOTE: %s: compiling from external source tree %s' % (d.getVar('PN', True), d.getVar('EXTERNALSRC', True)))

index 4e77168..2464efd 100644
@@ -264,6 +264,7 @@ class PatchTree(PatchSet):
 
 class GitApplyTree(PatchTree):
     patch_line_prefix = '%% original patch'
+    ignore_commit_prefix = '%% ignore'
 
     def __init__(self, dir, d):
         PatchTree.__init__(self, dir, d)

@@ -384,6 +385,8 @@ class GitApplyTree(PatchTree):
                             if line.startswith(GitApplyTree.patch_line_prefix):
                                 outfile = line.split()[-1].strip()
                                 continue
+                            if line.startswith(GitApplyTree.ignore_commit_prefix):
+                                continue
                             patchlines.append(line)
                     if not outfile:
                         outfile = os.path.basename(srcfile)

78 lines not on this page · raw message

index ba393a8..4d11e04 100644
@@ -137,6 +137,7 @@ class CmakeRecipeHandler(RecipeHandler):
         pkgcm_re = re.compile('pkg_check_modules\s*\(\s*[a-zA-Z0-9-_]+\s*(REQUIRED)?\s+([^)\s]+)\s*\)', re.IGNORECASE)
         pkgsm_re = re.compile('pkg_search_module\s*\(\s*[a-zA-Z0-9-_]+\s*(REQUIRED)?((\s+[^)\s]+)+)\s*\)', re.IGNORECASE)
         findpackage_re = re.compile('find_package\s*\(\s*([a-zA-Z0-9-_]+)\s*.*', re.IGNORECASE)
+        findlibrary_re = re.compile('find_library\s*\(\s*[a-zA-Z0-9-_]+\s*(NAMES\s+)?([a-zA-Z0-9-_ ]+)\s*.*')
         checklib_re = re.compile('check_library_exists\s*\(\s*([^\s)]+)\s*.*', re.IGNORECASE)
         include_re = re.compile('include\s*\(\s*([^)\s]*)\s*\)', re.IGNORECASE)
         subdir_re = re.compile('add_subdirectory\s*\(\s*([^)\s]*)\s*([^)\s]*)\s*\)', re.IGNORECASE)

@@ -215,6 +216,15 @@ class CmakeRecipeHandler(RecipeHandler):
                         lib = interpret_value(res.group(1))
                         if not lib.startswith('$'):
                             libdeps.append(lib)
+                    res = findlibrary_re.match(line)
+                    if res:
+                        libs = res.group(2).split()
+                        for lib in libs:
+                            if lib in ['HINTS', 'PATHS', 'PATH_SUFFIXES', 'DOC', 'NAMES_PER_DIR'] or lib.startswith(('NO_', 'CMAKE_', 'ONLY_CMAKE_')):
+                                break
+                            lib = interpret_value(lib)
+                            if not lib.startswith('$'):
+                                libdeps.append(lib)
                     if line.lower().startswith('useswig'):
                         deps.append('swig-native')
                         continue

index 5f90b10..ee27f8d 100644
@@ -247,6 +247,35 @@ def determine_from_filename(srcfile):
         pv = None
     return (pn, pv)
 
+def determine_from_url(srcuri):
+    """Determine name and version from a URL"""
+    pn = None
+    pv = None
+    parseres = urlparse.urlparse(srcuri.lower())
+    if parseres.path:
+        if 'github.com' in parseres.netloc:
+            res = re.search(r'.*/(.*?)/archive/(.*)-final\.(tar|zip)', parseres.path)
+            if res:
+                pn = res.group(1).strip().replace('_', '-')
+                pv = res.group(2).strip().replace('_', '.')
+            else:
+                res = re.search(r'.*/(.*?)/archive/v?(.*)\.(tar|zip)', parseres.path)
+                if res:
+                    pn = res.group(1).strip().replace('_', '-')
+                    pv = res.group(2).strip().replace('_', '.')
+        elif 'bitbucket.org' in parseres.netloc:
+            res = re.search(r'.*/(.*?)/get/[a-zA-Z_-]*([0-9][0-9a-zA-Z_.]*)\.(tar|zip)', parseres.path)
+            if res:
+                pn = res.group(1).strip().replace('_', '-')
+                pv = res.group(2).strip().replace('_', '.')
+
+        if not pn and not pv:
+            srcfile = os.path.basename(parseres.path.rstrip('/'))
+            pn, pv = determine_from_filename(srcfile)
+
+    logger.debug('Determined from source URL: name = "%s", version = "%s"' % (pn, pv))
+    return (pn, pv)
+
 def supports_srcrev(uri):
     localdata = bb.data.createCopy(tinfoil.config_data)
     # This is a bit sad, but if you don't have this set there can be some

@@ -430,15 +459,12 @@ def create_recipe(args):
         realpv = None
 
     if srcuri and not realpv or not pn:
-        parseres = urlparse.urlparse(srcuri)
-        if parseres.path:
-            srcfile = os.path.basename(parseres.path.rstrip('/'))
-            name_pn, name_pv = determine_from_filename(srcfile)
-            logger.debug('Determined from filename: name = "%s", version = "%s"' % (name_pn, name_pv))
-            if name_pn and not pn:
-                pn = name_pn
-            if name_pv and not realpv:
-                realpv = name_pv
+        name_pn, name_pv = determine_from_url(srcuri)
+        if name_pn and not pn:
+            pn = name_pn
+        if name_pv and not realpv:
+            realpv = name_pv
+
 
     if not srcuri:
         lines_before.append('# No information for SRC_URI yet (only an external source tree was specified)')

index d95cb08..345cabb 100644
@@ -570,7 +570,8 @@ class DevtoolTests(DevtoolBase):
         self.track_for_cleanup(self.workspacedir)
         self.add_command_to_tearDown('bitbake-layers remove-layer */workspace')
         # (don't bother with cleaning the recipe on teardown, we won't be building it)
-        result = runCmd('devtool modify %s -x %s' % (testrecipe, tempdir))
+        # We don't use -x here so that we test the behaviour of devtool modify without it
+        result = runCmd('devtool modify %s %s' % (testrecipe, tempdir))
         # Check git repo
         self._check_src_repo(tempdir)
         # Add a couple of commits

index 262ba09..f1b2e12 100644
@@ -685,11 +685,11 @@ def modify(args, config, basepath, workspace):
     else:
         srctree = get_default_srctree(config, args.recipename)
 
-    if not args.extract and not os.path.isdir(srctree):
-        raise DevtoolError("directory %s does not exist or not a directory "
-                           "(specify -x to extract source from recipe)" %
+    if args.no_extract and not os.path.isdir(srctree):
+        raise DevtoolError("--no-extract specified and source path %s does "
+                           "not exist or is not a directory" %
                            srctree)
-    if args.extract:
+    if not args.no_extract:
         tinfoil = _prep_extract_operation(config, basepath, args.recipename)
         if not tinfoil:
             # Error already shown

@@ -720,7 +720,7 @@ def modify(args, config, basepath, workspace):
 
     initial_rev = None
     commits = []
-    if args.extract:
+    if not args.no_extract:
         initial_rev = _extract_source(srctree, False, args.branch, False, rd)
         if not initial_rev:
             return 1

@@ -1319,7 +1319,9 @@ def register_commands(subparsers, context):
     parser_modify.add_argument('recipename', help='Name of existing recipe to edit (just name - no version, path or extension)')
     parser_modify.add_argument('srctree', nargs='?', help='Path to external source tree. If not specified, a subdirectory of %s will be used.' % defsrctree)
     parser_modify.add_argument('--wildcard', '-w', action="store_true", help='Use wildcard for unversioned bbappend')
-    parser_modify.add_argument('--extract', '-x', action="store_true", help='Extract source as well')
+    group = parser_modify.add_mutually_exclusive_group()
+    group.add_argument('--extract', '-x', action="store_true", help='Extract source for recipe (default)')
+    group.add_argument('--no-extract', '-n', action="store_true", help='Do not extract source, expect it to exist')
     group = parser_modify.add_mutually_exclusive_group()
     group.add_argument('--same-dir', '-s', help='Build in same directory as source', action="store_true")
     group.add_argument('--no-same-dir', help='Force build in a separate build directory', action="store_true")

index f1b2e12..d12cc2e 100644
@@ -51,7 +51,7 @@ def add(args, config, basepath, workspace):
                 raise DevtoolError('URI specified as positional argument as well as -f/--fetch')
             args.fetchuri = args.recipename
             args.recipename = ''
-    elif '://' in args.srctree:
+    elif args.srctree and '://' in args.srctree:
         if not args.fetchuri:
             if args.fetch:
                 raise DevtoolError('URI specified as positional argument as well as -f/--fetch')

index ee27f8d..f6d7515 100644
@@ -251,7 +251,7 @@ def determine_from_url(srcuri):
     """Determine name and version from a URL"""
     pn = None
     pv = None
-    parseres = urlparse.urlparse(srcuri.lower())
+    parseres = urlparse.urlparse(srcuri.lower().split(';', 1)[0])
     if parseres.path:
         if 'github.com' in parseres.netloc:
             res = re.search(r'.*/(.*?)/archive/(.*)-final\.(tar|zip)', parseres.path)

index f6d7515..daf13fb 100644
@@ -291,10 +291,12 @@ def supports_srcrev(uri):
 
 def reformat_git_uri(uri):
     '''Convert any http[s]://....git URI into git://...;protocol=http[s]'''
-    res = re.match('(https?)://([^;]+\.git)(;.*)?$', uri)
-    if res:
-        # Need to switch the URI around so that the git fetcher is used
-        return 'git://%s;protocol=%s%s' % (res.group(2), res.group(1), res.group(3) or '')
+    checkuri = uri.split(';', 1)[0]
+    if checkuri.endswith('.git') or '/git/' in checkuri:
+        res = re.match('(https?)://([^;]+(\.git)?)(;.*)?$', uri)
+        if res:
+            # Need to switch the URI around so that the git fetcher is used
+            return 'git://%s;protocol=%s%s' % (res.group(2), res.group(1), res.group(4) or '')
     return uri
 
 def create_recipe(args):

index daf13fb..3e4bab8 100644
@@ -369,6 +369,7 @@ def create_recipe(args):
                     if len(splitline) > 1:
                         if splitline[0] == 'origin' and '://' in splitline[1]:
                             srcuri = reformat_git_uri(splitline[1])
+                            srcsubdir = 'git'
                             break
 
     if args.src_subdir:

index 345cabb..e5a2134 100644
@@ -201,6 +201,43 @@ class DevtoolTests(DevtoolBase):
             bindir = bindir[1:]
         self.assertTrue(os.path.isfile(os.path.join(installdir, bindir, 'pv')), 'pv binary not found in D')
 
+    def test_devtool_add_git_local(self):
+        # Fetch source from a remote URL, but do it outside of devtool
+        tempdir = tempfile.mkdtemp(prefix='devtoolqa')
+        self.track_for_cleanup(tempdir)
+        pn = 'dbus-wait'
+        # We choose an https:// git URL here to check rewriting the URL works
+        url = 'https://git.yoctoproject.org/git/dbus-wait'
+        # Force fetching to "noname" subdir so we verify we're picking up the name from autoconf
+        # instead of the directory name
+        result = runCmd('git clone %s noname' % url, cwd=tempdir)
+        srcdir = os.path.join(tempdir, 'noname')
+        self.assertTrue(os.path.isfile(os.path.join(srcdir, 'configure.ac')), 'Unable to find configure script in source directory')
+        # Test devtool add
+        self.track_for_cleanup(self.workspacedir)
+        self.add_command_to_tearDown('bitbake-layers remove-layer */workspace')
+        # Don't specify a name since we should be able to auto-detect it
+        result = runCmd('devtool add %s' % srcdir)
+        self.assertTrue(os.path.exists(os.path.join(self.workspacedir, 'conf', 'layer.conf')), 'Workspace directory not created')
+        # Check the recipe name is correct
+        recipefile = get_bb_var('FILE', pn)
+        self.assertIn('%s_git.bb' % pn, recipefile, 'Recipe file incorrectly named')
+        self.assertIn(recipefile, result.output)
+        # Test devtool status
+        result = runCmd('devtool status')
+        self.assertIn(pn, result.output)
+        self.assertIn(srcdir, result.output)
+        self.assertIn(recipefile, result.output)
+        checkvars = {}
+        checkvars['LICENSE'] = 'GPLv2'
+        checkvars['LIC_FILES_CHKSUM'] = 'file://COPYING;md5=b234ee4d69f5fce4486a80fdaf4a4263'
+        checkvars['S'] = '${WORKDIR}/git'
+        checkvars['PV'] = '0.1+git${SRCPV}'
+        checkvars['SRC_URI'] = 'git://git.yoctoproject.org/git/dbus-wait;protocol=https'
+        checkvars['SRCREV'] = '${AUTOREV}'
+        checkvars['DEPENDS'] = set(['dbus'])
+        self._test_recipe_contents(recipefile, checkvars, [])
+
     @testcase(1162)
     def test_devtool_add_library(self):
         # We don't have the ability to pick up this dependency automatically yet...