devtool / recipetool fixes

index cad5015..05e0faa 100644
@@ -415,16 +415,24 @@ class GitApplyTree(PatchTree):
             out = runcmd(["sh", "-c", " ".join(shellcmd)], tree)
             if out:
                 for srcfile in out.split():
-                    patchlines = []
-                    outfile = None
-                    with open(srcfile, 'r') as f:
-                        for line in f:
-                            if line.startswith(GitApplyTree.patch_line_prefix):
-                                outfile = line.split()[-1].strip()
-                                continue
-                            if line.startswith(GitApplyTree.ignore_commit_prefix):
-                                continue
-                            patchlines.append(line)
+                    for encoding in ['utf-8', 'latin-1']:
+                        patchlines = []
+                        outfile = None
+                        try:
+                            with open(srcfile, 'r', encoding=encoding) as f:
+                                for line in f:
+                                    if line.startswith(GitApplyTree.patch_line_prefix):
+                                        outfile = line.split()[-1].strip()
+                                        continue
+                                    if line.startswith(GitApplyTree.ignore_commit_prefix):
+                                        continue
+                                    patchlines.append(line)
+                        except UnicodeDecodeError:
+                            continue
+                        break
+                    else:
+                        raise PatchError('Unable to find a character encoding to decode %s' % srcfile)
+
                     if not outfile:
                         outfile = os.path.basename(srcfile)
                     with open(os.path.join(outdir, outfile), 'w') as of:

index e7dd8af..a0d78dd 100644
@@ -400,8 +400,16 @@ def get_recipe_local_files(d, patches=False):
                     bb.utils.exec_flat_python_func('patch_path', uri, fetch, '')):
                 continue
             # Skip files that are referenced by absolute path
-            if not os.path.isabs(fetch.ud[uri].basepath):
-                ret[fetch.ud[uri].basepath] = fetch.localpath(uri)
+            fname = fetch.ud[uri].basepath
+            if os.path.isabs(fname):
+                continue
+            # Handle subdir=
+            subdir = fetch.ud[uri].parm.get('subdir', '')
+            if subdir:
+                if os.path.isabs(subdir):
+                    continue
+                fname = os.path.join(subdir, fname)
+            ret[fname] = fetch.localpath(uri)
     return ret
 
 

index 98451da..baef23e 100644
@@ -307,6 +307,13 @@ def _move_file(src, dst):
         bb.utils.mkdirhier(dst_d)
     shutil.move(src, dst)
 
+def _copy_file(src, dst):
+    """Copy a file. Creates all the directory components of destination path."""
+    dst_d = os.path.dirname(dst)
+    if dst_d:
+        bb.utils.mkdirhier(dst_d)
+    shutil.copy(src, dst)
+
 def _git_ls_tree(repodir, treeish='HEAD', recursive=False):
     """List contents of a git treeish"""
     import bb

@@ -1050,6 +1057,23 @@ def _export_local_files(srctree, rd, destdir):
             elif fname != '.gitignore':
                 added[fname] = None
 
+        workdir = rd.getVar('WORKDIR', True)
+        s = rd.getVar('S', True)
+        if not s.endswith(os.sep):
+            s += os.sep
+
+        if workdir != s:
+            # Handle files where subdir= was specified
+            for fname in list(existing_files.keys()):
+                # FIXME handle both subdir starting with BP and not?
+                fworkpath = os.path.join(workdir, fname)
+                if fworkpath.startswith(s):
+                    fpath = os.path.join(srctree, os.path.relpath(fworkpath, s))
+                    if os.path.exists(fpath):
+                        origpath = existing_files.pop(fname)
+                        if not filecmp.cmp(origpath, fpath):
+                            updated[fpath] = origpath
+
         removed = existing_files
     return (updated, added, removed)
 

@@ -1122,7 +1146,12 @@ def _update_recipe_srcrev(srctree, rd, appendlayerdir, wildcard_version, no_remo
             files_dir = _determine_files_dir(rd)
             for basepath, path in upd_f.items():
                 logger.info('Updating file %s' % basepath)
-                _move_file(os.path.join(local_files_dir, basepath), path)
+                if os.path.isabs(basepath):
+                    # Original file (probably with subdir pointing inside source tree)
+                    # so we do not want to move it, just copy
+                    _copy_file(basepath, path)
+                else:
+                    _move_file(os.path.join(local_files_dir, basepath), path)
                 update_srcuri= True
             for basepath, path in new_f.items():
                 logger.info('Adding new file %s' % basepath)

@@ -1205,7 +1234,12 @@ def _update_recipe_patch(recipename, workspace, srctree, rd, appendlayerdir, wil
             # Update existing files
             for basepath, path in upd_f.items():
                 logger.info('Updating file %s' % basepath)
-                _move_file(os.path.join(local_files_dir, basepath), path)
+                if os.path.isabs(basepath):
+                    # Original file (probably with subdir pointing inside source tree)
+                    # so we do not want to move it, just copy
+                    _copy_file(basepath, path)
+                else:
+                    _move_file(os.path.join(local_files_dir, basepath), path)
                 updatefiles = True
             for basepath, path in upd_p.items():
                 patchfn = os.path.join(patches_dir, basepath)

index f784f94..0ad748e 100644
@@ -570,7 +570,7 @@ class AutotoolsRecipeHandler(RecipeHandler):
             elif keyword == 'AX_LIB_TAGLIB':
                 deps.append('taglib')
             elif keyword == 'AX_PKG_SWIG':
-                deps.append('swig')
+                deps.append('swig-native')
             elif keyword == 'AX_PROG_XSLTPROC':
                 deps.append('libxslt-native')
             elif keyword == 'AX_WITH_CURSES':

index aff13cf..fb9806d 100644
@@ -504,8 +504,10 @@ class PythonRecipeHandler(RecipeHandler):
         for dep in scanned_deps:
             mapped = provided_packages.get(dep)
             if mapped:
+                logger.debug('Mapped %s to %s' % (dep, mapped))
                 mapped_deps.add(mapped)
             else:
+                logger.debug('Could not map %s' % dep)
                 unmapped_deps.add(dep)
         return mapped_deps, unmapped_deps
 

@@ -566,6 +568,8 @@ class PythonRecipeHandler(RecipeHandler):
                     continue
 
                 if fn.startswith(dynload_dir + os.sep):
+                    if '/.debug/' in fn:
+                        continue
                     base = os.path.basename(fn)
                     provided = base.split('.', 1)[0]
                     packages[provided] = os.path.basename(pkgdatafile)

index a08352e..7cbd614 100644
@@ -399,6 +399,11 @@ def create_recipe(args):
                     if '<html' in f.read(100).lower():
                         logger.error('Fetching "%s" returned a single HTML page - check the URL is correct and functional' % fetchuri)
                         sys.exit(1)
+        if os.path.exists(os.path.join(srctree, '.gitmodules')) and srcuri.startswith('git://'):
+            srcuri = 'gitsm://' + srcuri[6:]
+            logger.info('Fetching submodules...')
+            bb.process.run('git submodule init', cwd=srctree)
+            bb.process.run('git submodule update', cwd=srctree)
 
         if is_package(fetchuri):
             tmpfdir = tempfile.mkdtemp(prefix='recipetool-')

@@ -658,8 +663,11 @@ def create_recipe(args):
             # devtool looks for this specific exit code, so don't change it
             sys.exit(15)
         else:
-            if srcuri and srcuri.startswith(('git://', 'hg://', 'svn://')):
-                outfile = '%s_%s.bb' % (pn, srcuri.split(':', 1)[0])
+            if srcuri and srcuri.startswith(('gitsm://', 'git://', 'hg://', 'svn://')):
+                suffix = srcuri.split(':', 1)[0]
+                if suffix == 'gitsm':
+                    suffix = 'git'
+                outfile = '%s_%s.bb' % (pn, suffix)
             elif realpv:
                 outfile = '%s_%s.bb' % (pn, realpv)
             else:

index 7cbd614..89ab748 100644
@@ -765,7 +765,10 @@ def create_recipe(args):
         logger.info('Recipe %s has been created; further editing may be required to make it fully functional' % outfile)
 
     if tempsrc:
-        shutil.rmtree(tempsrc)
+        if args.keep_temp:
+            logger.info('Preserving temporary directory %s' % tempsrc)
+        else:
+            shutil.rmtree(tempsrc)
 
     return 0
 

@@ -1048,5 +1051,6 @@ def register_commands(subparsers):
     parser_create.add_argument('--also-native', help='Also add native variant (i.e. support building recipe for the build host as well as the target machine)', action='store_true')
     parser_create.add_argument('--src-subdir', help='Specify subdirectory within source tree to use', metavar='SUBDIR')
     parser_create.add_argument('-a', '--autorev', help='When fetching from a git repository, set SRCREV in the recipe to a floating revision instead of fixed', action="store_true")
+    parser_create.add_argument('--keep-temp', action="store_true", help='Keep temporary directory (for debugging)')
     parser_create.set_defaults(func=create_recipe)
 

index 89ab748..f1ce28c 100644
@@ -482,51 +482,8 @@ def create_recipe(args):
     # We need a blank line here so that patch_recipe_lines can rewind before the LICENSE comments
     lines_before.append('')
 
-    licvalues = guess_license(srctree_use)
-    lic_files_chksum = []
-    lic_unknown = []
-    if licvalues:
-        licenses = []
-        for licvalue in licvalues:
-            if not licvalue[0] in licenses:
-                licenses.append(licvalue[0])
-            lic_files_chksum.append('file://%s;md5=%s' % (licvalue[1], licvalue[2]))
-            if licvalue[0] == 'Unknown':
-                lic_unknown.append(licvalue[1])
-        lines_before.append('# WARNING: the following LICENSE and LIC_FILES_CHKSUM values are best guesses - it is')
-        lines_before.append('# your responsibility to verify that the values are complete and correct.')
-        if len(licvalues) > 1:
-            lines_before.append('#')
-            lines_before.append('# NOTE: multiple licenses have been detected; if that is correct you should separate')
-            lines_before.append('# these in the LICENSE value using & if the multiple licenses all apply, or | if there')
-            lines_before.append('# is a choice between the multiple licenses. If in doubt, check the accompanying')
-            lines_before.append('# documentation to determine which situation is applicable.')
-        if lic_unknown:
-            lines_before.append('#')
-            lines_before.append('# The following license files were not able to be identified and are')
-            lines_before.append('# represented as "Unknown" below, you will need to check them yourself:')
-            for licfile in lic_unknown:
-                lines_before.append('#   %s' % licfile)
-            lines_before.append('#')
-    else:
-        lines_before.append('# Unable to find any files that looked like license statements. Check the accompanying')
-        lines_before.append('# documentation and source headers and set LICENSE and LIC_FILES_CHKSUM accordingly.')
-        lines_before.append('#')
-        lines_before.append('# NOTE: LICENSE is being set to "CLOSED" to allow you to at least start building - if')
-        lines_before.append('# this is not accurate with respect to the licensing of the software being built (it')
-        lines_before.append('# will not be in most cases) you must specify the correct value before using this')
-        lines_before.append('# recipe for anything other than initial testing/development!')
-        licenses = ['CLOSED']
-    pkg_license = extravalues.pop('LICENSE', None)
-    if pkg_license:
-        if licenses == ['Unknown']:
-            lines_before.append('# NOTE: The following LICENSE value was determined from the original package metadata')
-            licenses = [pkg_license]
-        else:
-            lines_before.append('# NOTE: Original package metadata indicates license is: %s' % pkg_license)
-    lines_before.append('LICENSE = "%s"' % ' '.join(licenses))
-    lines_before.append('LIC_FILES_CHKSUM = "%s"' % ' \\\n                    '.join(lic_files_chksum))
-    lines_before.append('')
+    handled = []
+    licvalues = handle_license_vars(srctree_use, lines_before, handled, extravalues, tinfoil.config_data)
 
     classes = []
 

@@ -623,9 +580,6 @@ def create_recipe(args):
     handlers = [item[0] for item in handlers]
 
     # Apply the handlers
-    handled = []
-    handled.append(('license', licvalues))
-
     if args.binary:
         classes.append('bin_package')
         handled.append('buildsystem')

@@ -772,6 +726,55 @@ def create_recipe(args):
 
     return 0
 
+def handle_license_vars(srctree, lines_before, handled, extravalues, d):
+    licvalues = guess_license(srctree, d)
+    lic_files_chksum = []
+    lic_unknown = []
+    if licvalues:
+        licenses = []
+        for licvalue in licvalues:
+            if not licvalue[0] in licenses:
+                licenses.append(licvalue[0])
+            lic_files_chksum.append('file://%s;md5=%s' % (licvalue[1], licvalue[2]))
+            if licvalue[0] == 'Unknown':
+                lic_unknown.append(licvalue[1])
+        lines_before.append('# WARNING: the following LICENSE and LIC_FILES_CHKSUM values are best guesses - it is')
+        lines_before.append('# your responsibility to verify that the values are complete and correct.')
+        if len(licvalues) > 1:
+            lines_before.append('#')
+            lines_before.append('# NOTE: multiple licenses have been detected; if that is correct you should separate')
+            lines_before.append('# these in the LICENSE value using & if the multiple licenses all apply, or | if there')
+            lines_before.append('# is a choice between the multiple licenses. If in doubt, check the accompanying')
+            lines_before.append('# documentation to determine which situation is applicable.')
+        if lic_unknown:
+            lines_before.append('#')
+            lines_before.append('# The following license files were not able to be identified and are')
+            lines_before.append('# represented as "Unknown" below, you will need to check them yourself:')
+            for licfile in lic_unknown:
+                lines_before.append('#   %s' % licfile)
+            lines_before.append('#')
+    else:
+        lines_before.append('# Unable to find any files that looked like license statements. Check the accompanying')
+        lines_before.append('# documentation and source headers and set LICENSE and LIC_FILES_CHKSUM accordingly.')
+        lines_before.append('#')
+        lines_before.append('# NOTE: LICENSE is being set to "CLOSED" to allow you to at least start building - if')
+        lines_before.append('# this is not accurate with respect to the licensing of the software being built (it')
+        lines_before.append('# will not be in most cases) you must specify the correct value before using this')
+        lines_before.append('# recipe for anything other than initial testing/development!')
+        licenses = ['CLOSED']
+    pkg_license = extravalues.pop('LICENSE', None)
+    if pkg_license:
+        if licenses == ['Unknown']:
+            lines_before.append('# NOTE: The following LICENSE value was determined from the original package metadata')
+            licenses = [pkg_license]
+        else:
+            lines_before.append('# NOTE: Original package metadata indicates license is: %s' % pkg_license)
+    lines_before.append('LICENSE = "%s"' % ' '.join(licenses))
+    lines_before.append('LIC_FILES_CHKSUM = "%s"' % ' \\\n                    '.join(lic_files_chksum))
+    lines_before.append('')
+    handled.append(('license', licvalues))
+    return licvalues
+
 def get_license_md5sums(d, static_only=False):
     import bb.utils
     md5sums = {}

@@ -892,9 +895,9 @@ def crunch_license(licfile):
     license = crunched_md5sums.get(md5val, None)
     return license, md5val, lictext
 
-def guess_license(srctree):
+def guess_license(srctree, d):
     import bb
-    md5sums = get_license_md5sums(tinfoil.config_data)
+    md5sums = get_license_md5sums(d)
 
     licenses = []
     licspecs = ['*LICEN[CS]E*', 'COPYING*', '*[Ll]icense*', 'LEGAL*', '[Ll]egal*', '*GPL*', 'README.lic*', 'COPYRIGHT*', '[Cc]opyright*']

index 55c7c3e..fce4c11 100644
@@ -16,6 +16,10 @@ def npm_oe_arch_map(target_arch, d):
 NPM_ARCH ?= "${@npm_oe_arch_map(d.getVar('TARGET_ARCH', True), d)}"
 
 npm_do_compile() {
+	# Copy in any additionally fetched modules
+	if [ -d ${WORKDIR}/node_modules ] ; then
+		cp -a ${WORKDIR}/node_modules ${S}/
+	fi
 	# changing the home directory to the working directory, the .npmrc will
 	# be created in this directory
 	export HOME=${WORKDIR}

138 lines not on this page · raw message

index f1ce28c..b6de999 100644
@@ -715,7 +715,13 @@ def create_recipe(args):
         sys.stdout.write('\n'.join(outlines) + '\n')
     else:
         with open(outfile, 'w') as f:
-            f.write('\n'.join(outlines) + '\n')
+            lastline = None
+            for line in outlines:
+                if not lastline and not line:
+                    # Skip extra blank lines
+                    continue
+                f.write('%s\n' % line)
+                lastline = line
         logger.info('Recipe %s has been created; further editing may be required to make it fully functional' % outfile)
 
     if tempsrc: