Developer workflow tools

index bb208a3..b3737bb 100644
@@ -19,7 +19,6 @@ SDKTARGETSYSROOT = "${SDKPATH}/sysroots/${REAL_MULTIMACH_TARGET_SYS}"
 
 inherit cross-canadian
 
-do_generate_content[nostamp] = "1"
 do_generate_content[cleandirs] = "${SDK_OUTPUT}"
 do_generate_content[dirs] = "${SDK_OUTPUT}/${SDKPATH}"
 python do_generate_content() {

@@ -58,7 +57,6 @@ create_sdk_files() {
 	toolchain_create_sdk_version ${SDK_OUTPUT}/${SDKPATH}/version-${REAL_MULTIMACH_TARGET_SYS}
 }
 
-do_install[nostamp] = "1"
 do_install() {
     install -d ${D}/${SDKPATH}
     install -m 0644 -t ${D}/${SDKPATH} ${SDK_OUTPUT}/${SDKPATH}/*

index 89cce40..692fbac 100644
@@ -1389,32 +1389,11 @@ python package_do_shlibs() {
 
     pkgdest = d.getVar('PKGDEST', True)
 
-    shlibs_dirs = d.getVar('SHLIBSDIRS', True).split()
     shlibswork_dir = d.getVar('SHLIBSWORKDIR', True)
 
     # Take shared lock since we're only reading, not writing
     lf = bb.utils.lockfile(d.expand("${PACKAGELOCK}"))
 
-    def read_shlib_providers():
-        list_re = re.compile('^(.*)\.list$')
-        # Go from least to most specific since the last one found wins
-        for dir in reversed(shlibs_dirs):
-            bb.debug(2, "Reading shlib providers in %s" % (dir))
-            if not os.path.exists(dir):
-                continue
-            for file in os.listdir(dir):
-                m = list_re.match(file)
-                if m:
-                    dep_pkg = m.group(1)
-                    fd = open(os.path.join(dir, file))
-                    lines = fd.readlines()
-                    fd.close()
-                    for l in lines:
-                        s = l.strip().split(":")
-                        if s[0] not in shlib_provider:
-                            shlib_provider[s[0]] = {}
-                        shlib_provider[s[0]][s[1]] = (dep_pkg, s[2])
-
     def linux_so(file, needed, sonames, renames, pkgver):
         needs_ldconfig = False
         ldir = os.path.dirname(file).replace(pkgdest + "/" + pkg, '')

@@ -1511,8 +1490,7 @@ python package_do_shlibs() {
         use_ldconfig = False
 
     needed = {}
-    shlib_provider = {}
-    read_shlib_providers()
+    shlib_provider = oe.package.read_shlib_providers(d)
 
     for pkg in packages.split():
         private_libs = d.getVar('PRIVATE_LIBS_' + pkg, True) or d.getVar('PRIVATE_LIBS', True) or ""

index f8b5322..ea6feaa 100644
@@ -97,3 +97,29 @@ def filedeprunner(arg):
         raise e
 
     return (pkg, provides, requires)
+
+
+def read_shlib_providers(d):
+    import re
+
+    shlib_provider = {}
+    shlibs_dirs = d.getVar('SHLIBSDIRS', True).split()
+    list_re = re.compile('^(.*)\.list$')
+    # Go from least to most specific since the last one found wins
+    for dir in reversed(shlibs_dirs):
+        bb.debug(2, "Reading shlib providers in %s" % (dir))
+        if not os.path.exists(dir):
+            continue
+        for file in os.listdir(dir):
+            m = list_re.match(file)
+            if m:
+                dep_pkg = m.group(1)
+                fd = open(os.path.join(dir, file))
+                lines = fd.readlines()
+                fd.close()
+                for l in lines:
+                    s = l.strip().split(":")
+                    if s[0] not in shlib_provider:
+                        shlib_provider[s[0]] = {}
+                    shlib_provider[s[0]][s[1]] = (dep_pkg, s[2])
+    return shlib_provider

index b085c9d..788f465 100644
@@ -219,7 +219,11 @@ class GitApplyTree(PatchTree):
             return _applypatchhelper(shellcmd, patch, force, reverse, run)
         except CmdError:
             shellcmd = ["git", "--git-dir=.", "apply", "-p%s" % patch['strippath']]
-            return _applypatchhelper(shellcmd, patch, force, reverse, run)
+            try:
+                output = _applypatchhelper(shellcmd, patch, force, reverse, run)
+            except CmdError:
+                output = PatchTree._applypatch(self, patch, force, reverse, run)
+            return output
 
 
 class QuiltTree(PatchSet):

index 788f465..2d56ba4 100644
@@ -202,6 +202,78 @@ class GitApplyTree(PatchTree):
     def __init__(self, dir, d):
         PatchTree.__init__(self, dir, d)
 
+    @staticmethod
+    def extractPatchHeader(patchfile):
+        """
+        Extract just the header lines from the top of a patch file
+        """
+        lines = []
+        with open(patchfile, 'r') as f:
+            for line in f.readlines():
+                if line.startswith('Index: ') or line.startswith('diff -') or line.startswith('---'):
+                    break
+                lines.append(line)
+        return lines
+
+    @staticmethod
+    def prepareCommit(patchfile):
+        """
+        Prepare a git commit command line based on the header from a patch file
+        (typically this is useful for patches that cannot be applied with "git am" due to formatting)
+        """
+        import tempfile
+        import re
+        author_re = re.compile('[\S ]+ <\S+@\S+\.\S+>')
+        # Process patch header and extract useful information
+        lines = GitApplyTree.extractPatchHeader(patchfile)
+        outlines = []
+        author = None
+        date = None
+        for line in lines:
+            if line.startswith('Subject: '):
+                subject = line.split(':', 1)[1]
+                # Remove any [PATCH][oe-core] etc.
+                subject = re.sub(r'\[.+?\]\s*', '', subject)
+                outlines.insert(0, '%s\n\n' % subject.strip())
+                continue
+            if line.startswith('From: ') or line.startswith('Author: '):
+                authorval = line.split(':', 1)[1].strip().replace('"', '')
+                # git is fussy about author formatting i.e. it must be Name <email@domain>
+                if author_re.match(authorval):
+                    author = authorval
+                    continue
+            if line.startswith('Date: '):
+                if date is None:
+                    dateval = line.split(':', 1)[1].strip()
+                    # Very crude check for date format, since git will blow up if it's not in the right
+                    # format. Without e.g. a python-dateutils dependency we can't do a whole lot more
+                    if len(dateval) > 12:
+                        date = dateval
+                continue
+            if line.startswith('Signed-off-by: '):
+                authorval = line.split(':', 1)[1].strip().replace('"', '')
+                # git is fussy about author formatting i.e. it must be Name <email@domain>
+                if author_re.match(authorval):
+                    author = authorval
+            outlines.append(line)
+        # Add a pointer to the original patch file name
+        if outlines and outlines[-1].strip():
+            outlines.append('\n')
+        outlines.append('(from original patch: %s)\n' % os.path.basename(patchfile))
+        # Write out commit message to a file
+        with tempfile.NamedTemporaryFile('w', delete=False) as tf:
+            tmpfile = tf.name
+            for line in outlines:
+                tf.write(line)
+        # Prepare git command
+        cmd = ["git", "commit", "-F", tmpfile]
+        # git doesn't like plain email addresses as authors
+        if author and '<' in author:
+            cmd.append('--author="%s"' % author)
+        if date:
+            cmd.append('--date="%s"' % date)
+        return (tmpfile, cmd)
+
     def _applypatch(self, patch, force = False, reverse = False, run = True):
         def _applypatchhelper(shellcmd, patch, force = False, reverse = False, run = True):
             if reverse:

@@ -218,11 +290,25 @@ class GitApplyTree(PatchTree):
             shellcmd = ["git", "--work-tree=.", "am", "-3", "-p%s" % patch['strippath']]
             return _applypatchhelper(shellcmd, patch, force, reverse, run)
         except CmdError:
+            # Fall back to git apply
             shellcmd = ["git", "--git-dir=.", "apply", "-p%s" % patch['strippath']]
             try:
                 output = _applypatchhelper(shellcmd, patch, force, reverse, run)
             except CmdError:
+                # Fall back to patch
                 output = PatchTree._applypatch(self, patch, force, reverse, run)
+            # Add all files
+            shellcmd = ["git", "add", "-f", "."]
+            output += runcmd(["sh", "-c", " ".join(shellcmd)], self.dir)
+            # Exclude the patches directory
+            shellcmd = ["git", "reset", "HEAD", self.patchdir]
+            output += runcmd(["sh", "-c", " ".join(shellcmd)], self.dir)
+            # Commit the result
+            (tmpfile, shellcmd) = self.prepareCommit(patch['file'])
+            try:
+                output += runcmd(["sh", "-c", " ".join(shellcmd)], self.dir)
+            finally:
+                os.remove(tmpfile)
             return output
 
 

index 2d56ba4..2bf3065 100644
@@ -287,7 +287,7 @@ class GitApplyTree(PatchTree):
             return runcmd(["sh", "-c", " ".join(shellcmd)], self.dir)
 
         try:
-            shellcmd = ["git", "--work-tree=.", "am", "-3", "-p%s" % patch['strippath']]
+            shellcmd = ["git", "--work-tree=.", "am", "-3", "--keep-cr", "-p%s" % patch['strippath']]
             return _applypatchhelper(shellcmd, patch, force, reverse, run)
         except CmdError:
             # Fall back to git apply

index 2bf3065..5227781 100644
@@ -290,6 +290,12 @@ class GitApplyTree(PatchTree):
             shellcmd = ["git", "--work-tree=.", "am", "-3", "--keep-cr", "-p%s" % patch['strippath']]
             return _applypatchhelper(shellcmd, patch, force, reverse, run)
         except CmdError:
+            # Need to abort the git am, or we'll still be within it at the end
+            try:
+                shellcmd = ["git", "--work-tree=.", "am", "--abort"]
+                runcmd(["sh", "-c", " ".join(shellcmd)], self.dir)
+            except CmdError:
+                pass
             # Fall back to git apply
             shellcmd = ["git", "--git-dir=.", "apply", "-p%s" % patch['strippath']]
             try:

index 5227781..b838be8 100644
@@ -199,6 +199,8 @@ class PatchTree(PatchSet):
         self.Pop(all=True)
 
 class GitApplyTree(PatchTree):
+    patch_line_prefix = '%% original patch'
+
     def __init__(self, dir, d):
         PatchTree.__init__(self, dir, d)
 

@@ -256,10 +258,6 @@ class GitApplyTree(PatchTree):
                 if author_re.match(authorval):
                     author = authorval
             outlines.append(line)
-        # Add a pointer to the original patch file name
-        if outlines and outlines[-1].strip():
-            outlines.append('\n')
-        outlines.append('(from original patch: %s)\n' % os.path.basename(patchfile))
         # Write out commit message to a file
         with tempfile.NamedTemporaryFile('w', delete=False) as tf:
             tmpfile = tf.name

@@ -274,7 +272,35 @@ class GitApplyTree(PatchTree):
             cmd.append('--date="%s"' % date)
         return (tmpfile, cmd)
 
+    @staticmethod
+    def extractPatches(tree, startcommit, outdir):
+        import tempfile
+        import shutil
+        tempdir = tempfile.mkdtemp(prefix='oepatch')
+        try:
+            shellcmd = ["git", "format-patch", startcommit, "-o", tempdir]
+            out = runcmd(["sh", "-c", " ".join(shellcmd)], tree)
+            if out:
+                for srcfile in out.split():
+                    patchlines = []
+                    outfile = None
+                    with open(srcfile, 'r') as f:
+                        for line in f:
+                            if line.startswith(GitApplyTree.patch_line_prefix):
+                                outfile = line.split()[-1].strip()
+                                continue
+                            patchlines.append(line)
+                    if not outfile:
+                        outfile = os.path.basename(srcfile)
+                    with open(os.path.join(outdir, outfile), 'w') as of:
+                        for line in patchlines:
+                            of.write(line)
+        finally:
+            shutil.rmtree(tempdir)
+
     def _applypatch(self, patch, force = False, reverse = False, run = True):
+        import shutil
+
         def _applypatchhelper(shellcmd, patch, force = False, reverse = False, run = True):
             if reverse:
                 shellcmd.append('-R')

@@ -286,36 +312,62 @@ class GitApplyTree(PatchTree):
 
             return runcmd(["sh", "-c", " ".join(shellcmd)], self.dir)
 
+        # Add hooks which add a pointer to the original patch file name in the commit message
+        commithook = os.path.join(self.dir, '.git', 'hooks', 'commit-msg')
+        commithook_backup = commithook + '.devtool-orig'
+        applyhook = os.path.join(self.dir, '.git', 'hooks', 'applypatch-msg')
+        applyhook_backup = applyhook + '.devtool-orig'
+        if os.path.exists(commithook):
+            shutil.move(commithook, commithook_backup)
+        if os.path.exists(applyhook):
+            shutil.move(applyhook, applyhook_backup)
+        with open(commithook, 'w') as f:
+            # NOTE: the formatting here is significant; if you change it you'll also need to
+            # change other places which read it back
+            f.write('echo >> $1\n')
+            f.write('echo "%s: $PATCHFILE" >> $1\n' % GitApplyTree.patch_line_prefix)
+        os.chmod(commithook, 0755)
+        shutil.copy2(commithook, applyhook)
         try:
-            shellcmd = ["git", "--work-tree=.", "am", "-3", "--keep-cr", "-p%s" % patch['strippath']]
-            return _applypatchhelper(shellcmd, patch, force, reverse, run)
-        except CmdError:
-            # Need to abort the git am, or we'll still be within it at the end
-            try:
-                shellcmd = ["git", "--work-tree=.", "am", "--abort"]
-                runcmd(["sh", "-c", " ".join(shellcmd)], self.dir)
-            except CmdError:
-                pass
-            # Fall back to git apply
-            shellcmd = ["git", "--git-dir=.", "apply", "-p%s" % patch['strippath']]
+            patchfilevar = 'PATCHFILE="%s"' % os.path.basename(patch['file'])
             try:
-                output = _applypatchhelper(shellcmd, patch, force, reverse, run)
+                shellcmd = [patchfilevar, "git", "--work-tree=.", "am", "-3", "--keep-cr", "-p%s" % patch['strippath']]
+                return _applypatchhelper(shellcmd, patch, force, reverse, run)
             except CmdError:
-                # Fall back to patch
-                output = PatchTree._applypatch(self, patch, force, reverse, run)
-            # Add all files
-            shellcmd = ["git", "add", "-f", "."]
-            output += runcmd(["sh", "-c", " ".join(shellcmd)], self.dir)
-            # Exclude the patches directory
-            shellcmd = ["git", "reset", "HEAD", self.patchdir]
-            output += runcmd(["sh", "-c", " ".join(shellcmd)], self.dir)
-            # Commit the result
-            (tmpfile, shellcmd) = self.prepareCommit(patch['file'])
-            try:
+                # Need to abort the git am, or we'll still be within it at the end
+                try:
+                    shellcmd = ["git", "--work-tree=.", "am", "--abort"]
+                    runcmd(["sh", "-c", " ".join(shellcmd)], self.dir)
+                except CmdError:
+                    pass
+                # Fall back to git apply
+                shellcmd = ["git", "--git-dir=.", "apply", "-p%s" % patch['strippath']]
+                try:
+                    output = _applypatchhelper(shellcmd, patch, force, reverse, run)
+                except CmdError:
+                    # Fall back to patch
+                    output = PatchTree._applypatch(self, patch, force, reverse, run)
+                # Add all files
+                shellcmd = ["git", "add", "-f", "."]
+                output += runcmd(["sh", "-c", " ".join(shellcmd)], self.dir)
+                # Exclude the patches directory
+                shellcmd = ["git", "reset", "HEAD", self.patchdir]
                 output += runcmd(["sh", "-c", " ".join(shellcmd)], self.dir)
-            finally:
-                os.remove(tmpfile)
-            return output
+                # Commit the result
+                (tmpfile, shellcmd) = self.prepareCommit(patch['file'])
+                try:
+                    shellcmd.insert(0, patchfilevar)
+                    output += runcmd(["sh", "-c", " ".join(shellcmd)], self.dir)
+                finally:
+                    os.remove(tmpfile)
+                return output
+        finally:
+            os.remove(commithook)
+            os.remove(applyhook)
+            if os.path.exists(commithook_backup):
+                shutil.move(commithook_backup, commithook)
+            if os.path.exists(applyhook_backup):
+                shutil.move(applyhook_backup, applyhook)
 
 
 class QuiltTree(PatchSet):

new file mode 100644
index 0000000..1758dce
@@ -0,0 +1,279 @@
+# Utility functions for reading and modifying recipes
+#
+# Some code borrowed from the OE layer index
+#
+# Copyright (C) 2013-2014 Intel Corporation
+#
+
+import sys
+import os
+import os.path
+import tempfile
+import textwrap
+import difflib
+import utils
+import shutil
+import re
+from collections import OrderedDict, defaultdict
+
+
+# Help us to find places to insert values
+recipe_progression = ['SUMMARY', 'DESCRIPTION', 'HOMEPAGE', 'BUGTRACKER', 'SECTION', 'LICENSE', 'LIC_FILES_CHKSUM', 'PROVIDES', 'DEPENDS', 'PR', 'PV', 'SRC_URI', 'do_fetch', 'do_unpack', 'do_patch', 'EXTRA_OECONF', 'do_configure', 'EXTRA_OEMAKE', 'do_compile', 'do_install', 'do_populate_sysroot', 'INITSCRIPT', 'USERADD', 'GROUPADD', 'PACKAGES', 'FILES', 'RDEPENDS', 'RRECOMMENDS', 'RSUGGESTS', 'RPROVIDES', 'RREPLACES', 'RCONFLICTS', 'ALLOW_EMPTY', 'do_package', 'do_deploy']
+# Variables that sometimes are a bit long but shouldn't be wrapped
+nowrap_vars = ['SUMMARY', 'HOMEPAGE', 'BUGTRACKER']
+list_vars = ['SRC_URI', 'LIC_FILES_CHKSUM']
+meta_vars = ['SUMMARY', 'DESCRIPTION', 'HOMEPAGE', 'BUGTRACKER', 'SECTION']
+
+
+def pn_to_recipe(cooker, pn):
+    """Convert a recipe name (PN) to the path to the recipe file"""
+    import bb.providers
+
+    if pn in cooker.recipecache.pkg_pn:
+        filenames = cooker.recipecache.pkg_pn[pn]
+        best = bb.providers.findBestProvider(pn, cooker.data, cooker.recipecache, cooker.recipecache.pkg_pn)
+        return best[3]
+    else:
+        return None
+
+
+def get_unavailable_reasons(cooker, pn):
+    """If a recipe could not be found, find out why if possible"""
+    import bb.taskdata
+    taskdata = bb.taskdata.TaskData(None, skiplist=cooker.skiplist)
+    return taskdata.get_reasons(pn)
+
+
+def parse_recipe(fn, d):
+    """Parse an individual recipe"""
+    import bb.cache
+    envdata = bb.cache.Cache.loadDataFull(fn, [], d)
+    return envdata
+
+
+def get_var_files(fn, varlist, d):
+    """Find the file in which each of a list of variables is set.
+    Note: requires variable history to be enabled when parsing.
+    """
+    envdata = parse_recipe(fn, d)
+    varfiles = {}
+    for v in varlist:
+        history = envdata.varhistory.variable(v)
+        files = []
+        for event in history:
+            if 'file' in event and not 'flag' in event:
+                files.append(event['file'])
+        if files:
+            actualfile = files[-1]
+        else:
+            actualfile = None
+        varfiles[v] = actualfile
+
+    return varfiles
+
+
+def patch_recipe_file(fn, values, patch=False, relpath=''):
+    """Update or insert variable values into a recipe file (assuming you
+       have already identified the exact file you want to update.)
+       Note that some manual inspection/intervention may be required
+       since this cannot handle all situations.
+    """
+    remainingnames = {}
+    for k in values.keys():
+        remainingnames[k] = recipe_progression.index(k) if k in recipe_progression else -1
+    remainingnames = OrderedDict(sorted(remainingnames.iteritems(), key=lambda x: x[1]))
+
+    with tempfile.NamedTemporaryFile('w', delete=False) as tf:
+        def outputvalue(name):
+            rawtext = '%s = "%s"\n' % (name, values[name])
+            if name in nowrap_vars:
+                tf.write(rawtext)
+            elif name in list_vars:
+                splitvalue = values[name].split()
+                if len(splitvalue) > 1:
+                    linesplit = ' \\\n' + (' ' * (len(name) + 4))
+                    tf.write('%s = "%s%s"\n' % (name, linesplit.join(splitvalue), linesplit))
+                else:
+                    tf.write(rawtext)
+            else:
+                wrapped = textwrap.wrap(rawtext)
+                for wrapline in wrapped[:-1]:
+                    tf.write('%s \\\n' % wrapline)
+                tf.write('%s\n' % wrapped[-1])
+
+        tfn = tf.name
+        with open(fn, 'r') as f:
+            # First runthrough - find existing names (so we know not to insert based on recipe_progression)
+            # Second runthrough - make the changes
+            existingnames = []
+            for runthrough in [1, 2]:
+                currname = None
+                for line in f:
+                    if not currname:
+                        insert = False
+                        for k in remainingnames.keys():
+                            for p in recipe_progression:
+                                if re.match('^%s[ ?:=]' % p, line):
+                                    if remainingnames[k] > -1 and recipe_progression.index(p) > remainingnames[k] and runthrough > 1 and not k in existingnames:
+                                        outputvalue(k)
+                                        del remainingnames[k]
+                                    break
+                        for k in remainingnames.keys():
+                            if re.match('^%s[ ?:=]' % k, line):
+                                currname = k
+                                if runthrough == 1:
+                                    existingnames.append(k)
+                                else:
+                                    del remainingnames[k]
+                                break
+                        if currname and runthrough > 1:
+                            outputvalue(currname)
+
+                    if currname:
+                        sline = line.rstrip()
+                        if not sline.endswith('\\'):
+                            currname = None
+                        continue
+                    if runthrough > 1:
+                        tf.write(line)
+                f.seek(0)
+        if remainingnames:
+            tf.write('\n')
+            for k in remainingnames.keys():
+                outputvalue(k)
+
+    with open(tfn, 'U') as f:
+        tolines = f.readlines()
+    if patch:
+        with open(fn, 'U') as f:
+            fromlines = f.readlines()
+        relfn = os.path.relpath(fn, relpath)
+        diff = difflib.unified_diff(fromlines, tolines, 'a/%s' % relfn, 'b/%s' % relfn)
+        os.remove(tfn)
+        return diff
+    else:
+        with open(fn, 'w') as f:
+            f.writelines(tolines)
+        os.remove(tfn)
+        return None
+
+def localise_file_vars(fn, varfiles, varlist):
+    """Given a list of variables and variable history (fetched with get_var_files())
+    find where each variable should be set/changed. This handles for example where a
+    recipe includes an inc file where variables might be changed - in most cases
+    we want to update the inc file when changing the variable value rather than adding
+    it to the recipe itself.
+    """
+    fndir = os.path.dirname(fn) + os.sep
+
+    first_meta_file = None
+    for v in meta_vars:
+        f = varfiles.get(v, None)
+        if f:
+            actualdir = os.path.dirname(f) + os.sep
+            if actualdir.startswith(fndir):
+                first_meta_file = f
+                break
+
+    filevars = defaultdict(list)
+    for v in varlist:
+        f = varfiles[v]
+        # Only return files that are in the same directory as the recipe or in some directory below there
+        # (this excludes bbclass files and common inc files that wouldn't be appropriate to set the variable
+        # in if we were going to set a value specific to this recipe)
+        if f:
+            actualfile = f
+        else:
+            # Variable isn't in a file, if it's one of the "meta" vars, use the first file with a meta var in it
+            if first_meta_file:
+                actualfile = first_meta_file
+            else:
+                actualfile = fn
+
+        actualdir = os.path.dirname(actualfile) + os.sep
+        if not actualdir.startswith(fndir):
+            actualfile = fn
+        filevars[actualfile].append(v)
+
+    return filevars
+
+def patch_recipe(d, fn, varvalues, patch=False, relpath=''):
+    """Modify a list of variable values in the specified recipe. Handles inc files if
+    used by the recipe.
+    """
+    varlist = varvalues.keys()
+    varfiles = get_var_files(fn, varlist, d)
+    locs = localise_file_vars(fn, varfiles, varlist)
+    patches = []
+    for f,v in locs.iteritems():
+        vals = {k: varvalues[k] for k in v}
+        patchdata = patch_recipe_file(f, vals, patch, relpath)
+        if patch:
+            patches.append(patchdata)
+
+    if patch:
+        return patches
+    else:
+        return None
+
+
+
+def copy_recipe_files(d, tgt_dir, whole_dir=False, download=True):
+    """Copy (local) recipe files, including both files included via include/require,
+    and files referred to in the SRC_URI variable."""
+    import bb.fetch2
+    import oe.path
+
+    # FIXME need a warning if the unexpanded SRC_URI value contains variable references
+
+    uris = (d.getVar('SRC_URI', True) or "").split()
+    fetch = bb.fetch2.Fetch(uris, d)
+    if download:
+        fetch.download()
+
+    # Copy local files to target directory and gather any remote files
+    bb_dir = os.path.dirname(d.getVar('FILE', True)) + os.sep
+    remotes = []
+    includes = [path for path in d.getVar('BBINCLUDED', True).split() if
+                path.startswith(bb_dir) and os.path.exists(path)]
+    for path in fetch.localpaths() + includes:
+        # Only import files that are under the meta directory
+        if path.startswith(bb_dir):
+            if not whole_dir:
+                relpath = os.path.relpath(path, bb_dir)
+                subdir = os.path.join(tgt_dir, os.path.dirname(relpath))
+                if not os.path.exists(subdir):
+                    os.makedirs(subdir)
+                shutil.copy2(path, os.path.join(tgt_dir, relpath))
+        else:
+            remotes.append(path)
+    # Simply copy whole meta dir, if requested
+    if whole_dir:
+        shutil.copytree(bb_dir, tgt_dir)
+
+    return remotes
+
+
+def get_recipe_patches(d):
+    """Get a list of the patches included in SRC_URI within a recipe."""
+    patchfiles = []
+    # Execute src_patches() defined in patch.bbclass - this works since that class
+    # is inherited globally
+    patches = bb.utils.exec_flat_python_func('src_patches', d)
+    for patch in patches:
+        _, _, local, _, _, parm = bb.fetch.decodeurl(patch)
+        patchfiles.append(local)
+    return patchfiles
+
+
+def validate_pn(pn):
+    """Perform validation on a recipe name (PN) for a new recipe."""
+    reserved_names = ['forcevariable', 'append', 'prepend', 'remove']
+    if not re.match('[0-9a-z-]+', pn):
+        return 'Recipe name "%s" is invalid: only characters 0-9, a-z and - are allowed' % pn
+    elif pn in reserved_names:
+        return 'Recipe name "%s" is invalid: is a reserved keyword' % pn
+    elif pn.startswith('pn-'):
+        return 'Recipe name "%s" is invalid: names starting with "pn-" are reserved' % pn
+    return ''
+

index 802bc2f..d29c1b1 100644
@@ -138,9 +138,9 @@ def get_bb_var(var, target=None, postconfig=None):
     val = None
     bbenv = get_bb_env(target, postconfig=postconfig)
     for line in bbenv.splitlines():
-        if line.startswith(var + "="):
+        if line.startswith(var + "=") or line.startswith("export " + var + "="):
             val = line.split('=')[1]
-            val = val.replace('\"','')
+            val = val.strip('\"')
             break
     return val
 

index 2ac6274..4e429d7 100644
@@ -49,5 +49,8 @@ python () {
 
         for task in d.getVar("SRCTREECOVEREDTASKS", True).split():
             bb.build.deltask(task, d)
+
+        # Ensure compilation happens every time
+        d.setVarFlag('do_compile', 'nostamp', '1')
 }
 

new file mode 100644
index 0000000..e69de29

413 lines not on this page · raw message

319 lines not on this page · raw message

99 lines not on this page · raw message

new file mode 100644
index 0000000..e786126
@@ -0,0 +1,60 @@
+# Script utility functions
+#
+# Copyright (C) 2014 Intel Corporation
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+import sys
+import os
+import logging
+import glob
+
+def logger_create(name):
+    logger = logging.getLogger(name)
+    loggerhandler = logging.StreamHandler()
+    loggerhandler.setFormatter(logging.Formatter("%(levelname)s: %(message)s"))
+    logger.addHandler(loggerhandler)
+    logger.setLevel(logging.INFO)
+    return logger
+
+def logger_setup_color(logger, color='auto'):
+    from bb.msg import BBLogFormatter
+    console = logging.StreamHandler(sys.stdout)
+    formatter = BBLogFormatter("%(levelname)s: %(message)s")
+    console.setFormatter(formatter)
+    logger.handlers = [console]
+    if color == 'always' or (color=='auto' and console.stream.isatty()):
+        formatter.enable_color()
+
+
+def load_plugins(logger, plugins, pluginpath):
+    import imp
+
+    def load_plugin(name):
+        logger.debug('Loading plugin %s' % name)
+        fp, pathname, description = imp.find_module(name, [pluginpath])
+        try:
+            return imp.load_module(name, fp, pathname, description)
+        finally:
+            if fp:
+                fp.close()
+
+    logger.debug('Loading plugins from %s...' % pluginpath)
+    for fn in glob.glob(os.path.join(pluginpath, '*.py')):
+        name = os.path.splitext(os.path.basename(fn))[0]
+        if name != '__init__':
+            plugin = load_plugin(name)
+            if hasattr(plugin, 'plugin_init'):
+                plugin.plugin_init(plugins)
+                plugins.append(plugin)

new file mode 100755
index 0000000..d6e1b97
@@ -0,0 +1,255 @@
+#!/usr/bin/env python
+
+# OpenEmbedded Development tool
+#
+# Copyright (C) 2014 Intel Corporation
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+import sys
+import os
+import argparse
+import glob
+import re
+import ConfigParser
+import subprocess
+import logging
+
+basepath = ''
+workspace = {}
+config = None
+context = None
+
+
+scripts_path = os.path.dirname(os.path.realpath(__file__))
+lib_path = scripts_path + '/lib'
+sys.path = sys.path + [lib_path]
+import scriptutils
+logger = scriptutils.logger_create('devtool')
+
+plugins = []
+
+
+class ConfigHandler(object):
+    config_file = ''
+    config_obj = None
+    init_path = ''
+    workspace_path = ''
+
+    def __init__(self, filename):
+        self.config_file = filename
+        self.config_obj = ConfigParser.SafeConfigParser()
+
+    def get(self, section, option, default=None):
+        try:
+            ret = self.config_obj.get(section, option)
+        except (ConfigParser.NoOptionError, ConfigParser.NoSectionError):
+            if default != None:
+                ret = default
+            else:
+                raise
+        return ret
+
+    def read(self):
+        if os.path.exists(self.config_file):
+            self.config_obj.read(self.config_file)
+
+            if self.config_obj.has_option('General', 'init_path'):
+                pth = self.get('General', 'init_path')
+                self.init_path = os.path.join(basepath, pth)
+                if not os.path.exists(self.init_path):
+                    logger.error('init_path %s specified in config file cannot be found' % pth)
+                    return False
+        else:
+            self.config_obj.add_section('General')
+
+        self.workspace_path = self.get('General', 'workspace_path', os.path.join(basepath, 'workspace'))
+        return True
+
+
+    def write(self):
+        logger.debug('writing to config file %s' % self.config_file)
+        self.config_obj.set('General', 'workspace_path', self.workspace_path)
+        with open(self.config_file, 'w') as f:
+            self.config_obj.write(f)
+
+class Context:
+    def __init__(self, **kwargs):
+        self.__dict__.update(kwargs)
+
+
+def read_workspace():
+    global workspace
+    workspace = {}
+    if not os.path.exists(os.path.join(config.workspace_path, 'conf', 'layer.conf')):
+        if context.fixed_setup:
+            logger.error("workspace layer not set up")
+            sys.exit(1)
+        else:
+            logger.info('Creating workspace layer in %s' % config.workspace_path)
+            _create_workspace(config.workspace_path, config, basepath)
+
+    logger.debug('Reading workspace in %s' % config.workspace_path)
+    externalsrc_re = re.compile(r'^EXTERNALSRC(_pn-[a-zA-Z0-9-]*)? =.*$')
+    for fn in glob.glob(os.path.join(config.workspace_path, 'appends', '*.bbappend')):
+        pn = os.path.splitext(os.path.basename(fn))[0].split('_')[0]
+        with open(fn, 'r') as f:
+            for line in f:
+                if externalsrc_re.match(line.rstrip()):
+                    splitval = line.split('=', 2)
+                    workspace[pn] = splitval[1].strip('" \n\r\t')
+                    break
+
+def create_workspace(args, config, basepath, workspace):
+    if args.directory:
+        workspacedir = os.path.abspath(args.directory)
+    else:
+        workspacedir = os.path.abspath(os.path.join(basepath, 'workspace'))
+    _create_workspace(workspacedir, config, basepath, args.create_only)
+
+def _create_workspace(workspacedir, config, basepath, create_only=False):
+    import bb
+
+    confdir = os.path.join(workspacedir, 'conf')
+    if os.path.exists(os.path.join(confdir, 'layer.conf')):
+        logger.info('Specified workspace already set up, leaving as-is')
+    else:
+        # Add a config file
+        bb.utils.mkdirhier(confdir)
+        with open(os.path.join(confdir, 'layer.conf'), 'w') as f:
+            f.write('# ### workspace layer auto-generated by devtool ###\n')
+            f.write('BBPATH =. "$' + '{LAYERDIR}:"\n')
+            f.write('BBFILES += "$' + '{LAYERDIR}/recipes/*/*.bb \\\n')
+            f.write('            $' + '{LAYERDIR}/appends/*.bbappend"\n')
+            f.write('BBFILE_COLLECTIONS += "workspacelayer"\n')
+            f.write('BBFILE_PATTERN_workspacelayer = "^$' + '{LAYERDIR}/"\n')
+            f.write('BBFILE_PATTERN_IGNORE_EMPTY_workspacelayer = "1"\n')
+            f.write('BBFILE_PRIORITY_workspacelayer = "99"\n')
+        # Add a README file
+        with open(os.path.join(workspacedir, 'README'), 'w') as f:
+            f.write('This layer was created by the OpenEmbedded devtool utility in order to\n')
+            f.write('contain recipes and bbappends. In most instances you should use the\n')
+            f.write('devtool utility to manage files within it rather than modifying files\n')
+            f.write('directly (although recipes added with "devtool add" will often need\n')
+            f.write('direct modification.)\n')
+            f.write('\nIf you no longer need to use devtool you can remove the path to this\n')
+            f.write('workspace layer from your conf/bblayers.conf file (and then delete the\n')
+            f.write('layer, if you wish).\n')
+    if not create_only:
+        # Add the workspace layer to bblayers.conf
+        bblayers_conf = os.path.join(basepath, 'conf', 'bblayers.conf')
+        if not os.path.exists(bblayers_conf):
+            logger.error('Unable to find bblayers.conf')
+            return -1
+        bb.utils.edit_bblayers_conf(bblayers_conf, workspacedir, config.workspace_path)
+        if config.workspace_path != workspacedir:
+            # Update our config to point to the new location
+            config.workspace_path = workspacedir
+            config.write()
+
+
+def main():
+    global basepath
+    global config
+    global context
+
+    context = Context(fixed_setup=False)
+
+    # Default basepath
+    basepath = os.path.dirname(os.path.abspath(__file__))
+    pth = basepath
+    while pth != '' and pth != os.sep:
+        if os.path.exists(os.path.join(pth, '.devtoolbase')):
+            context.fixed_setup = True
+            basepath = pth
+            break
+        pth = os.path.dirname(pth)
+
+    parser = argparse.ArgumentParser(description="OpenEmbedded development tool",
+                                     epilog="Use %(prog)s <command> --help to get help on a specific command")
+    parser.add_argument('--basepath', help='Base directory of SDK / build directory')
+    parser.add_argument('-d', '--debug', help='Enable debug output', action='store_true')
+    parser.add_argument('-q', '--quiet', help='Print only errors', action='store_true')
+    parser.add_argument('--color', help='Colorize output', choices=['auto', 'always', 'never'], default='auto')
+
+    subparsers = parser.add_subparsers(dest="subparser_name")
+
+    if not context.fixed_setup:
+        parser_create_workspace = subparsers.add_parser('create-workspace', help='Set up a workspace')
+        parser_create_workspace.add_argument('directory', nargs='?', help='Directory for the workspace')
+        parser_create_workspace.add_argument('--create-only', action="store_true", help='Only create the workspace, do not alter configuration')
+        parser_create_workspace.set_defaults(func=create_workspace)
+
+    scriptutils.load_plugins(logger, plugins, os.path.join(scripts_path, 'lib', 'devtool'))
+    for plugin in plugins:
+        if hasattr(plugin, 'register_commands'):
+            plugin.register_commands(subparsers, context)
+
+    args = parser.parse_args()
+
+    if args.debug:
+        logger.setLevel(logging.DEBUG)
+    elif args.quiet:
+        logger.setLevel(logging.ERROR)
+
+    if args.basepath:
+        # Override
+        basepath = args.basepath
+    elif not context.fixed_setup:
+        basepath = os.environ.get('BUILDDIR')
+        if not basepath:
+            logger.error("This script can only be run after initialising the build environment (e.g. by using oe-init-build-env)")
+            sys.exit(1)
+
+    logger.debug('Using basepath %s' % basepath)
+
+    config = ConfigHandler(os.path.join(basepath, 'conf', 'devtool.conf'))
+    if not config.read():
+        return -1
+
+    bitbake_subdir = config.get('General', 'bitbake_subdir', '')
+    if bitbake_subdir:
+        # Normally set for use within the SDK
+        logger.debug('Using bitbake subdir %s' % bitbake_subdir)
+        sys.path.insert(0, os.path.join(basepath, bitbake_subdir, 'lib'))
+        core_meta_subdir = config.get('General', 'core_meta_subdir')
+        sys.path.insert(0, os.path.join(basepath, core_meta_subdir, 'lib'))
+    else:
+        # Standard location
+        import scriptpath
+        bitbakepath = scriptpath.add_bitbake_lib_path()
+        if not bitbakepath:
+            logger.error("Unable to find bitbake by searching parent directory of this script or PATH")
+            sys.exit(1)
+        logger.debug('Using standard bitbake path %s' % bitbakepath)
+        scriptpath.add_oe_lib_path()
+
+    scriptutils.logger_setup_color(logger, args.color)
+
+    if args.subparser_name != 'create-workspace':
+        read_workspace()
+
+    ret = args.func(args, config, basepath, workspace)
+
+    return ret
+
+
+if __name__ == "__main__":
+    try:
+        ret = main()
+    except Exception:
+        ret = 1
+        import traceback
+        traceback.print_exc(5)
+    sys.exit(ret)

78 lines not on this page · raw message

545 lines not on this page · raw message

new file mode 100644
index 0000000..bd23e95
@@ -0,0 +1,100 @@
+# Development tool - deploy/undeploy command plugin
+#
+# Copyright (C) 2014 Intel Corporation
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+import os
+import subprocess
+import logging
+from devtool import exec_build_env_command
+
+logger = logging.getLogger('devtool')
+
+def plugin_init(pluginlist):
+    pass
+
+
+def deploy(args, config, basepath, workspace):
+    import re
+    from devtool import exec_build_env_command
+
+    if not args.recipename in workspace:
+        logger.error("no recipe named %s in your workspace" % args.recipename)
+        return -1
+    try:
+        host, destdir = args.target.split(':')
+    except ValueError:
+        destdir = '/'
+    else:
+        args.target = host
+
+    deploy_dir = os.path.join(basepath, 'target_deploy', args.target)
+    deploy_file = os.path.join(deploy_dir, args.recipename + '.list')
+
+    if os.path.exists(deploy_file):
+        undeploy(args)
+
+    stdout, stderr = exec_build_env_command(config.init_path, basepath, 'bitbake -e %s' % args.recipename, shell=True)
+    recipe_outdir = re.search(r'^D="(.*)"', stdout, re.MULTILINE).group(1)
+    ret = subprocess.call('scp -qr %s/* %s:%s' % (recipe_outdir, args.target, destdir), shell=True)
+    if ret != 0:
+        return ret
+
+    logger.info('Successfully deployed %s' % recipe_outdir)
+
+    if not os.path.exists(deploy_dir):
+        os.makedirs(deploy_dir)
+
+    files_list = []
+    for root, _, files in os.walk(recipe_outdir):
+        for filename in files:
+            filename = os.path.relpath(os.path.join(root, filename), recipe_outdir)
+            files_list.append(os.path.join(destdir, filename))
+
+    with open(deploy_file, 'w') as fobj:
+        fobj.write('\n'.join(files_list))
+
+    return 0
+
+def undeploy(args, config, basepath, workspace):
+
+    deploy_file = os.path.join(basepath, 'target_deploy', args.target, args.recipename + '.list')
+    if not os.path.exists(deploy_file):
+         logger.error('%s has not been deployed' % args.recipename)
+         return -1
+
+    ret = subprocess.call("scp -q %s %s:/tmp" % (deploy_file, args.target), shell=True)
+    if ret != 0:
+        logger.error('Failed to copy %s to %s' % (deploy, args.target))
+        return -1
+
+    ret = subprocess.call("ssh %s 'xargs -n1 rm -f </tmp/%s'" % (args.target, os.path.basename(deploy_file)), shell=True)
+    if ret == 0:
+        logger.info('Successfully undeployed %s' % args.recipename)
+        os.remove(deploy_file)
+
+    return ret
+
+
+def register_commands(subparsers, context):
+    parser_deploy = subparsers.add_parser('deploy-target', help='Deploy recipe output files to live target machine')
+    parser_deploy.add_argument('recipename', help='Recipe to deploy')
+    parser_deploy.add_argument('target', help='Live target machine running an ssh server: user@hostname[:destdir]')
+    parser_deploy.set_defaults(func=deploy)
+
+    parser_undeploy = subparsers.add_parser('undeploy-target', help='Undeploy recipe output files in live target machine')
+    parser_undeploy.add_argument('recipename', help='Recipe to undeploy')
+    parser_undeploy.add_argument('target', help='Live target machine running an ssh server: user@hostname')
+    parser_undeploy.set_defaults(func=undeploy)

new file mode 100644
index 0000000..e8ff536
@@ -0,0 +1,239 @@
+import unittest
+import os
+import logging
+import re
+import shutil
+import tempfile
+import glob
+
+import oeqa.utils.ftools as ftools
+from oeqa.selftest.base import oeSelfTest
+from oeqa.utils.commands import runCmd, bitbake, get_bb_var
+from oeqa.utils.decorators import testcase
+
+class DevtoolTests(oeSelfTest):
+
+    def test_create_workspace(self):
+        # Check preconditions
+        workspacedir = os.path.join(self.builddir, 'workspace')
+        self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory')
+        result = runCmd('bitbake-layers show-layers')
+        self.assertTrue('/workspace' not in result.output, 'This test cannot be run with a workspace layer in bblayers.conf')
+        # Try creating a workspace layer with a specific path
+        tempdir = tempfile.mkdtemp(prefix='devtoolqa')
+        self.track_for_cleanup(tempdir)
+        result = runCmd('devtool create-workspace %s' % tempdir)
+        self.assertTrue(os.path.isfile(os.path.join(tempdir, 'conf', 'layer.conf')))
+        result = runCmd('bitbake-layers show-layers')
+        self.assertTrue(tempdir in result.output)
+        # Try creating a workspace layer with the default path
+        self.track_for_cleanup(workspacedir)
+        self.add_command_to_tearDown('bitbake-layers remove-layer */workspace')
+        result = runCmd('devtool create-workspace')
+        self.assertTrue(os.path.isfile(os.path.join(workspacedir, 'conf', 'layer.conf')))
+        result = runCmd('bitbake-layers show-layers')
+        self.assertTrue(tempdir not in result.output)
+        self.assertTrue(workspacedir in result.output)
+
+    def test_recipetool_create(self):
+        # Try adding a recipe
+        tempdir = tempfile.mkdtemp(prefix='devtoolqa')
+        self.track_for_cleanup(tempdir)
+        tempsrc = os.path.join(tempdir, 'srctree')
+        os.makedirs(tempsrc)
+        recipefile = os.path.join(tempdir, 'logrotate_3.8.7.bb')
+        srcuri = 'https://fedorahosted.org/releases/l/o/logrotate/logrotate-3.8.7.tar.gz'
+        result = runCmd('recipetool create -o %s %s -x %s' % (recipefile, srcuri, tempsrc))
+        self.assertTrue(os.path.isfile(recipefile))
+        checkvars = {}
+        checkvars['LICENSE'] = 'GPLv2'
+        checkvars['LIC_FILES_CHKSUM'] = 'file://COPYING;md5=18810669f13b87348459e611d31ab760'
+        checkvars['SRC_URI'] = 'https://fedorahosted.org/releases/l/o/logrotate/logrotate-${PV}.tar.gz'
+        checkvars['SRC_URI[md5sum]'] = '99e08503ef24c3e2e3ff74cc5f3be213'
+        checkvars['SRC_URI[sha256sum]'] = 'f6ba691f40e30e640efa2752c1f9499a3f9738257660994de70a45fe00d12b64'
+        with open(recipefile, 'r') as f:
+            for line in f:
+                if '=' in line:
+                    splitline = line.split('=', 1)
+                    var = splitline[0].rstrip()
+                    value = splitline[1].strip().strip('"')
+                    if var in checkvars:
+                        needvalue = checkvars.pop(var)
+                        self.assertEqual(value, needvalue)
+                if line.startswith('inherit '):
+                    inherits = line.split()[1:]
+
+        self.assertEqual(checkvars, {}, 'Some variables not found')
+
+    def test_recipetool_create_git(self):
+        # Try adding a recipe
+        tempdir = tempfile.mkdtemp(prefix='devtoolqa')
+        self.track_for_cleanup(tempdir)
+        tempsrc = os.path.join(tempdir, 'srctree')
+        os.makedirs(tempsrc)
+        recipefile = os.path.join(tempdir, 'libmatchbox.bb')
+        srcuri = 'git://git.yoctoproject.org/libmatchbox'
+        result = runCmd('recipetool create -o %s %s -x %s' % (recipefile, srcuri, tempsrc))
+        self.assertTrue(os.path.isfile(recipefile))
+        checkvars = {}
+        checkvars['LICENSE'] = 'LGPLv2.1'
+        checkvars['LIC_FILES_CHKSUM'] = 'file://COPYING;md5=7fbc338309ac38fefcd64b04bb903e34'
+        checkvars['S'] = '${WORKDIR}/git'
+        checkvars['PV'] = '1.0+git${SRCPV}'
+        checkvars['SRC_URI'] = srcuri
+        checkvars['DEPENDS'] = 'libpng pango libx11 libxext'
+        inherits = []
+        with open(recipefile, 'r') as f:
+            for line in f:
+                if '=' in line:
+                    splitline = line.split('=', 1)
+                    var = splitline[0].rstrip()
+                    value = splitline[1].strip().strip('"')
+                    if var in checkvars:
+                        needvalue = checkvars.pop(var)
+                        self.assertEqual(value, needvalue)
+                if line.startswith('inherit '):
+                    inherits = line.split()[1:]
+
+        self.assertEqual(checkvars, {}, 'Some variables not found')
+
+        self.assertIn('autotools', inherits, 'Missing inherit of autotools')
+        self.assertIn('pkgconfig', inherits, 'Missing inherit of pkgconfig')
+
+    def test_devtool_add(self):
+        # Check preconditions
+        workspacedir = os.path.join(self.builddir, 'workspace')
+        self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory')
+        # Fetch source
+        tempdir = tempfile.mkdtemp(prefix='devtoolqa')
+        self.track_for_cleanup(tempdir)
+        url = 'http://www.ivarch.com/programs/sources/pv-1.5.3.tar.bz2'
+        result = runCmd('wget %s' % url, cwd=tempdir)
+        result = runCmd('tar xfv pv-1.5.3.tar.bz2', cwd=tempdir)
+        srcdir = os.path.join(tempdir, 'pv-1.5.3')
+        self.assertTrue(os.path.isfile(os.path.join(srcdir, 'configure')))
+        # Test devtool add
+        self.track_for_cleanup(workspacedir)
+        self.add_command_to_tearDown('bitbake -c cleansstate pv')
+        self.add_command_to_tearDown('bitbake-layers remove-layer */workspace')
+        result = runCmd('devtool add pv %s' % srcdir)
+        self.assertTrue(os.path.exists(os.path.join(workspacedir, 'conf', 'layer.conf')), 'Workspace directory not created')
+        # Test devtool status
+        result = runCmd('devtool status')
+        self.assertIn('pv', result.output)
+        self.assertIn(srcdir, result.output)
+        # Clean up anything in the workdir/sysroot/sstate cache (have to do this *after* devtool add since the recipe only exists then)
+        bitbake('pv -c cleansstate')
+        # Test devtool build
+        result = runCmd('devtool build pv')
+        installdir = get_bb_var('D', 'pv')
+        self.assertTrue(installdir)
+        bindir = get_bb_var('bindir', 'pv')
+        self.assertTrue(bindir)
+        if bindir[0] == '/':
+            bindir = bindir[1:]
+        self.assertTrue(os.path.isfile(os.path.join(installdir, bindir, 'pv')), 'pv binary not found in D')
+
+    def test_devtool_modify(self):
+        # Check preconditions
+        workspacedir = os.path.join(self.builddir, 'workspace')
+        self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory')
+        # Clean up anything in the workdir/sysroot/sstate cache
+        bitbake('mdadm -c cleansstate')
+        # Try modifying a recipe
+        tempdir = tempfile.mkdtemp(prefix='devtoolqa')
+        self.track_for_cleanup(tempdir)
+        self.track_for_cleanup(workspacedir)
+        self.add_command_to_tearDown('bitbake-layers remove-layer */workspace')
+        self.add_command_to_tearDown('bitbake -c clean mdadm')
+        result = runCmd('devtool modify mdadm -x %s' % tempdir)
+        self.assertTrue(os.path.exists(os.path.join(tempdir, 'Makefile')), 'Extracted source could not be found')
+        self.assertTrue(os.path.isdir(os.path.join(tempdir, '.git')), 'git repository for external source tree not found')
+        self.assertTrue(os.path.exists(os.path.join(workspacedir, 'conf', 'layer.conf')), 'Workspace directory not created')
+        matches = glob.glob(os.path.join(workspacedir, 'appends', 'mdadm_*.bbappend'))
+        self.assertTrue(matches, 'bbappend not created')
+        # Test devtool status
+        result = runCmd('devtool status')
+        self.assertIn('mdadm', result.output)
+        self.assertIn(tempdir, result.output)
+        # Check git repo
+        result = runCmd('git status --porcelain', cwd=tempdir)
+        self.assertEqual(result.output.strip(), "", 'Created git repo is not clean')
+        result = runCmd('git symbolic-ref HEAD', cwd=tempdir)
+        self.assertEqual(result.output.strip(), "refs/heads/devtool", 'Wrong branch in git repo')
+        # Try building
+        bitbake('mdadm')
+        # Try making (minor) modifications to the source
+        result = runCmd("sed -i 's!^\.TH.*!.TH MDADM 8 \"\" v9.999-custom!' %s" % os.path.join(tempdir, 'mdadm.8.in'))
+        bitbake('mdadm -c package')
+        pkgd = get_bb_var('PKGD', 'mdadm')
+        self.assertTrue(pkgd)
+        mandir = get_bb_var('mandir', 'mdadm')
+        self.assertTrue(mandir)
+        if mandir[0] == '/':
+            mandir = mandir[1:]
+        with open(os.path.join(pkgd, mandir, 'man8', 'mdadm.8'), 'r') as f:
+            for line in f:
+                if line.startswith('.TH'):
+                    self.assertEqual(line.rstrip(), '.TH MDADM 8 "" v9.999-custom', 'man file not modified')
+        # Test devtool reset
+        result = runCmd('devtool reset mdadm')
+        result = runCmd('devtool status')
+        self.assertNotIn('mdadm', result.output)
+
+    def test_devtool_update_recipe(self):
+        # Check preconditions
+        workspacedir = os.path.join(self.builddir, 'workspace')
+        self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory')
+        testrecipe = 'minicom'
+        recipefile = get_bb_var('FILE', testrecipe)
+        result = runCmd('git status . --porcelain', cwd=os.path.dirname(recipefile))
+        self.assertEqual(result.output.strip(), "", '%s recipe is not clean' % testrecipe)
+        # First, modify a recipe
+        tempdir = tempfile.mkdtemp(prefix='devtoolqa')
+        self.track_for_cleanup(tempdir)
+        self.track_for_cleanup(workspacedir)
+        self.add_command_to_tearDown('bitbake-layers remove-layer */workspace')
+        # (don't bother with cleaning the recipe on teardown, we won't be building it)
+        result = runCmd('devtool modify %s -x %s' % (testrecipe, tempdir))
+        # Check git repo
+        self.assertTrue(os.path.isdir(os.path.join(tempdir, '.git')), 'git repository for external source tree not found')
+        result = runCmd('git status --porcelain', cwd=tempdir)
+        self.assertEqual(result.output.strip(), "", 'Created git repo is not clean')
+        result = runCmd('git symbolic-ref HEAD', cwd=tempdir)
+        self.assertEqual(result.output.strip(), "refs/heads/devtool", 'Wrong branch in git repo')
+        # Add a couple of commits
+        # FIXME: this only tests adding, need to also test update and remove
+        result = runCmd('echo "Additional line" >> README', cwd=tempdir)
+        result = runCmd('git commit -a -m "Change the README"', cwd=tempdir)
+        result = runCmd('echo "A new file" > devtool-new-file', cwd=tempdir)
+        result = runCmd('git add devtool-new-file', cwd=tempdir)
+        result = runCmd('git commit -m "Add a new file"', cwd=tempdir)
+        self.add_command_to_tearDown('cd %s; rm %s/*.patch; git checkout %s %s' % (os.path.dirname(recipefile), testrecipe, testrecipe, os.path.basename(recipefile)))
+        result = runCmd('devtool update-recipe %s' % testrecipe)
+        result = runCmd('git status . --porcelain', cwd=os.path.dirname(recipefile))
+        self.assertNotEqual(result.output.strip(), "", '%s recipe should be modified' % testrecipe)
+        status = result.output.splitlines()
+        self.assertEqual(len(status), 3, 'Less/more files modified than expected. Entire status:\n%s' % result.output)
+        for line in status:
+            if line.endswith('0001-Change-the-README.patch'):
+                self.assertEqual(line[:3], '?? ', 'Unexpected status in line: %s' % line)
+            elif line.endswith('0002-Add-a-new-file.patch'):
+                self.assertEqual(line[:3], '?? ', 'Unexpected status in line: %s' % line)
+            elif re.search('minicom_[^_]*.bb$', line):
+                self.assertEqual(line[:3], ' M ', 'Unexpected status in line: %s' % line)
+            else:
+                self.assertTrue(False, 'Unexpected modified file in status: %s' % line)
+
+    def test_devtool_extract(self):
+        # Check preconditions
+        workspacedir = os.path.join(self.builddir, 'workspace')
+        self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory')
+        tempdir = tempfile.mkdtemp(prefix='devtoolqa')
+        # Try devtool extract
+        self.track_for_cleanup(tempdir)
+        self.track_for_cleanup(workspacedir)
+        self.add_command_to_tearDown('bitbake-layers remove-layer */workspace')
+        result = runCmd('devtool extract remake %s' % tempdir)
+        self.assertTrue(os.path.exists(os.path.join(tempdir, 'Makefile.am')), 'Extracted source could not be found')
+        self.assertTrue(os.path.isdir(os.path.join(tempdir, '.git')), 'git repository for external source tree not found')