Tinfoil rework - OE-Core changes

index 1589feb..ae83aab 100644
@@ -29,18 +29,9 @@ meta_vars = ['SUMMARY', 'DESCRIPTION', 'HOMEPAGE', 'BUGTRACKER', 'SECTION']
 
 def pn_to_recipe(cooker, pn, mc=''):
     """Convert a recipe name (PN) to the path to the recipe file"""
-    import bb.providers
 
-    if pn in cooker.recipecaches[mc].pkg_pn:
-        best = bb.providers.findBestProvider(pn, cooker.data, cooker.recipecaches[mc], cooker.recipecaches[mc].pkg_pn)
-        return best[3]
-    elif pn in cooker.recipecaches[mc].providers:
-        filenames = cooker.recipecaches[mc].providers[pn]
-        eligible, foundUnique = bb.providers.filterProviders(filenames, pn, cooker.expanded_data, cooker.recipecaches[mc])
-        filename = eligible[0]
-        return filename
-    else:
-        return None
+    best = cooker.findBestProvider(pn, mc)
+    return best[3]
 
 
 def get_unavailable_reasons(cooker, pn):

index 5cd0f74..47cb6b7 100644
@@ -215,7 +215,7 @@ def runqemu(pn, ssh=True):
     import bb.build
 
     tinfoil = bb.tinfoil.Tinfoil()
-    tinfoil.prepare(False)
+    tinfoil.prepare(config_only=False, quiet=True)
     try:
         tinfoil.logger.setLevel(logging.WARNING)
         import oeqa.targetcontrol

index 47cb6b7..6e05995 100644
@@ -221,9 +221,7 @@ def runqemu(pn, ssh=True):
         import oeqa.targetcontrol
         tinfoil.config_data.setVar("TEST_LOG_DIR", "${WORKDIR}/testimage")
         tinfoil.config_data.setVar("TEST_QEMUBOOT_TIMEOUT", "1000")
-        import oe.recipeutils
-        recipefile = oe.recipeutils.pn_to_recipe(tinfoil.cooker, pn)
-        recipedata = oe.recipeutils.parse_recipe(tinfoil.cooker, recipefile, [])
+        recipedata = tinfoil.parse_recipe(pn)
 
         # The QemuRunner log is saved out, but we need to ensure it is at the right
         # log level (and then ensure that since it's a child of the BitBake logger,

index 31ecb65..99c5534 100644
@@ -120,34 +120,22 @@ def setup_tinfoil(config_only=False, basepath=None, tracking=False):
         os.chdir(orig_cwd)
     return tinfoil
 
-def get_recipe_file(cooker, pn):
-    """Find recipe file corresponding a package name"""
-    import oe.recipeutils
-    recipefile = oe.recipeutils.pn_to_recipe(cooker, pn)
-    if not recipefile:
-        skipreasons = oe.recipeutils.get_unavailable_reasons(cooker, pn)
-        if skipreasons:
-            logger.error('\n'.join(skipreasons))
-        else:
-            logger.error("Unable to find any recipe file matching %s" % pn)
-    return recipefile
-
 def parse_recipe(config, tinfoil, pn, appends, filter_workspace=True):
-    """Parse recipe of a package"""
-    import oe.recipeutils
-    recipefile = get_recipe_file(tinfoil.cooker, pn)
-    if not recipefile:
-        # Error already logged
+    """Parse the specified recipe"""
+    try:
+        recipefile = tinfoil.get_recipe_file(pn)
+    except bb.providers.NoProvider as e:
+        logger.error(str(e))
         return None
     if appends:
-        append_files = tinfoil.cooker.collection.get_file_appends(recipefile)
+        append_files = tinfoil.get_file_appends(recipefile)
         if filter_workspace:
             # Filter out appends from the workspace
             append_files = [path for path in append_files if
                             not path.startswith(config.workspace_path)]
     else:
         append_files = None
-    return oe.recipeutils.parse_recipe(tinfoil.cooker, recipefile, append_files)
+    return tinfoil.parse_recipe_file(recipefile, appends, append_files)
 
 def check_workspace_recipe(workspace, pn, checksrc=True, bbclassextend=False):
     """

index c4c7bf6..db7dffa 100644
@@ -156,7 +156,7 @@ def deploy(args, config, basepath, workspace):
     tinfoil = setup_tinfoil(basepath=basepath)
     try:
         try:
-            rd = oe.recipeutils.parse_recipe_simple(tinfoil.cooker, args.recipename, tinfoil.config_data)
+            rd = tinfoil.parse_recipe(args.recipename)
         except Exception as e:
             raise DevtoolError('Exception parsing recipe %s: %s' %
                             (args.recipename, e))

index e4d2a57..87d3f5d 100644
@@ -224,7 +224,7 @@ def add(args, config, basepath, workspace):
 
     tinfoil = setup_tinfoil(config_only=True, basepath=basepath)
     try:
-        rd = oe.recipeutils.parse_recipe(tinfoil.cooker, recipefile, None)
+        rd = tinfoil.parse_recipe_file(recipefile, False)
         if not rd:
             return 1
 

index a4239f1..52f9ab1 100644
@@ -320,7 +320,7 @@ def _create_new_recipe(newpv, md5, sha256, srcrev, srcbranch, workspace, tinfoil
         newvalues['SRC_URI[md5sum]'] = md5
         newvalues['SRC_URI[sha256sum]'] = sha256
 
-    rd = oe.recipeutils.parse_recipe(tinfoil.cooker, fullpath, None)
+    rd = tinfoil.parse_recipe_file(fullpath, False)
     oe.recipeutils.patch_recipe(rd, fullpath, newvalues)
 
     return fullpath, copied

index 1e0fc1e..3e85a0c 100644
@@ -97,25 +97,12 @@ def find_target_file(targetpath, d, pkglist=None):
                             recipes[targetpath].append('!%s' % pn)
     return recipes
 
-def _get_recipe_file(cooker, pn):
-    import oe.recipeutils
-    recipefile = oe.recipeutils.pn_to_recipe(cooker, pn)
-    if not recipefile:
-        skipreasons = oe.recipeutils.get_unavailable_reasons(cooker, pn)
-        if skipreasons:
-            logger.error('\n'.join(skipreasons))
-        else:
-            logger.error("Unable to find any recipe file matching %s" % pn)
-    return recipefile
-
 def _parse_recipe(pn, tinfoil):
-    import oe.recipeutils
-    recipefile = _get_recipe_file(tinfoil.cooker, pn)
-    if not recipefile:
-        # Error already logged
+    try:
+        rd = tinfoil.parse_recipe(pn)
+    except bb.providers.NoProvider as e:
+        logger.error(str(e))
         return None
-    append_files = tinfoil.cooker.collection.get_file_appends(recipefile)
-    rd = oe.recipeutils.parse_recipe(tinfoil.cooker, recipefile, append_files)
     return rd
 
 def determine_file_source(targetpath, rd):

index fbdd7bc..3760840 100644
@@ -39,18 +39,6 @@ def tinfoil_init(instance):
     tinfoil = instance
 
 
-def _get_recipe_file(cooker, pn):
-    import oe.recipeutils
-    recipefile = oe.recipeutils.pn_to_recipe(cooker, pn)
-    if not recipefile:
-        skipreasons = oe.recipeutils.get_unavailable_reasons(cooker, pn)
-        if skipreasons:
-            logger.error('\n'.join(skipreasons))
-        else:
-            logger.error("Unable to find any recipe file matching %s" % pn)
-    return recipefile
-
-
 def layer(layerpath):
     if not os.path.exists(os.path.join(layerpath, 'conf', 'layer.conf')):
         raise argparse.ArgumentTypeError('{0!r} must be a path to a valid layer'.format(layerpath))

@@ -60,7 +48,7 @@ def layer(layerpath):
 def newappend(args):
     import oe.recipeutils
 
-    recipe_path = _get_recipe_file(tinfoil.cooker, args.target)
+    recipe_path = tinfoil.get_recipe_file(args.target)
 
     rd = tinfoil.config_data.createCopy()
     rd.setVar('FILE', recipe_path)

index 85701c0..9de315a 100644
@@ -51,7 +51,7 @@ def setvar(args):
     if args.recipe_only:
         patches = [oe.recipeutils.patch_recipe_file(args.recipefile, varvalues, patch=args.patch)]
     else:
-        rd = oe.recipeutils.parse_recipe(tinfoil.cooker, args.recipefile, None)
+        rd = tinfoil.parse_recipe_file(args.recipefile, False)
         if not rd:
             return 1
         patches = oe.recipeutils.patch_recipe(rd, args.recipefile, varvalues, patch=args.patch)

index 024fe43..19673e6 100644
@@ -98,6 +98,9 @@ def get_lic_checksum_file_list(d):
     filelist = []
     lic_files = d.getVar("LIC_FILES_CHKSUM", True) or ''
     tmpdir = d.getVar("TMPDIR", True)
+    s = d.getVar("S", True)
+    b = d.getVar("B", True)
+    workdir = d.getVar("WORKDIR", True)
 
     urls = lic_files.split()
     for url in urls:

@@ -109,7 +112,7 @@ def get_lic_checksum_file_list(d):
                 raise bb.fetch.MalformedUrl(url)
 
             if path[0] == '/':
-                if path.startswith(tmpdir):
+                if path.startswith((tmpdir, s, b, workdir)):
                     continue
                 filelist.append(path + ":" + str(os.path.exists(path)))
         except bb.fetch.MalformedUrl:

index 2c1f58c..7ebae28 100644
@@ -10,6 +10,75 @@ PATCH_GIT_USER_EMAIL ?= "oe.patch@oe"
 
 inherit terminal
 
+python () {
+    if d.getVar('PATCHTOOL', True) == 'git' and d.getVar('PATCH_COMMIT_FUNCTIONS', True) == '1':
+        tasks = list(filter(lambda k: d.getVarFlag(k, "task", True), d.keys()))
+        extratasks = []
+        def follow_chain(task, endtask, chain=None):
+            if not chain:
+                chain = []
+            chain.append(task)
+            for othertask in tasks:
+                if othertask == task:
+                    continue
+                if task == endtask:
+                    for ctask in chain:
+                        if ctask not in extratasks:
+                            extratasks.append(ctask)
+                else:
+                    deps = d.getVarFlag(othertask, 'deps', False)
+                    if task in deps:
+                        follow_chain(othertask, endtask, chain)
+            chain.pop()
+        follow_chain('do_unpack', 'do_patch')
+        try:
+            extratasks.remove('do_unpack')
+        except ValueError:
+            # For some recipes do_unpack doesn't exist, ignore it
+            pass
+
+        d.appendVarFlag('do_patch', 'prefuncs', ' patch_task_patch_prefunc')
+        for task in extratasks:
+            d.appendVarFlag(task, 'postfuncs', ' patch_task_postfunc')
+}
+
+python patch_task_patch_prefunc() {
+    # Prefunc for do_patch
+    func = d.getVar('BB_RUNTASK', True)
+    srcsubdir = d.getVar('S', True)
+
+    patchdir = os.path.join(srcsubdir, 'patches')
+    if os.path.exists(patchdir):
+        if os.listdir(patchdir):
+            d.setVar('PATCH_HAS_PATCHES_DIR', '1')
+        else:
+            os.rmdir(patchdir)
+}
+
+python patch_task_postfunc() {
+    # Prefunc for task functions between do_unpack and do_patch
+    import oe.patch
+    import shutil
+    func = d.getVar('BB_RUNTASK', True)
+    srcsubdir = d.getVar('S', True)
+
+    if os.path.exists(srcsubdir):
+        if func == 'do_patch':
+            haspatches = (d.getVar('PATCH_HAS_PATCHES_DIR', True) == '1')
+            patchdir = os.path.join(srcsubdir, 'patches')
+            if os.path.exists(patchdir):
+                shutil.rmtree(patchdir)
+                if haspatches:
+                    stdout, _ = bb.process.run('git status --porcelain patches', cwd=srcsubdir)
+                    if stdout:
+                        bb.process.run('git checkout patches', cwd=srcsubdir)
+        stdout, _ = bb.process.run('git status --porcelain .', cwd=srcsubdir)
+        if stdout:
+            useroptions = []
+            oe.patch.GitApplyTree.gitCommandUserOptions(useroptions, d=d)
+            bb.process.run('git add .; git %s commit -a -m "Committing changes from %s\n\n%s"' % (' '.join(useroptions), func, oe.patch.GitApplyTree.ignore_commit_prefix + ' - from %s' % func), cwd=srcsubdir)
+}
+
 def src_patches(d, all=False, expand=True):
     workdir = d.getVar('WORKDIR', True)
     fetch = bb.fetch2.Fetch([], d)

index 87d3f5d..06c508c 100644
@@ -442,41 +442,6 @@ class BbTaskExecutor(object):
             self.executed.append(func)
 
 
-class PatchTaskExecutor(BbTaskExecutor):
-    def __init__(self, rdata):
-        import oe.patch
-        self.check_git = False
-        self.useroptions = []
-        oe.patch.GitApplyTree.gitCommandUserOptions(self.useroptions, d=rdata)
-        super(PatchTaskExecutor, self).__init__(rdata)
-
-    def exec_func(self, func, report):
-        from oe.patch import GitApplyTree
-        srcsubdir = self.rdata.getVar('S', True)
-        haspatches = False
-        if func == 'do_patch':
-            patchdir = os.path.join(srcsubdir, 'patches')
-            if os.path.exists(patchdir):
-                if os.listdir(patchdir):
-                    haspatches = True
-                else:
-                    os.rmdir(patchdir)
-
-        super(PatchTaskExecutor, self).exec_func(func, report)
-        if self.check_git and os.path.exists(srcsubdir):
-            if func == 'do_patch':
-                if os.path.exists(patchdir):
-                    shutil.rmtree(patchdir)
-                    if haspatches:
-                        stdout, _ = bb.process.run('git status --porcelain patches', cwd=srcsubdir)
-                        if stdout:
-                            bb.process.run('git checkout patches', cwd=srcsubdir)
-
-            stdout, _ = bb.process.run('git status --porcelain', cwd=srcsubdir)
-            if stdout:
-                bb.process.run('git add .; git %s commit -a -m "Committing changes from %s\n\n%s"' % (' '.join(self.useroptions), func, GitApplyTree.ignore_commit_prefix + ' - from %s' % func), cwd=srcsubdir)
-
-
 def _prep_extract_operation(config, basepath, recipename, tinfoil=None):
     """HACK: Ugly workaround for making sure that requirements are met when
        trying to extract a package. Returns the tinfoil instance to be used."""

@@ -563,7 +528,7 @@ def _extract_source(srctree, keep_temp, devbranch, sync, d):
             # We don't want to move the source to STAGING_KERNEL_DIR here
             crd.setVar('STAGING_KERNEL_DIR', '${S}')
 
-        task_executor = PatchTaskExecutor(crd)
+        task_executor = BbTaskExecutor(crd)
 
         crd.setVar('EXTERNALSRC_forcevariable', '')
 

index 7ebae28..0e5b602 100644
@@ -80,110 +80,13 @@ python patch_task_postfunc() {
 }
 
 def src_patches(d, all=False, expand=True):
-    workdir = d.getVar('WORKDIR', True)
-    fetch = bb.fetch2.Fetch([], d)
-    patches = []
-    sources = []
-    for url in fetch.urls:
-        local = patch_path(url, fetch, workdir, expand)
-        if not local:
-            if all:
-                local = fetch.localpath(url)
-                sources.append(local)
-            continue
-
-        urldata = fetch.ud[url]
-        parm = urldata.parm
-        patchname = parm.get('pname') or os.path.basename(local)
-
-        apply, reason = should_apply(parm, d)
-        if not apply:
-            if reason:
-                bb.note("Patch %s %s" % (patchname, reason))
-            continue
-
-        patchparm = {'patchname': patchname}
-        if "striplevel" in parm:
-            striplevel = parm["striplevel"]
-        elif "pnum" in parm:
-            #bb.msg.warn(None, "Deprecated usage of 'pnum' url parameter in '%s', please use 'striplevel'" % url)
-            striplevel = parm["pnum"]
-        else:
-            striplevel = '1'
-        patchparm['striplevel'] = striplevel
-
-        patchdir = parm.get('patchdir')
-        if patchdir:
-            patchparm['patchdir'] = patchdir
-
-        localurl = bb.fetch.encodeurl(('file', '', local, '', '', patchparm))
-        patches.append(localurl)
-
-    if all:
-        return sources
-
-    return patches
-
-def patch_path(url, fetch, workdir, expand=True):
-    """Return the local path of a patch, or None if this isn't a patch"""
-
-    local = fetch.localpath(url)
-    base, ext = os.path.splitext(os.path.basename(local))
-    if ext in ('.gz', '.bz2', '.Z'):
-        if expand:
-            local = os.path.join(workdir, base)
-        ext = os.path.splitext(base)[1]
-
-    urldata = fetch.ud[url]
-    if "apply" in urldata.parm:
-        apply = oe.types.boolean(urldata.parm["apply"])
-        if not apply:
-            return
-    elif ext not in (".diff", ".patch"):
-        return
-
-    return local
+    import oe.patch
+    return oe.patch.src_patches(d, all, expand)
 
 def should_apply(parm, d):
     """Determine if we should apply the given patch"""
-
-    if "mindate" in parm or "maxdate" in parm:
-        pn = d.getVar('PN', True)
-        srcdate = d.getVar('SRCDATE_%s' % pn, True)
-        if not srcdate:
-            srcdate = d.getVar('SRCDATE', True)
-
-        if srcdate == "now":
-            srcdate = d.getVar('DATE', True)
-
-        if "maxdate" in parm and parm["maxdate"] < srcdate:
-            return False, 'is outdated'
-
-        if "mindate" in parm and parm["mindate"] > srcdate:
-            return False, 'is predated'
-
-
-    if "minrev" in parm:
-        srcrev = d.getVar('SRCREV', True)
-        if srcrev and srcrev < parm["minrev"]:
-            return False, 'applies to later revisions'
-
-    if "maxrev" in parm:
-        srcrev = d.getVar('SRCREV', True)
-        if srcrev and srcrev > parm["maxrev"]:
-            return False, 'applies to earlier revisions'
-
-    if "rev" in parm:
-        srcrev = d.getVar('SRCREV', True)
-        if srcrev and parm["rev"] not in srcrev:
-            return False, "doesn't apply to revision"
-
-    if "notrev" in parm:
-        srcrev = d.getVar('SRCREV', True)
-        if srcrev and parm["notrev"] in srcrev:
-            return False, "doesn't apply to revision"
-
-    return True, None
+    import oe.patch
+    return oe.patch.should_apply(parm, d)
 
 should_apply[vardepsexclude] = "DATE SRCDATE"
 

110 lines not on this page · raw message

index ae83aab..92fa431 100644
@@ -382,6 +382,7 @@ def copy_recipe_files(d, tgt_dir, whole_dir=False, download=True):
 
 def get_recipe_local_files(d, patches=False, archives=False):
     """Get a list of local files in SRC_URI within a recipe."""
+    import oe.patch
     uris = (d.getVar('SRC_URI', True) or "").split()
     fetch = bb.fetch2.Fetch(uris, d)
     # FIXME this list should be factored out somewhere else (such as the

@@ -393,7 +394,7 @@ def get_recipe_local_files(d, patches=False, archives=False):
     for uri in uris:
         if fetch.ud[uri].type == 'file':
             if (not patches and
-                    bb.utils.exec_flat_python_func('patch_path', uri, fetch, '', expand=False)):
+                    oe.patch.patch_path(uri, fetch, '', expand=False)):
                 continue
             # Skip files that are referenced by absolute path
             fname = fetch.ud[uri].basepath

@@ -418,10 +419,9 @@ def get_recipe_local_files(d, patches=False, archives=False):
 
 def get_recipe_patches(d):
     """Get a list of the patches included in SRC_URI within a recipe."""
+    import oe.patch
+    patches = oe.patch.src_patches(d, expand=False)
     patchfiles = []
-    # Execute src_patches() defined in patch.bbclass - this works since that class
-    # is inherited globally
-    patches = bb.utils.exec_flat_python_func('src_patches', d, expand=False)
     for patch in patches:
         _, _, local, _, _, parm = bb.fetch.decodeurl(patch)
         patchfiles.append(local)

@@ -438,9 +438,7 @@ def get_recipe_patched_files(d):
         change mode ('A' for add, 'D' for delete or 'M' for modify)
     """
     import oe.patch
-    # Execute src_patches() defined in patch.bbclass - this works since that class
-    # is inherited globally
-    patches = bb.utils.exec_flat_python_func('src_patches', d, expand=False)
+    patches = oe.patch.src_patches(d, expand=False)
     patchedfiles = {}
     for patch in patches:
         _, _, patchfile, _, _, parm = bb.fetch.decodeurl(patch)

index 1052cd2..882b702 100755
@@ -73,6 +73,7 @@ def main():
         logger.error("Unable to find bitbake by searching parent directory of this script or PATH")
         sys.exit(1)
     logger.debug('Found bitbake path: %s' % bitbakepath)
+    scriptpath.add_oe_lib_path()
 
     scriptutils.logger_setup_color(logger, global_args.color)
 

index 06c508c..fbd8a71 100644
@@ -378,7 +378,7 @@ def extract(args, config, basepath, workspace):
             return 1
 
         srctree = os.path.abspath(args.srctree)
-        initial_rev = _extract_source(srctree, args.keep_temp, args.branch, False, rd)
+        initial_rev = _extract_source(srctree, args.keep_temp, args.branch, False, rd, tinfoil)
         logger.info('Source tree extracted to %s' % srctree)
 
         if initial_rev:

@@ -402,7 +402,7 @@ def sync(args, config, basepath, workspace):
             return 1
 
         srctree = os.path.abspath(args.srctree)
-        initial_rev = _extract_source(srctree, args.keep_temp, args.branch, True, rd)
+        initial_rev = _extract_source(srctree, args.keep_temp, args.branch, True, rd, tinfoil)
         logger.info('Source tree %s synchronized' % srctree)
 
         if initial_rev:

@@ -412,35 +412,6 @@ def sync(args, config, basepath, workspace):
     finally:
         tinfoil.shutdown()
 
-class BbTaskExecutor(object):
-    """Class for executing bitbake tasks for a recipe
-
-    FIXME: This is very awkward. Unfortunately it's not currently easy to
-    properly execute tasks outside of bitbake itself, until then this has to
-    suffice if we are to handle e.g. linux-yocto's extra tasks
-    """
-
-    def __init__(self, rdata):
-        self.rdata = rdata
-        self.executed = []
-
-    def exec_func(self, func, report):
-        """Run bitbake task function"""
-        if not func in self.executed:
-            deps = self.rdata.getVarFlag(func, 'deps', False)
-            if deps:
-                for taskdepfunc in deps:
-                    self.exec_func(taskdepfunc, True)
-            if report:
-                logger.info('Executing %s...' % func)
-            fn = self.rdata.getVar('FILE', True)
-            localdata = bb.build._task_data(fn, func, self.rdata)
-            try:
-                bb.build.exec_func(func, localdata)
-            except bb.build.FuncFailed as e:
-                raise DevtoolError(str(e))
-            self.executed.append(func)
-
 
 def _prep_extract_operation(config, basepath, recipename, tinfoil=None):
     """HACK: Ugly workaround for making sure that requirements are met when

@@ -464,21 +435,10 @@ def _prep_extract_operation(config, basepath, recipename, tinfoil=None):
     return tinfoil
 
 
-def _extract_source(srctree, keep_temp, devbranch, sync, d):
+def _extract_source(srctree, keep_temp, devbranch, sync, d, tinfoil):
     """Extract sources of a recipe"""
-    import bb.event
     import oe.recipeutils
 
-    def eventfilter(name, handler, event, d):
-        """Bitbake event filter for devtool extract operation"""
-        if name == 'base_eventhandler':
-            return True
-        else:
-            return False
-
-    if hasattr(bb.event, 'set_eventfilter'):
-        bb.event.set_eventfilter(eventfilter)
-
     pn = d.getVar('PN', True)
 
     _check_compatible_recipe(pn, d)

@@ -504,19 +464,15 @@ def _extract_source(srctree, keep_temp, devbranch, sync, d):
         bb.utils.mkdirhier(srctree)
         os.rmdir(srctree)
 
-    # We don't want notes to be printed, they are too verbose
-    origlevel = bb.logger.getEffectiveLevel()
-    if logger.getEffectiveLevel() > logging.DEBUG:
-        bb.logger.setLevel(logging.WARNING)
-
     initial_rev = None
     tempdir = tempfile.mkdtemp(prefix='devtool')
     try:
+        tinfoil.logger.setLevel(logging.WARNING)
+
         crd = d.createCopy()
         # Make a subdir so we guard against WORKDIR==S
         workdir = os.path.join(tempdir, 'workdir')
         crd.setVar('WORKDIR', workdir)
-        crd.setVar('T', os.path.join(tempdir, 'temp'))
         if not crd.getVar('S', True).startswith(workdir):
             # Usually a shared workdir recipe (kernel, gcc)
             # Try to set a reasonable default

@@ -528,21 +484,55 @@ def _extract_source(srctree, keep_temp, devbranch, sync, d):
             # We don't want to move the source to STAGING_KERNEL_DIR here
             crd.setVar('STAGING_KERNEL_DIR', '${S}')
 
-        task_executor = BbTaskExecutor(crd)
+        is_kernel_yocto = bb.data.inherits_class('kernel-yocto', d)
+        if not is_kernel_yocto:
+            crd.setVar('PATCHTOOL', 'git')
+            crd.setVar('PATCH_COMMIT_FUNCTIONS', '1')
+
+        # Apply our changes to the datastore to the server's datastore
+        for key in crd.localkeys():
+            tinfoil.config_data.setVar('%s_pn-%s' % (key, pn), crd.getVar(key, False))
+
+        tinfoil.config_data.setVar('STAMPS_DIR', os.path.join(tempdir, 'stamps'))
+        tinfoil.config_data.setVar('T', os.path.join(tempdir, 'temp'))
+        tinfoil.config_data.setVar('BUILDCFG_FUNCS', '')
+        tinfoil.config_data.setVar('BUILDCFG_HEADER', '')
+
+        tinfoil.set_event_mask(['bb.event.BuildStarted',
+                                'bb.event.BuildCompleted',
+                                'bb.event.TaskStarted',
+                                'logging.LogRecord',
+                                'bb.command.CommandCompleted',
+                                'bb.command.CommandFailed',
+                                'bb.build.TaskStarted',
+                                'bb.build.TaskSucceeded',
+                                'bb.build.TaskFailedSilent'])
+
+        def runtask(target, task):
+            if tinfoil.build_file(target, task):
+                while True:
+                    event = tinfoil.wait_event(0.25)
+                    if event:
+                        if isinstance(event, bb.command.CommandCompleted):
+                            break
+                        elif isinstance(event, bb.command.CommandFailed):
+                            raise DevtoolError('Task do_%s failed: %s' % (task, event.error))
+                        elif isinstance(event, bb.build.TaskStarted):
+                            logger.info('Executing %s...' % event._task)
+                        elif isinstance(event, logging.LogRecord):
+                            if event.levelno <= logging.INFO:
+                                continue
+                            logger.handle(event)
+
+        # we need virtual:native:/path/to/recipe if it's a BBCLASSEXTEND
+        fn = tinfoil.get_recipe_file(pn)
+        runtask(fn, 'unpack')
 
-        crd.setVar('EXTERNALSRC_forcevariable', '')
-
-        logger.info('Fetching %s...' % pn)
-        task_executor.exec_func('do_fetch', False)
-        logger.info('Unpacking...')
-        task_executor.exec_func('do_unpack', False)
         if bb.data.inherits_class('kernel-yocto', d):
             # Extra step for kernel to populate the source directory
-            logger.info('Doing kernel checkout...')
-            task_executor.exec_func('do_kernel_checkout', False)
-        srcsubdir = crd.getVar('S', True)
+            runtask(fn, 'kernel_checkout')
 
-        task_executor.check_git = True
+        srcsubdir = crd.getVar('S', True)
 
         # Move local source files into separate subdir
         recipe_patches = [os.path.basename(patch) for patch in

@@ -572,7 +562,7 @@ def _extract_source(srctree, keep_temp, devbranch, sync, d):
                          os.path.basename(fname) not in recipe_patches]
             # Force separate S so that patch files can be left out from srctree
             srcsubdir = tempfile.mkdtemp(dir=workdir)
-            crd.setVar('S', srcsubdir)
+            tinfoil.config_data.setVar('S_task-patch', srcsubdir)
             # Move source files to S
             for path in src_files:
                 _move_file(os.path.join(workdir, path),

@@ -595,10 +585,8 @@ def _extract_source(srctree, keep_temp, devbranch, sync, d):
         (stdout, _) = bb.process.run('git rev-parse HEAD', cwd=srcsubdir)
         initial_rev = stdout.rstrip()
 
-        crd.setVar('PATCHTOOL', 'git')
-
         logger.info('Patching...')
-        task_executor.exec_func('do_patch', False)
+        runtask(fn, 'patch')
 
         bb.process.run('git tag -f devtool-patched', cwd=srcsubdir)
 

@@ -606,7 +594,7 @@ def _extract_source(srctree, keep_temp, devbranch, sync, d):
         if bb.data.inherits_class('kernel-yocto', d):
             # Store generate and store kernel config
             logger.info('Generating kernel config')
-            task_executor.exec_func('do_configure', False)
+            runtask(fn, 'configure')
             kconfig = os.path.join(crd.getVar('B', True), '.config')
 
 

@@ -667,8 +655,6 @@ def _extract_source(srctree, keep_temp, devbranch, sync, d):
             shutil.copy2(kconfig, srctree)
 
     finally:
-        bb.logger.setLevel(origlevel)
-
         if keep_temp:
             logger.info('Preserving temporary directory %s' % tempdir)
         else:

@@ -773,7 +759,7 @@ def modify(args, config, basepath, workspace):
         initial_rev = None
         commits = []
         if not args.no_extract:
-            initial_rev = _extract_source(srctree, False, args.branch, False, rd)
+            initial_rev = _extract_source(srctree, False, args.branch, False, rd, tinfoil)
             if not initial_rev:
                 return 1
             logger.info('Source tree extracted to %s' % srctree)

index 52f9ab1..d89e9a2 100644
@@ -363,7 +363,7 @@ def upgrade(args, config, basepath, workspace):
 
         rf = None
         try:
-            rev1 = standard._extract_source(srctree, False, 'devtool-orig', False, rd)
+            rev1 = standard._extract_source(srctree, False, 'devtool-orig', False, rd, tinfoil)
             rev2, md5, sha256 = _extract_new_source(args.version, srctree, args.no_patch,
                                                     args.srcrev, args.branch, args.keep_temp,
                                                     tinfoil, rd)

index 92fa431..26c926f 100644
@@ -52,28 +52,6 @@ def parse_recipe(cooker, fn, appendfiles):
     return envdata
 
 
-def parse_recipe_simple(cooker, pn, d, appends=True):
-    """
-    Parse a recipe and optionally all bbappends that apply to it
-    in the current configuration.
-    """
-    import bb.providers
-
-    recipefile = pn_to_recipe(cooker, pn)
-    if not recipefile:
-        skipreasons = get_unavailable_reasons(cooker, pn)
-        # We may as well re-use bb.providers.NoProvider here
-        if skipreasons:
-            raise bb.providers.NoProvider(skipreasons)
-        else:
-            raise bb.providers.NoProvider('Unable to find any recipe file matching %s' % pn)
-    if appends:
-        appendfiles = cooker.collection.get_file_appends(recipefile)
-    else:
-        appendfiles = None
-    return parse_recipe(cooker, recipefile, appendfiles)
-
-
 def get_var_files(fn, varlist, d):
     """Find the file in which each of a list of variables is set.
     Note: requires variable history to be enabled when parsing.

index fbd8a71..c52b006 100644
@@ -497,6 +497,7 @@ def _extract_source(srctree, keep_temp, devbranch, sync, d, tinfoil):
         tinfoil.config_data.setVar('T', os.path.join(tempdir, 'temp'))
         tinfoil.config_data.setVar('BUILDCFG_FUNCS', '')
         tinfoil.config_data.setVar('BUILDCFG_HEADER', '')
+        tinfoil.config_data.setVar('BB_HASH_IGNORE_MISMATCH', '1')
 
         tinfoil.set_event_mask(['bb.event.BuildStarted',
                                 'bb.event.BuildCompleted',

index 0c32c50..656898a 100755
@@ -288,11 +288,14 @@ def main():
     scriptutils.logger_setup_color(logger, global_args.color)
 
     if global_args.bbpath is None:
-        tinfoil = setup_tinfoil(config_only=True, basepath=basepath)
         try:
-            global_args.bbpath = tinfoil.config_data.getVar('BBPATH', True)
-        finally:
-            tinfoil.shutdown()
+            tinfoil = setup_tinfoil(config_only=True, basepath=basepath)
+            try:
+                global_args.bbpath = tinfoil.config_data.getVar('BBPATH', True)
+            finally:
+                tinfoil.shutdown()
+        except bb.BBHandledException:
+            return 2
 
     for path in [scripts_path] + global_args.bbpath.split(':'):
         pluginpath = os.path.join(path, 'lib', 'devtool')

index 7db286f..5f4e828 100644
@@ -439,7 +439,15 @@ class DevtoolTests(DevtoolBase):
         # Try building
         bitbake('mdadm')
         # Try making (minor) modifications to the source
-        result = runCmd("sed -i 's!^\.TH.*!.TH MDADM 8 \"\" v9.999-custom!' %s" % os.path.join(tempdir, 'mdadm.8.in'))
+        modfile = os.path.join(tempdir, 'mdadm.8.in')
+        result = runCmd("sed -i 's!^\.TH.*!.TH MDADM 8 \"\" v9.999-custom!' %s" % modfile)
+        sedline = ''
+        with open(modfile, 'r') as f:
+            for line in f:
+                if line.startswith('.TH'):
+                    sedline = line.rstrip()
+                    break
+        self.assertEqual(sedline, '.TH MDADM 8 "" v9.999-custom', 'man .in file not modified (sed failed)')
         bitbake('mdadm -c package')
         pkgd = get_bb_var('PKGD', 'mdadm')
         self.assertTrue(pkgd, 'Could not query PKGD variable')

@@ -447,10 +455,11 @@ class DevtoolTests(DevtoolBase):
         self.assertTrue(mandir, 'Could not query mandir variable')
         if mandir[0] == '/':
             mandir = mandir[1:]
-        with open(os.path.join(pkgd, mandir, 'man8', 'mdadm.8'), 'r') as f:
+        manfile = os.path.join(pkgd, mandir, 'man8', 'mdadm.8')
+        with open(manfile, 'r') as f:
             for line in f:
                 if line.startswith('.TH'):
-                    self.assertEqual(line.rstrip(), '.TH MDADM 8 "" v9.999-custom', 'man file not modified. man searched file path: %s' % os.path.join(pkgd, mandir, 'man8', 'mdadm.8'))
+                    self.assertEqual(line.rstrip(), '.TH MDADM 8 "" v9.999-custom', 'man file not modified. man searched file path: %s' % manfile)
         # Test devtool reset
         stampprefix = get_bb_var('STAMP', 'mdadm')
         result = runCmd('devtool reset mdadm')

new file mode 100644
index 0000000..4f70e0d
@@ -0,0 +1,146 @@
+import unittest
+import os
+import re
+import bb.tinfoil
+
+from oeqa.selftest.base import oeSelfTest
+from oeqa.utils.commands import runCmd, get_bb_var
+from oeqa.utils.decorators import testcase
+
+class TinfoilTests(oeSelfTest):
+    """ Basic tests for the tinfoil API """
+
+    def test_getvar(self):
+        with bb.tinfoil.Tinfoil() as tinfoil:
+            tinfoil.prepare(True)
+            machine = tinfoil.config_data.getVar('MACHINE', True)
+            if not machine:
+                self.fail('Unable to get MACHINE value - returned %s' % machine)
+
+    def test_expand(self):
+        with bb.tinfoil.Tinfoil() as tinfoil:
+            tinfoil.prepare(True)
+            expr = '${@os.getpid()}'
+            pid = tinfoil.config_data.expand(expr)
+            if not pid:
+                self.fail('Unable to expand "%s" - returned %s' % (expr, pid))
+
+    def test_getvar_bb_origenv(self):
+        with bb.tinfoil.Tinfoil() as tinfoil:
+            tinfoil.prepare(True)
+            origenv = tinfoil.config_data.getVar('BB_ORIGENV', False)
+            if not origenv:
+                self.fail('Unable to get BB_ORIGENV value - returned %s' % origenv)
+            self.assertEqual(origenv.getVar('HOME', False), os.environ['HOME'])
+
+    def test_parse_recipe(self):
+        with bb.tinfoil.Tinfoil() as tinfoil:
+            tinfoil.prepare(config_only=False, quiet=2)
+            testrecipe = 'mdadm'
+            best = tinfoil.find_best_provider(testrecipe)
+            if not best:
+                self.fail('Unable to find recipe providing %s' % testrecipe)
+            rd = tinfoil.parse_recipe_file(best[3])
+            self.assertEqual(testrecipe, rd.getVar('PN', True))
+
+    def test_parse_recipe_copy_expand(self):
+        with bb.tinfoil.Tinfoil() as tinfoil:
+            tinfoil.prepare(config_only=False, quiet=2)
+            testrecipe = 'mdadm'
+            best = tinfoil.find_best_provider(testrecipe)
+            if not best:
+                self.fail('Unable to find recipe providing %s' % testrecipe)
+            rd = tinfoil.parse_recipe_file(best[3])
+            # Check we can get variable values
+            self.assertEqual(testrecipe, rd.getVar('PN', True))
+            # Check that expanding a value that includes a variable reference works
+            self.assertEqual(testrecipe, rd.getVar('BPN', True))
+            # Now check that changing the referenced variable's value in a copy gives that
+            # value when expanding
+            localdata = bb.data.createCopy(rd)
+            localdata.setVar('PN', 'hello')
+            self.assertEqual('hello', localdata.getVar('BPN', True))
+
+    def test_parse_recipe_initial_datastore(self):
+        with bb.tinfoil.Tinfoil() as tinfoil:
+            tinfoil.prepare(config_only=False, quiet=2)
+            testrecipe = 'mdadm'
+            best = tinfoil.find_best_provider(testrecipe)
+            if not best:
+                self.fail('Unable to find recipe providing %s' % testrecipe)
+            dcopy = bb.data.createCopy(tinfoil.config_data)
+            dcopy.setVar('MYVARIABLE', 'somevalue')
+            rd = tinfoil.parse_recipe_file(best[3], config_data=dcopy)
+            # Check we can get variable values
+            self.assertEqual('somevalue', rd.getVar('MYVARIABLE', True))
+
+    def test_list_recipes(self):
+        with bb.tinfoil.Tinfoil() as tinfoil:
+            tinfoil.prepare(config_only=False, quiet=2)
+            # Check pkg_pn
+            checkpns = ['tar', 'automake', 'coreutils', 'm4-native', 'nativesdk-gcc']
+            pkg_pn = tinfoil.cooker.recipecaches[''].pkg_pn
+            for pn in checkpns:
+                self.assertIn(pn, pkg_pn)
+            # Check pkg_fn
+            checkfns = {'nativesdk-gcc': '^virtual:nativesdk:.*', 'coreutils': '.*/coreutils_.*.bb'}
+            for fn, pn in tinfoil.cooker.recipecaches[''].pkg_fn.items():
+                if pn in checkpns:
+                    if pn in checkfns:
+                        self.assertTrue(re.match(checkfns[pn], fn), 'Entry for %s: %s did not match %s' % (pn, fn, checkfns[pn]))
+                    checkpns.remove(pn)
+            if checkpns:
+                self.fail('Unable to find pkg_fn entries for: %s' % ', '.join(checkpns))
+
+    def test_wait_event(self):
+        with bb.tinfoil.Tinfoil() as tinfoil:
+            tinfoil.prepare(config_only=True)
+            # Need to drain events otherwise events that will be masked will still be in the queue
+            while tinfoil.wait_event(0.25):
+                pass
+            tinfoil.set_event_mask(['bb.event.FilesMatchingFound', 'bb.command.CommandCompleted'])
+            pattern = 'conf'
+            res = tinfoil.run_command('findFilesMatchingInDir', pattern, 'conf/machine')
+            self.assertTrue(res)
+
+            eventreceived = False
+            waitcount = 5
+            while waitcount > 0:
+                event = tinfoil.wait_event(1)
+                if event:
+                    if isinstance(event, bb.command.CommandCompleted):
+                        break
+                    elif isinstance(event, bb.event.FilesMatchingFound):
+                        self.assertEqual(pattern, event._pattern)
+                        self.assertIn('qemuarm.conf', event._matches)
+                        eventreceived = True
+                    else:
+                        self.fail('Unexpected event: %s' % event)
+
+                waitcount = waitcount - 1
+
+            self.assertNotEqual(waitcount, 0, 'Timed out waiting for CommandCompleted event from bitbake server')
+            self.assertTrue(eventreceived, 'Did not receive FilesMatchingFound event from bitbake server')
+
+    def test_setvariable_clean(self):
+        # First check that setVariable affects the datastore
+        with bb.tinfoil.Tinfoil() as tinfoil:
+            tinfoil.prepare(config_only=True)
+            tinfoil.run_command('setVariable', 'TESTVAR', 'specialvalue')
+            self.assertEqual(tinfoil.config_data.getVar('TESTVAR', True), 'specialvalue', 'Value set using setVariable is not reflected in client-side getVar()')
+
+        # Now check that the setVariable's effects are no longer present
+        # (this may legitimately break in future if we stop reinitialising
+        # the datastore, in which case we'll have to reconsider use of
+        # setVariable entirely)
+        with bb.tinfoil.Tinfoil() as tinfoil:
+            tinfoil.prepare(config_only=True)
+            self.assertNotEqual(tinfoil.config_data.getVar('TESTVAR', True), 'specialvalue', 'Value set using setVariable is still present!')
+
+        # Now check that setVar on the main datastore works (uses setVariable internally)
+        with bb.tinfoil.Tinfoil() as tinfoil:
+            tinfoil.prepare(config_only=True)
+            tinfoil.config_data.setVar('TESTVAR', 'specialvalue')
+            value = tinfoil.run_command('getVariable', 'TESTVAR')
+            self.assertEqual(value, 'specialvalue', 'Value set using config_data.setVar() is not reflected in config_data.getVar()')
+