devtool fixes

index 81a44d4..3bc84c7 100644
@@ -177,6 +177,16 @@ def _parse_recipe(config, tinfoil, pn, appends):
     return oe.recipeutils.parse_recipe(recipefile, append_files,
                                        tinfoil.config_data)
 
+
+def _ls_tree(directory):
+    """Recursive listing of files in a directory"""
+    ret = []
+    for root, dirs, files in os.walk(directory):
+        ret.extend([os.path.relpath(os.path.join(root, fname), directory) for
+                    fname in files])
+    return ret
+
+
 def extract(args, config, basepath, workspace):
     import bb
 

@@ -196,6 +206,7 @@ def extract(args, config, basepath, workspace):
 
 def _extract_source(srctree, keep_temp, devbranch, d):
     import bb.event
+    import oe.recipeutils
 
     def eventfilter(name, handler, event, d):
         if name == 'base_eventhandler':

@@ -264,7 +275,21 @@ def _extract_source(srctree, keep_temp, devbranch, d):
         logger.info('Unpacking...')
         exec_task_func('do_unpack', False)
         srcsubdir = crd.getVar('S', True)
-        if srcsubdir != workdir and os.path.dirname(srcsubdir) != workdir:
+        if srcsubdir == workdir:
+            # Find non-patch sources that were "unpacked" to srctree directory
+            recipe_patches = [os.path.basename(patch) for patch in
+                              oe.recipeutils.get_recipe_patches(crd)]
+            src_files = [fname for fname in _ls_tree(workdir) if
+                         os.path.basename(fname) not in recipe_patches]
+            # Force separate S so that patch files can be left out from srctree
+            srcsubdir = tempfile.mkdtemp(dir=workdir)
+            crd.setVar('S', srcsubdir)
+            # Move source files to S
+            for path in src_files:
+                tgt_dir = os.path.join(srcsubdir, os.path.dirname(path))
+                bb.utils.mkdirhier(tgt_dir)
+                shutil.move(os.path.join(workdir, path), tgt_dir)
+        elif os.path.dirname(srcsubdir) != workdir:
             # Handle if S is set to a subdirectory of the source
             srcsubdir = os.path.join(workdir, os.path.relpath(srcsubdir, workdir).split(os.sep)[0])
 

index 78ae0aa..f19d032 100644
@@ -75,7 +75,6 @@ def setup_tinfoil():
         sys.exit(1)
 
     import bb.tinfoil
-    import logging
     tinfoil = bb.tinfoil.Tinfoil()
     tinfoil.prepare(False)
     tinfoil.logger.setLevel(logging.WARNING)

index f016b23..e46ee05 100644
@@ -28,7 +28,6 @@ def plugin_init(pluginlist):
 
 def deploy(args, config, basepath, workspace):
     import re
-    from devtool import exec_build_env_command
 
     if not args.recipename in workspace:
         logger.error("no recipe named %s in your workspace" % args.recipename)

index 3bc84c7..3e7b54e 100644
@@ -19,7 +19,6 @@ import os
 import sys
 import re
 import shutil
-import glob
 import tempfile
 import logging
 import argparse

index e46ee05..4f968c6 100644
@@ -42,7 +42,7 @@ def deploy(args, config, basepath, workspace):
     deploy_dir = os.path.join(basepath, 'target_deploy', args.target)
     deploy_file = os.path.join(deploy_dir, args.recipename + '.list')
 
-    stdout, stderr = exec_build_env_command(config.init_path, basepath, 'bitbake -e %s' % args.recipename, shell=True)
+    stdout, _ = exec_build_env_command(config.init_path, basepath, 'bitbake -e %s' % args.recipename, shell=True)
     recipe_outdir = re.search(r'^D="(.*)"', stdout, re.MULTILINE).group(1)
     if not os.path.exists(recipe_outdir) or not os.listdir(recipe_outdir):
         logger.error('No files to deploy - have you built the %s recipe? If so, the install step has not installed any files.' % args.recipename)

@@ -50,7 +50,7 @@ def deploy(args, config, basepath, workspace):
 
     if args.dry_run:
         print('Files to be deployed for %s on target %s:' % (args.recipename, args.target))
-        for root, dirs, files in os.walk(recipe_outdir):
+        for root, _, files in os.walk(recipe_outdir):
             for fn in files:
                 print('  %s' % os.path.join(destdir, os.path.relpath(root, recipe_outdir), fn))
         return 0

index 3e7b54e..a9dd3b2 100644
@@ -94,7 +94,7 @@ def add(args, config, basepath, workspace):
         source = srctree
     if args.version:
         extracmdopts += ' -V %s' % args.version
-    stdout, stderr = exec_build_env_command(config.init_path, basepath, 'recipetool --color=%s create -o %s "%s" %s' % (color, recipefile, source, extracmdopts))
+    stdout, _ = exec_build_env_command(config.init_path, basepath, 'recipetool --color=%s create -o %s "%s" %s' % (color, recipefile, source, extracmdopts))
     logger.info('Recipe %s has been automatically created; further editing may be required to make it fully functional' % recipefile)
 
     _add_md5(config, args.recipename, recipefile)

index f19d032..8866512 100644
@@ -16,7 +16,7 @@
 # You should have received a copy of the GNU General Public License along
 # with this program; if not, write to the Free Software Foundation, Inc.,
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
+"""Devtool plugins module"""
 
 import os
 import sys

@@ -26,6 +26,7 @@ import logging
 logger = logging.getLogger('devtool')
 
 def exec_build_env_command(init_path, builddir, cmd, watch=False, **options):
+    """Run a program in bitbake build context"""
     import bb
     if not 'cwd' in options:
         options["cwd"] = builddir

@@ -49,6 +50,7 @@ def exec_build_env_command(init_path, builddir, cmd, watch=False, **options):
         return bb.process.run('%s%s' % (init_prefix, cmd), **options)
 
 def exec_watch(cmd, **options):
+    """Run program with stdout shown on sys.stdout"""
     if isinstance(cmd, basestring) and not "shell" in options:
         options["shell"] = True
 

@@ -68,6 +70,7 @@ def exec_watch(cmd, **options):
     return buf
 
 def setup_tinfoil():
+    """Initialize tinfoil api from bitbake"""
     import scriptpath
     bitbakepath = scriptpath.add_bitbake_lib_path()
     if not bitbakepath:

index 4f968c6..3c7abfa 100644
@@ -14,6 +14,7 @@
 # You should have received a copy of the GNU General Public License along
 # with this program; if not, write to the Free Software Foundation, Inc.,
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+"""Devtool plugin containing the deploy subcommands"""
 
 import os
 import subprocess

@@ -23,10 +24,12 @@ from devtool import exec_build_env_command
 logger = logging.getLogger('devtool')
 
 def plugin_init(pluginlist):
+    """Plugin initialization"""
     pass
 
 
 def deploy(args, config, basepath, workspace):
+    """Entry point for the devtool 'deploy' subcommand"""
     import re
 
     if not args.recipename in workspace:

@@ -87,7 +90,7 @@ def deploy(args, config, basepath, workspace):
     return 0
 
 def undeploy(args, config, basepath, workspace):
-
+    """Entry point for the devtool 'undeploy' subcommand"""
     deploy_file = os.path.join(basepath, 'target_deploy', args.target, args.recipename + '.list')
     if not os.path.exists(deploy_file):
          logger.error('%s has not been deployed' % args.recipename)

@@ -122,6 +125,7 @@ def undeploy(args, config, basepath, workspace):
 
 
 def register_commands(subparsers, context):
+    """Register devtool subcommands from the deploy plugin"""
     parser_deploy = subparsers.add_parser('deploy-target', help='Deploy recipe output files to live target machine')
     parser_deploy.add_argument('recipename', help='Recipe to deploy')
     parser_deploy.add_argument('target', help='Live target machine running an ssh server: user@hostname[:destdir]')

index a9dd3b2..2f8b194 100644
@@ -14,6 +14,7 @@
 # You should have received a copy of the GNU General Public License along
 # with this program; if not, write to the Free Software Foundation, Inc.,
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+"""Devtool standard plugins"""
 
 import os
 import sys

@@ -28,10 +29,12 @@ from devtool import exec_build_env_command, setup_tinfoil
 logger = logging.getLogger('devtool')
 
 def plugin_init(pluginlist):
+    """Plugin initialization"""
     pass
 
 
 def add(args, config, basepath, workspace):
+    """Entry point for the devtool 'add' subcommand"""
     import bb
     import oe.recipeutils
 

@@ -119,6 +122,7 @@ def add(args, config, basepath, workspace):
 
 
 def _check_compatible_recipe(pn, d):
+    """Check if the recipe is supported by devtool"""
     if pn == 'perf':
         logger.error("The perf recipe does not actually check out source and thus cannot be supported by this tool")
         return False

@@ -151,6 +155,7 @@ def _check_compatible_recipe(pn, d):
 
 
 def _get_recipe_file(cooker, pn):
+    """Find recipe file corresponding a package name"""
     import oe.recipeutils
     recipefile = oe.recipeutils.pn_to_recipe(cooker, pn)
     if not recipefile:

@@ -187,6 +192,7 @@ def _ls_tree(directory):
 
 
 def extract(args, config, basepath, workspace):
+    """Entry point for the devtool 'extract' subcommand"""
     import bb
 
     tinfoil = setup_tinfoil()

@@ -204,10 +210,12 @@ def extract(args, config, basepath, workspace):
 
 
 def _extract_source(srctree, keep_temp, devbranch, d):
+    """Extract sources of a recipe"""
     import bb.event
     import oe.recipeutils
 
     def eventfilter(name, handler, event, d):
+        """Bitbake event filter for devtool extract operation"""
         if name == 'base_eventhandler':
             return True
         else:

@@ -257,6 +265,7 @@ def _extract_source(srctree, keep_temp, devbranch, d):
         # are to handle e.g. linux-yocto's extra tasks
         executed = []
         def exec_task_func(func, report):
+            """Run specific bitbake task for a recipe"""
             if not func in executed:
                 deps = crd.getVarFlag(func, 'deps')
                 if deps:

@@ -343,12 +352,15 @@ def _extract_source(srctree, keep_temp, devbranch, d):
     return initial_rev
 
 def _add_md5(config, recipename, filename):
+    """Record checksum of a recipe to the md5-file of the workspace"""
     import bb.utils
     md5 = bb.utils.md5_file(filename)
     with open(os.path.join(config.workspace_path, '.devtool_md5'), 'a') as f:
         f.write('%s|%s|%s\n' % (recipename, os.path.relpath(filename, config.workspace_path), md5))
 
 def _check_preserve(config, recipename):
+    """Check if a recipe was manually changed and needs to be saved in 'attic'
+       directory"""
     import bb.utils
     origfile = os.path.join(config.workspace_path, '.devtool_md5')
     newfile = os.path.join(config.workspace_path, '.devtool_md5_new')

@@ -382,6 +394,7 @@ def _check_preserve(config, recipename):
 
 
 def modify(args, config, basepath, workspace):
+    """Entry point for the devtool 'modify' subcommand"""
     import bb
     import oe.recipeutils
 

@@ -481,6 +494,7 @@ def modify(args, config, basepath, workspace):
 
 
 def update_recipe(args, config, basepath, workspace):
+    """Entry point for the devtool 'update-recipe' subcommand"""
     if not args.recipename in workspace:
         logger.error("no recipe named %s in your workspace" % args.recipename)
         return -1

@@ -511,7 +525,7 @@ def update_recipe(args, config, basepath, workspace):
         mode = args.mode
 
     def remove_patches(srcuri, patchlist):
-        # Remove any patches that we don't need
+        """Remove patches"""
         updated = False
         for patch in patchlist:
             patchfile = os.path.basename(patch)

@@ -663,6 +677,7 @@ def update_recipe(args, config, basepath, workspace):
 
 
 def status(args, config, basepath, workspace):
+    """Entry point for the devtool 'status' subcommand"""
     if workspace:
         for recipe, value in workspace.iteritems():
             print("%s: %s" % (recipe, value))

@@ -672,6 +687,7 @@ def status(args, config, basepath, workspace):
 
 
 def reset(args, config, basepath, workspace):
+    """Entry point for the devtool 'reset' subcommand"""
     import bb.utils
     if args.recipename:
         if args.all:

@@ -713,6 +729,7 @@ def reset(args, config, basepath, workspace):
 
 
 def build(args, config, basepath, workspace):
+    """Entry point for the devtool 'build' subcommand"""
     import bb
     if not args.recipename in workspace:
         logger.error("no recipe named %s in your workspace" % args.recipename)

@@ -724,6 +741,7 @@ def build(args, config, basepath, workspace):
 
 
 def register_commands(subparsers, context):
+    """Register devtool subcommands from this plugin"""
     parser_add = subparsers.add_parser('add', help='Add a new recipe',
                                        description='Adds a new recipe')
     parser_add.add_argument('recipename', help='Name for new recipe to add')

index 3c7abfa..078c74b 100644
@@ -93,8 +93,8 @@ def undeploy(args, config, basepath, workspace):
     """Entry point for the devtool 'undeploy' subcommand"""
     deploy_file = os.path.join(basepath, 'target_deploy', args.target, args.recipename + '.list')
     if not os.path.exists(deploy_file):
-         logger.error('%s has not been deployed' % args.recipename)
-         return -1
+        logger.error('%s has not been deployed' % args.recipename)
+        return -1
 
     if args.dry_run:
         print('Previously deployed files to be un-deployed for %s on target %s:' % (args.recipename, args.target))

index 8866512..5a06c78 100644
@@ -44,13 +44,14 @@ def exec_build_env_command(init_path, builddir, cmd, watch=False, **options):
     if watch:
         if sys.stdout.isatty():
             # Fool bitbake into thinking it's outputting to a terminal (because it is, indirectly)
-            cmd = 'script -q -c "%s" /dev/null' % cmd
+            cmd = 'script -e -q -c "%s" /dev/null' % cmd
         return exec_watch('%s%s' % (init_prefix, cmd), **options)
     else:
         return bb.process.run('%s%s' % (init_prefix, cmd), **options)
 
 def exec_watch(cmd, **options):
     """Run program with stdout shown on sys.stdout"""
+    import bb
     if isinstance(cmd, basestring) and not "shell" in options:
         options["shell"] = True
 

@@ -67,7 +68,11 @@ def exec_watch(cmd, **options):
             buf += out
         elif out == '' and process.poll() != None:
             break
-    return buf
+
+    if process.returncode != 0:
+        raise bb.process.ExecutionError(cmd, process.returncode, buf, None)
+
+    return buf, None
 
 def setup_tinfoil():
     """Initialize tinfoil api from bitbake"""

index 2f8b194..61c0df9 100644
@@ -97,8 +97,12 @@ def add(args, config, basepath, workspace):
         source = srctree
     if args.version:
         extracmdopts += ' -V %s' % args.version
-    stdout, _ = exec_build_env_command(config.init_path, basepath, 'recipetool --color=%s create -o %s "%s" %s' % (color, recipefile, source, extracmdopts))
-    logger.info('Recipe %s has been automatically created; further editing may be required to make it fully functional' % recipefile)
+    try:
+        stdout, _ = exec_build_env_command(config.init_path, basepath, 'recipetool --color=%s create -o %s "%s" %s' % (color, recipefile, source, extracmdopts))
+        logger.info('Recipe %s has been automatically created; further editing may be required to make it fully functional' % recipefile)
+    except bb.process.ExecutionError as e:
+        logger.error('Command \'%s\' failed:\n%s' % (e.command, e.stdout))
+        return 1
 
     _add_md5(config, args.recipename, recipefile)
 

@@ -688,7 +692,7 @@ def status(args, config, basepath, workspace):
 
 def reset(args, config, basepath, workspace):
     """Entry point for the devtool 'reset' subcommand"""
-    import bb.utils
+    import bb
     if args.recipename:
         if args.all:
             logger.error("Recipe cannot be specified if -a/--all is used")

@@ -708,7 +712,11 @@ def reset(args, config, basepath, workspace):
     for pn in recipes:
         if not args.no_clean:
             logger.info('Cleaning sysroot for recipe %s...' % pn)
-            exec_build_env_command(config.init_path, basepath, 'bitbake -c clean %s' % pn)
+            try:
+                exec_build_env_command(config.init_path, basepath, 'bitbake -c clean %s' % pn)
+            except bb.process.ExecutionError as e:
+                logger.error('Command \'%s\' failed, output:\n%s\nIf you wish, you may specify -n/--no-clean to skip running this command when resetting' % (e.command, e.stdout))
+                return 1
 
         _check_preserve(config, pn)
 

@@ -735,7 +743,11 @@ def build(args, config, basepath, workspace):
         logger.error("no recipe named %s in your workspace" % args.recipename)
         return -1
     build_task = config.get('Build', 'build_task', 'populate_sysroot')
-    exec_build_env_command(config.init_path, basepath, 'bitbake -c %s %s' % (build_task, args.recipename), watch=True)
+    try:
+        exec_build_env_command(config.init_path, basepath, 'bitbake -c %s %s' % (build_task, args.recipename), watch=True)
+    except bb.process.ExecutionError as e:
+        # We've already seen the output since watch=True, so just ensure we return something to the user
+        return e.exitcode
 
     return 0
 

index 19d97b6..0689fb0 100644
@@ -44,13 +44,36 @@ def get_unavailable_reasons(cooker, pn):
     return taskdata.get_reasons(pn)
 
 
-def parse_recipe(fn, appends, d):
-    """Parse an individual recipe"""
+def parse_recipe(fn, appendfiles, d):
+    """
+    Parse an individual recipe file, optionally with a list of
+    bbappend files.
+    """
     import bb.cache
-    envdata = bb.cache.Cache.loadDataFull(fn, appends, d)
+    envdata = bb.cache.Cache.loadDataFull(fn, appendfiles, d)
     return envdata
 
 
+def parse_recipe_simple(cooker, pn, d, appends=True):
+    """
+    Parse a recipe and optionally all bbappends that apply to it
+    in the current configuration.
+    """
+    import bb.providers
+
+    recipefile = pn_to_recipe(cooker, pn)
+    if not recipefile:
+        skipreasons = get_unavailable_reasons(cooker, pn)
+        # We may as well re-use bb.providers.NoProvider here
+        if skipreasons:
+            raise bb.providers.NoProvider(skipreasons)
+        else:
+            raise bb.providers.NoProvider('Unable to find any recipe file matching %s' % pn)
+    if appends:
+        appendfiles = cooker.collection.get_file_appends(recipefile)
+    return parse_recipe(recipefile, appendfiles, d)
+
+
 def get_var_files(fn, varlist, d):
     """Find the file in which each of a list of variables is set.
     Note: requires variable history to be enabled when parsing.

index 078c74b..92a3cb4 100644
@@ -19,7 +19,7 @@
 import os
 import subprocess
 import logging
-from devtool import exec_build_env_command
+from devtool import exec_build_env_command, setup_tinfoil
 
 logger = logging.getLogger('devtool')
 

@@ -31,6 +31,7 @@ def plugin_init(pluginlist):
 def deploy(args, config, basepath, workspace):
     """Entry point for the devtool 'deploy' subcommand"""
     import re
+    import oe.recipeutils
 
     if not args.recipename in workspace:
         logger.error("no recipe named %s in your workspace" % args.recipename)

@@ -45,8 +46,13 @@ def deploy(args, config, basepath, workspace):
     deploy_dir = os.path.join(basepath, 'target_deploy', args.target)
     deploy_file = os.path.join(deploy_dir, args.recipename + '.list')
 
-    stdout, _ = exec_build_env_command(config.init_path, basepath, 'bitbake -e %s' % args.recipename, shell=True)
-    recipe_outdir = re.search(r'^D="(.*)"', stdout, re.MULTILINE).group(1)
+    tinfoil = setup_tinfoil()
+    try:
+        rd = oe.recipeutils.parse_recipe_simple(tinfoil.cooker, args.recipename, tinfoil.config_data)
+    except Exception as e:
+        logger.error('Exception parsing recipe %s: %s' % (args.recipename, e))
+        return 2
+    recipe_outdir = rd.getVar('D', True)
     if not os.path.exists(recipe_outdir) or not os.listdir(recipe_outdir):
         logger.error('No files to deploy - have you built the %s recipe? If so, the install step has not installed any files.' % args.recipename)
         return -1

index 841831c..0100eb8 100755
@@ -2,7 +2,7 @@
 
 # OpenEmbedded Development tool
 #
-# Copyright (C) 2014 Intel Corporation
+# Copyright (C) 2014-2015 Intel Corporation
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 2 as

@@ -99,6 +99,8 @@ def read_workspace():
         else:
             logger.info('Creating workspace layer in %s' % config.workspace_path)
             _create_workspace(config.workspace_path, config, basepath)
+    if not context.fixed_setup:
+        _enable_workspace_layer(config.workspace_path, config, basepath)
 
     logger.debug('Reading workspace in %s' % config.workspace_path)
     externalsrc_re = re.compile(r'^EXTERNALSRC(_pn-[^ =]+)? =.*$')

@@ -116,9 +118,11 @@ def create_workspace(args, config, basepath, workspace):
         workspacedir = os.path.abspath(args.layerpath)
     else:
         workspacedir = os.path.abspath(os.path.join(basepath, 'workspace'))
-    _create_workspace(workspacedir, config, basepath, args.create_only)
+    _create_workspace(workspacedir, config, basepath)
+    if not args.create_only:
+        _enable_workspace_layer(workspacedir, config, basepath)
 
-def _create_workspace(workspacedir, config, basepath, create_only=False):
+def _create_workspace(workspacedir, config, basepath):
     import bb
 
     confdir = os.path.join(workspacedir, 'conf')

@@ -146,17 +150,21 @@ def _create_workspace(workspacedir, config, basepath, create_only=False):
             f.write('\nIf you no longer need to use devtool you can remove the path to this\n')
             f.write('workspace layer from your conf/bblayers.conf file (and then delete the\n')
             f.write('layer, if you wish).\n')
-    if not create_only:
-        # Add the workspace layer to bblayers.conf
-        bblayers_conf = os.path.join(basepath, 'conf', 'bblayers.conf')
-        if not os.path.exists(bblayers_conf):
-            logger.error('Unable to find bblayers.conf')
-            return -1
-        bb.utils.edit_bblayers_conf(bblayers_conf, workspacedir, config.workspace_path)
-        if config.workspace_path != workspacedir:
-            # Update our config to point to the new location
-            config.workspace_path = workspacedir
-            config.write()
+
+def _enable_workspace_layer(workspacedir, config, basepath):
+    """Ensure the workspace layer is in bblayers.conf"""
+    import bb
+    bblayers_conf = os.path.join(basepath, 'conf', 'bblayers.conf')
+    if not os.path.exists(bblayers_conf):
+        logger.error('Unable to find bblayers.conf')
+        return -1
+    _, added = bb.utils.edit_bblayers_conf(bblayers_conf, workspacedir, config.workspace_path)
+    if added:
+        logger.info('Enabling workspace layer in bblayers.conf')
+    if config.workspace_path != workspacedir:
+        # Update our config to point to the new location
+        config.workspace_path = workspacedir
+        config.write()
 
 
 def main():