Extensible SDK fixesExtensible SDK fixes

index cf7fada..979578c 100644
@@ -28,11 +28,10 @@ class BuildSystem(object):
         layers.append(corebase)
 
         corebase_files = self.d.getVar('COREBASE_FILES', True).split()
-
-        # bitbake belongs in corebase so make sure it goes there
-        if "bitbake" not in corebase_files:
-            corebase_files.append("bitbake")
         corebase_files = [corebase + '/' +x for x in corebase_files]
+        # Make sure bitbake goes in
+        bitbake_dir = bb.__file__.rsplit('/', 3)[0]
+        corebase_files.append(bitbake_dir)
 
         for layer in layers:
             layerconf = os.path.join(layer, 'conf', 'layer.conf')

index a36bf16..da5ec2c 100644
@@ -15,7 +15,8 @@ SDK_RDEPENDS_append_task-populate-sdk-ext = " ${SDK_TARGETS}"
 
 SDK_RELOCATE_AFTER_INSTALL_task-populate-sdk-ext = "0"
 
-SDK_META_CONF_WHITELIST ?= "MACHINE DISTRO PACKAGE_CLASSES"
+SDK_LOCAL_CONF_WHITELIST ?= ""
+SDK_INHERIT_BLACKLIST ?= "buildhistory icecc"
 
 SDK_TARGETS ?= "${PN}"
 OE_INIT_ENV_SCRIPT ?= "oe-init-build-env"

@@ -109,15 +110,33 @@ python copy_buildsystem () {
         f.write('    "\n')
 
     # Create local.conf
+    local_conf_whitelist = d.getVar('SDK_LOCAL_CONF_WHITELIST', True).split()
+    def handle_var(varname, origvalue, op, newlines):
+        if origvalue.strip().startswith('/') and not varname in local_conf_whitelist:
+            newlines.append('# Removed original setting of %s\n' % varname)
+            return None, op, 0, True
+        else:
+            return origvalue, op, 0, True
+    varlist = ['[^#=+ ]*']
+    builddir = d.getVar('TOPDIR', True)
+    with open(builddir + '/conf/local.conf', 'r') as f:
+        oldlines = f.readlines()
+    (updated, newlines) = bb.utils.edit_metadata(oldlines, varlist, handle_var)
+
     with open(baseoutpath + '/conf/local.conf', 'w') as f:
         f.write('# WARNING: this configuration has been automatically generated and in\n')
         f.write('# most cases should not be edited. If you need more flexibility than\n')
         f.write('# this configuration provides, it is strongly suggested that you set\n')
         f.write('# up a proper instance of the full build system and use that instead.\n\n')
+        for line in newlines:
+            f.write(line)
 
         f.write('INHERIT += "%s"\n\n' % 'uninative')
         f.write('CONF_VERSION = "%s"\n\n' % d.getVar('CONF_VERSION', False))
 
+        # Some classes are not suitable for SDK, remove them from INHERIT
+        f.write('INHERIT_remove = "%s"\n' % d.getVar('SDK_INHERIT_BLACKLIST'))
+    
         # This is a bit of a hack, but we really don't want these dependencies
         # (we're including them in the SDK as nativesdk- versions instead)
         f.write('POKYQEMUDEPS_forcevariable = ""\n\n')

@@ -134,8 +153,6 @@ python copy_buildsystem () {
         # Ensure locked sstate cache objects are re-used without error
         f.write('SIGGEN_LOCKEDSIGS_CHECK_LEVEL = "warn"\n\n')
 
-        for varname in d.getVar('SDK_META_CONF_WHITELIST', True).split():
-            f.write('%s = "%s"\n' % (varname, d.getVar(varname, True)))
         f.write('require conf/locked-sigs.inc\n')
         f.write('require conf/work-config.inc\n')
 

index da5ec2c..df87edb 100644
@@ -137,11 +137,6 @@ python copy_buildsystem () {
         # Some classes are not suitable for SDK, remove them from INHERIT
         f.write('INHERIT_remove = "%s"\n' % d.getVar('SDK_INHERIT_BLACKLIST'))
     
-        # This is a bit of a hack, but we really don't want these dependencies
-        # (we're including them in the SDK as nativesdk- versions instead)
-        f.write('POKYQEMUDEPS_forcevariable = ""\n\n')
-        f.write('EXTRA_IMAGEDEPENDS_remove = "qemu-native qemu-helper-native"\n\n')
-
         # Bypass the default connectivity check if any
         f.write('CONNECTIVITY_CHECK_URIS = ""\n\n')
 

index df87edb..d2ebcef 100644
@@ -217,11 +217,13 @@ sdk_ext_postinst() {
 	# For now this is where uninative.bbclass expects the tarball
 	mv *-nativesdk-libc.tar.* $target_sdk_dir/`dirname ${oe_init_build_env_path}`
 
-	printf "Preparing build system...\n"
-	# dash which is /bin/sh on Ubuntu will not preserve the
-	# current working directory when first ran, nor will it set $1 when
-	# sourcing a script. That is why this has to look so ugly.
-	sh -c ". buildtools/environment-setup* > preparing_build_system.log && cd $target_sdk_dir/`dirname ${oe_init_build_env_path}` && set $target_sdk_dir && . $target_sdk_dir/${oe_init_build_env_path} $target_sdk_dir >> preparing_build_system.log && bitbake ${SDK_TARGETS} >> preparing_build_system.log" || { echo "SDK preparation failed: see `pwd`/preparing_build_system.log" ; exit 1 ; }
+	if [ "$prepare_buildsystem" != "no" ]; then
+	    printf "Preparing build system...\n"
+	    # dash which is /bin/sh on Ubuntu will not preserve the
+	    # current working directory when first ran, nor will it set $1 when
+	    # sourcing a script. That is why this has to look so ugly.
+	    sh -c ". buildtools/environment-setup* > preparing_build_system.log && cd $target_sdk_dir/`dirname ${oe_init_build_env_path}` && set $target_sdk_dir && . $target_sdk_dir/${oe_init_build_env_path} $target_sdk_dir >> preparing_build_system.log && bitbake ${SDK_TARGETS} >> preparing_build_system.log" || { echo "SDK preparation failed: see `pwd`/preparing_build_system.log" ; exit 1 ; }
+	fi
 	echo done
 }
 

index 3a50991..23cb69a 100644
@@ -32,7 +32,7 @@ answer=""
 relocate=1
 savescripts=0
 verbose=0
-while getopts ":yd:DRS" OPT; do
+while getopts ":yd:nDRS" OPT; do
 	case $OPT in
 	y)
 		answer="Y"

@@ -41,6 +41,9 @@ while getopts ":yd:DRS" OPT; do
 	d)
 		target_sdk_dir=$OPTARG
 		;;
+	n)
+		prepare_buildsystem="no"
+		;;
 	D)
 		verbose=1
 		;;

@@ -55,6 +58,8 @@ while getopts ":yd:DRS" OPT; do
 		echo "Usage: $(basename $0) [-y] [-d <dir>]"
 		echo "  -y         Automatic yes to all prompts"
 		echo "  -d <dir>   Install the SDK to <dir>"
+		echo "======== Extensible SDK only options ============"
+		echo "  -n         Do not prepare the build system"
 		echo "======== Advanced DEBUGGING ONLY OPTIONS ========"
 		echo "  -S         Save relocation scripts"
 		echo "  -R         Do not relocate executables"

new file mode 100755
index 0000000..21d4d52
@@ -0,0 +1,143 @@
+#!/usr/bin/env python
+
+# OpenEmbedded SDK Tool
+
+# oe-publish-sdk publish <ext-sdk> <destination>
+# <ext-sdk>: extensible SDK to publish
+# <destination>: local or remote location which servers as an SDK update server
+# e.g.
+# oe-publish-sdk publish sdk-ext.sh /mnt/poky/sdk-ext
+# oe-publish-sdk publish sdk-ext.sh user@host:/opt/poky/sdk-ext
+#
+
+import sys
+import os
+import argparse
+import glob
+import re
+import subprocess
+import logging
+import shutil
+import errno
+
+scripts_path = os.path.dirname(os.path.realpath(__file__))
+lib_path = scripts_path + '/lib'
+sys.path = sys.path + [lib_path]
+import scriptutils
+logger = scriptutils.logger_create('sdktool')
+
+def mkdir(d):
+    try:
+        os.makedirs(d)
+    except OSError as e:
+        if e.errno != errno.EEXIST:
+            raise e
+
+def publish(args):
+    logger.debug("In publish function")
+    target_sdk = args.sdk
+    destination = args.dest
+    logger.debug("target_sdk = %s, update_server = %s" % (target_sdk, destination))
+    sdk_basename = os.path.basename(target_sdk)
+
+    # Ensure the SDK exists
+    if not os.path.exists(target_sdk):
+        logger.error("%s doesn't exist" % target_sdk)
+        return -1
+    
+    if ':' in destination:
+        is_remote = True
+        host, destdir = destination.split(':')
+        dest_sdk = os.path.join(destdir, sdk_basename)
+    else:
+        is_remote = False
+        dest_sdk = os.path.join(destination, sdk_basename)
+        
+    # Making sure the directory exists
+    logger.debug("Making sure the destination directory exists")
+    if not is_remote:
+        mkdir(destination)
+    else:
+        cmd = "ssh %s 'mkdir -p %s'" % (host, destdir)
+        ret = subprocess.call(cmd, shell=True)
+        if ret != 0:
+            logger.error("Making directory %s on %s failed" % (destdir, host))
+            return ret
+        
+    # Copying the SDK to the destination
+    logger.debug("Copying the SDK to destination")
+    if not is_remote:
+        if os.path.exists(dest_sdk):
+            os.remove(dest_sdk)
+        if (os.stat(target_sdk).st_dev == os.stat(destination).st_dev):
+            os.link(target_sdk, dest_sdk)
+        else:
+            shutil.copy(target_sdk, dest_sdk)
+    else:
+        cmd = "scp %s %s" % (target_sdk, destination)
+        ret = subprocess.call(cmd, shell=True)
+        if ret != 0:
+            logger.error("scp %s %s failed" % (target_sdk, destination))
+            return ret
+    
+    # Unpack the SDK
+    logger.debug("Unpacking SDK")
+    if not is_remote:
+        cmd = "sh %s -n -y -d %s" % (dest_sdk, destination)
+        ret = subprocess.call(cmd, shell=True)
+        if ret == 0:
+            logger.info('Successfully unpacked %s to %s' % (dest_sdk, destination))
+        else:
+            logger.error('Failed to unpack %s to %s' % (dest_sdk, destination))
+            return ret
+    else:
+        cmd = "ssh %s 'sh %s -n -y -d %s'" % (host, dest_sdk, destdir)
+        ret = subprocess.call(cmd, shell=True)
+        if ret == 0:
+            logger.info('Successfully unpacked %s to %s' % (dest_sdk, destdir))
+        else:
+            logger.error('Failed to unpack %s to %s' % (dest_sdk, destdir))
+            return ret
+
+    # Setting up the git repo
+    if not is_remote:
+        cmd = 'cd %s/layers; if [ ! -e .git ]; then git init .; mv .git/hooks/post-update.sample .git/hooks/post-update; fi; git add .; git commit -m "init repo" || true;' % destination
+    else:
+        cmd = "ssh %s 'cd %s/layers; if [ ! -e .git ]; then git init .; mv .git/hooks/post-update.sample .git/hooks/post-update; fi; git add .; git commit -m \"init repo\" || true;'" % (host, destdir)
+    ret = subprocess.call(cmd, shell=True)
+    if ret == 0:
+        logger.info('Git repo set up successfully')
+    else:
+        logger.error('Failed to set up git repo')
+    return ret
+
+
+def main():
+    parser = argparse.ArgumentParser(description="OpenEmbedded development tool",
+                                     epilog="Use %(prog)s <subcommand> --help to get help on a specific command")
+    parser.add_argument('-d', '--debug', help='Enable debug output', action='store_true')
+    parser.add_argument('-q', '--quiet', help='Print only errors', action='store_true')
+
+    parser.add_argument('sdk', help='Extensible SDK to publish')
+    parser.add_argument('dest', help='Destination to publish SDK to')
+    
+    parser.set_defaults(func=publish)
+
+    args = parser.parse_args()
+
+    if args.debug:
+        logger.setLevel(logging.DEBUG)
+    elif args.quiet:
+        logger.setLevel(logging.ERROR)
+
+    ret = args.func(args)
+    return ret
+
+if __name__ == "__main__":
+    try:
+        ret = main()
+    except Exception:
+        ret = 1
+        import traceback
+        traceback.print_exc(5)
+    sys.exit(ret)

index d2ebcef..151ae1d 100644
@@ -84,6 +84,8 @@ python copy_buildsystem () {
     config.set('General', 'bitbake_subdir', conf_bbpath)
     config.set('General', 'init_path', conf_initpath)
     config.set('General', 'core_meta_subdir', core_meta_subdir)
+    config.add_section('SDK')
+    config.set('SDK', 'sdk_targets', d.getVar('SDK_TARGETS', True))
     bb.utils.mkdirhier(os.path.join(baseoutpath, 'conf'))
     with open(os.path.join(baseoutpath, 'conf', 'devtool.conf'), 'w') as f:
         config.write(f)

new file mode 100644
index 0000000..9bc21ea
@@ -0,0 +1,188 @@
+# Development tool - sdk-update command plugin
+
+import os
+import subprocess
+import logging
+import glob
+import shutil
+import errno
+import sys
+from devtool import exec_build_env_command, setup_tinfoil
+
+logger = logging.getLogger('devtool')
+
+def plugin_init(pluginlist):
+    """Plugin initialization"""
+    pass
+
+def parse_locked_sigs(sigfile_path):
+    """Return <pn:task>:<hash> dictionary"""
+    sig_dict = {}
+    with open(sigfile_path) as f:
+        lines = f.readlines()
+        for line in lines:
+            if ':' in line:
+                taskkey, _, hashval = line.rpartition(':')
+                sig_dict[taskkey.strip()] = hashval.split()[0]
+    return sig_dict
+
+def generate_update_dict(sigfile_new, sigfile_old):
+    """Return a dict containing <pn:task>:<hash> which indicates what need to be updated"""
+    update_dict = {}
+    sigdict_new = parse_locked_sigs(sigfile_new)
+    sigdict_old = parse_locked_sigs(sigfile_old)
+    for k in sigdict_new:
+        if k not in sigdict_old:
+            update_dict[k] = sigdict_new[k]
+            continue
+        if sigdict_new[k] != sigdict_old[k]:
+            update_dict[k] = sigdict_new[k]
+            continue
+    return update_dict
+
+def get_sstate_objects(update_dict, newsdk_path):
+    """Return a list containing sstate objects which are to be installed"""
+    sstate_objects = []
+    # Ensure newsdk_path points to an extensible SDK
+    sstate_dir = os.path.join(newsdk_path, 'sstate-cache')
+    if not os.path.exists(sstate_dir):
+        logger.error("sstate-cache directory not found under %s" % newsdk_path)
+        raise
+    for k in update_dict:
+        files = set()
+        hashval = update_dict[k]
+        p = sstate_dir + '/' + hashval[:2] + '/*' + hashval + '*.tgz'
+        files |= set(glob.glob(p))
+        p = sstate_dir + '/*/' + hashval[:2] + '/*' + hashval + '*.tgz'
+        files |= set(glob.glob(p))
+        files = list(files)
+        if len(files) == 1:
+            sstate_objects.extend(files)
+        elif len(files) > 1:
+            logger.error("More than one matching sstate object found for %s" % hashval)
+        
+    return sstate_objects
+
+def mkdir(d):
+    try:
+        os.makedirs(d)
+    except OSError as e:
+        if e.errno != errno.EEXIST:
+            raise e
+
+def install_sstate_objects(sstate_objects, src_sdk, dest_sdk):
+    """Install sstate objects into destination SDK"""
+    sstate_dir = os.path.join(dest_sdk, 'sstate-cache')
+    if not os.path.exists(sstate_dir):
+        logger.error("Missing sstate-cache directory in %s, it might not be an extensible SDK." % dest_sdk)
+        raise
+    for sb in sstate_objects:
+        dst = sb.replace(src_sdk, dest_sdk)
+        destdir = os.path.dirname(dst)
+        mkdir(destdir)
+        logger.debug("Copying %s to %s" % (sb, dst))
+        shutil.copy(sb, dst)
+
+def sdk_update(args, config, basepath, workspace):
+    # Fetch locked-sigs.inc file from remote/local destination
+    logger.debug("updateserver: %s" % args.updateserver)
+
+    # Make sure we are using sdk-update from within SDK
+    logger.debug("basepath = %s" % basepath)
+    old_locked_sig_file_path = os.path.join(basepath, 'conf/locked-sigs.inc')
+    if not os.path.exists(old_locked_sig_file_path):
+        logger.error("Not using devtool's sdk-update command from within an extensible SDK. Please specify correct basepath via --basepath option")
+        return -1
+    else:
+        logger.debug("Found conf/locked-sigs.inc in %s" % basepath)
+        
+    if ':' in args.updateserver:
+        is_remote = True
+    else:
+        is_remote = False
+        
+    if not is_remote:
+        # devtool sdk-update /local/path/to/latest/sdk
+        new_locked_sig_file_path = os.path.join(args.updateserver, 'conf/locked-sigs.inc')
+        if not os.path.exists(new_locked_sig_file_path):
+            logger.error("%s doesn't exist or is not an extensible SDK" % args.updateserver)
+            return -1
+        else:
+            logger.debug("Found conf/locked-sigs.inc in %s" % args.updateserver)
+        update_dict = generate_update_dict(new_locked_sig_file_path, old_locked_sig_file_path)
+        logger.debug("update_dict = %s" % update_dict)
+        sstate_objects = get_sstate_objects(update_dict, args.updateserver)
+        logger.debug("sstate_objects = %s" % sstate_objects)
+        if len(sstate_objects) == 0:
+            logger.info("No need to update.")
+            return 0
+        logger.info("Installing sstate objects into %s", basepath)
+        install_sstate_objects(sstate_objects, args.updateserver.rstrip('/'), basepath)
+        logger.info("Updating configuration files")
+        new_conf_dir = os.path.join(args.updateserver, 'conf')
+        old_conf_dir = os.path.join(basepath, 'conf')
+        shutil.rmtree(old_conf_dir)
+        shutil.copytree(new_conf_dir, old_conf_dir)
+        logger.info("Updating layers")
+        new_layers_dir = os.path.join(args.updateserver, 'layers')
+        old_layers_dir = os.path.join(basepath, 'layers')
+        shutil.rmtree(old_layers_dir)
+        shutil.copytree(new_layers_dir, old_layers_dir)
+    else:
+        # devtool sdk-update http://myhost/sdk
+        tmpsdk_dir = '/tmp/sdk-ext'
+        if os.path.exists(tmpsdk_dir):
+            shutil.rmtree(tmpsdk_dir)
+        os.makedirs(tmpsdk_dir)
+        os.makedirs(os.path.join(tmpsdk_dir, 'conf'))
+        # Fetch locked-sigs.inc from update server
+        ret = subprocess.call("wget -q -O - %s/conf/locked-sigs.inc > %s/locked-sigs.inc" % (args.updateserver, os.path.join(tmpsdk_dir, 'conf')), shell=True)
+        if ret != 0:
+            logger.error("Fetching conf/locked-sigs.inc from %s to %s/locked-sigs.inc failed" % (args.updateserver, os.path.join(tmpsdk_dir, 'conf')))
+            return ret
+        else:
+            logger.info("Fetching conf/locked-sigs.inc from %s to %s/locked-sigs.inc succeeded" % (args.updateserver, os.path.join(tmpsdk_dir, 'conf')))
+        new_locked_sig_file_path = os.path.join(tmpsdk_dir, 'conf/locked-sigs.inc')
+        update_dict = generate_update_dict(new_locked_sig_file_path, old_locked_sig_file_path)
+        logger.debug("update_dict = %s" % update_dict)
+        if len(update_dict) == 0:
+            logger.info("No need to update.")
+            return 0
+        # Update metadata
+        logger.debug("Updating meta data via git ...")
+        # Try using 'git pull', if failed, use 'git clone'
+        if os.path.exists(os.path.join(basepath, 'layers/.git')):
+            ret = subprocess.call("cd layers && git pull", shell=True)
+        else:
+            ret = -1
+        if ret != 0:
+            ret = subprocess.call("rm -rf layers && git clone %s/layers" % args.updateserver, shell=True)
+        if ret != 0:
+            logger.error("Updating meta data via git failed")
+            return ret
+        logger.debug("Updating conf files ...")
+        conf_files = ['local.conf', 'bblayers.conf', 'devtool.conf', 'work-config.inc', 'locked-sigs.inc']
+        for conf in conf_files:
+            ret = subprocess.call("wget -q -O - %s/conf/%s > conf/%s" % (args.updateserver, conf, conf), shell=True)
+            if ret != 0:
+                logger.error("Update %s failed" % conf)
+                return ret
+        with open(os.path.join(basepath, 'conf/local.conf'), 'a') as f:
+            f.write('SSTATE_MIRRORS_append = " file://.* %s/sstate-cache/PATH \\n "\n' % args.updateserver)
+
+    # Run bitbake command for the whole SDK
+    sdk_targets = config.get('SDK', 'sdk_targets')
+    logger.info("Executing 'bitbake %s' ... (This may take some time.)" % sdk_targets)
+    try:
+        exec_build_env_command(config.init_path, basepath, 'bitbake %s' % sdk_targets)
+    except:
+        logger.error('bitbake %s failed' % sdk_targets)
+        return -1
+    return 0
+
+def register_commands(subparsers, context):
+    """Register devtool subcommands from the sdk plugin"""
+    if context.fixed_setup:
+        parser_sdk = subparsers.add_parser('sdk-update', help='Update SDK components from a nominated destination')
+        parser_sdk.add_argument('updateserver', help='The update server to fetch latest SDK components')
+        parser_sdk.set_defaults(func=sdk_update)