Extensible SDK fixes

index de8a20c..49de74e 100755
@@ -57,13 +57,14 @@ for f in files:
     destdir = os.path.dirname(dst)
     mkdir(destdir)
 
+    src = os.path.realpath(f)
     if os.path.exists(dst):
         os.remove(dst)
-    if (os.stat(f).st_dev == os.stat(destdir).st_dev):
+    if (os.stat(src).st_dev == os.stat(destdir).st_dev):
         print('linking')
-        os.link(f, dst)
+        os.link(src, dst)
     else:
         print('copying')
-        shutil.copyfile(f, dst)
+        shutil.copyfile(src, dst)
 
 print('Done!')

index f6b0834..f758d98 100644
@@ -233,6 +233,14 @@ python copy_buildsystem () {
         f.write('    $' + '{SDKBASEMETAPATH}/workspace \\\n')
         f.write('    "\n')
 
+    # Copy uninative tarball
+    # For now this is where uninative.bbclass expects the tarball
+    uninative_file = d.expand('${SDK_DEPLOY}/${BUILD_ARCH}-nativesdk-libc.tar.bz2')
+    uninative_checksum = bb.utils.sha256_file(uninative_file)
+    uninative_outdir = '%s/downloads/uninative/%s' % (baseoutpath, uninative_checksum)
+    bb.utils.mkdirhier(uninative_outdir)
+    shutil.copy(uninative_file, uninative_outdir)
+
     env_whitelist = (d.getVar('BB_ENV_EXTRAWHITE', True) or '').split()
     env_whitelist_values = {}
 

@@ -267,7 +275,8 @@ python copy_buildsystem () {
             # Write a newline just in case there's none at the end of the original
             f.write('\n')
 
-            f.write('INHERIT += "%s"\n\n' % 'uninative')
+            f.write('INHERIT += "%s"\n' % 'uninative')
+            f.write('UNINATIVE_CHECKSUM[%s] = "%s"\n\n' % (d.getVar('BUILD_ARCH', True), uninative_checksum))
             f.write('CONF_VERSION = "%s"\n\n' % d.getVar('CONF_VERSION', False))
 
             # Some classes are not suitable for SDK, remove them from INHERIT

@@ -446,18 +455,10 @@ install_tools() {
 	lnr ${SDK_OUTPUT}/${SDKPATH}/${scriptrelpath}/recipetool ${SDK_OUTPUT}/${SDKPATHNATIVE}${bindir_nativesdk}/recipetool
 	touch ${SDK_OUTPUT}/${SDKPATH}/.devtoolbase
 
-	localconf=${SDK_OUTPUT}/${SDKPATH}/conf/local.conf
-
 	# find latest buildtools-tarball and install it
 	buildtools_path=`ls -t1 ${SDK_DEPLOY}/${@extsdk_get_buildtools_filename(d)} | head -n1`
 	install $buildtools_path ${SDK_OUTPUT}/${SDKPATH}
 
-	# For now this is where uninative.bbclass expects the tarball
-	chksum=`sha256sum ${SDK_DEPLOY}/${BUILD_ARCH}-nativesdk-libc.tar.bz2 | cut -f 1 -d ' '`
-	install -d ${SDK_OUTPUT}/${SDKPATH}/downloads/uninative/$chksum/
-	install ${SDK_DEPLOY}/${BUILD_ARCH}-nativesdk-libc.tar.bz2 ${SDK_OUTPUT}/${SDKPATH}/downloads/uninative/$chksum/
-	echo "UNINATIVE_CHECKSUM[${BUILD_ARCH}] = '$chksum'" >> ${SDK_OUTPUT}/${SDKPATH}/conf/local.conf
-
 	install -m 0644 ${COREBASE}/meta/files/ext-sdk-prepare.py ${SDK_OUTPUT}/${SDKPATH}
 }
 do_populate_sdk_ext[file-checksums] += "${COREBASE}/meta/files/ext-sdk-prepare.py:True"

index f758d98..c16e228 100644
@@ -108,9 +108,14 @@ def create_filtered_tasklist(d, sdkbasepath, tasklistfile, conf_initpath):
     # Create a temporary build directory that we can pass to the env setup script
     shutil.copyfile(sdkbasepath + '/conf/local.conf', sdkbasepath + '/conf/local.conf.bak')
     try:
+        # Force the use of sstate from the build system
         with open(sdkbasepath + '/conf/local.conf', 'a') as f:
             f.write('\nSSTATE_DIR_forcevariable = "%s"\n' % d.getVar('SSTATE_DIR', True))
             f.write('SSTATE_MIRRORS_forcevariable = ""\n')
+            # Drop uninative if the build isn't using it (or else NATIVELSBSTRING will
+            # be different and we won't be able to find our native sstate)
+            if not bb.data.inherits_class('uninative', d):
+                f.write('INHERIT_remove = "uninative"\n')
 
         # Unfortunately the default SDKPATH (or even a custom value) may contain characters that bitbake
         # will not allow in its COREBASE path, so we need to rename the directory temporarily

index c16e228..a075a09 100644
@@ -85,10 +85,11 @@ SDK_EXT_HOST_MANIFEST = "${SDK_DEPLOY}/${TOOLCHAINEXT_OUTPUTNAME}.host.manifest"
 
 SDK_TITLE_task-populate-sdk-ext = "${@d.getVar('DISTRO_NAME', True) or d.getVar('DISTRO', True)} Extensible SDK"
 
-def clean_esdk_builddir(sdkbasepath):
+def clean_esdk_builddir(d, sdkbasepath):
     """Clean up traces of the fake build for create_filtered_tasklist()"""
     import shutil
-    cleanpaths = 'tmp cache conf/sanity_info conf/templateconf.cfg downloads'.split()
+    cleanpaths = 'cache conf/sanity_info conf/templateconf.cfg'.split()
+    cleanpaths.append(os.path.basename(d.getVar('TMPDIR', True)))
     for pth in cleanpaths:
         fullpth = os.path.join(sdkbasepath, pth)
         if os.path.isdir(fullpth):

@@ -140,7 +141,7 @@ def create_filtered_tasklist(d, sdkbasepath, tasklistfile, conf_initpath):
             os.rename(temp_sdkbasepath, sdkbasepath)
         # Clean out residue of running bitbake, which check_sstate_task_list()
         # will effectively do
-        clean_esdk_builddir(sdkbasepath)
+        clean_esdk_builddir(d, sdkbasepath)
     finally:
         os.replace(sdkbasepath + '/conf/local.conf.bak', sdkbasepath + '/conf/local.conf')
 

index 4d3faf6..3d5a746 100644
@@ -201,21 +201,25 @@ def merge_lockedsigs(copy_tasks, lockedsigs_main, lockedsigs_extra, merged_outpu
         write_sigs_file(merged_output, arch_order, merged)
 
 def create_locked_sstate_cache(lockedsigs, input_sstate_cache, output_sstate_cache, d, fixedlsbstring="", filterfile=None):
+    import shutil
     bb.note('Generating sstate-cache...')
 
     nativelsbstring = d.getVar('NATIVELSBSTRING', True)
     bb.process.run("gen-lockedsig-cache %s %s %s %s %s" % (lockedsigs, input_sstate_cache, output_sstate_cache, nativelsbstring, filterfile or ''))
-    if fixedlsbstring:
+    if fixedlsbstring and nativelsbstring != fixedlsbstring:
         nativedir = output_sstate_cache + '/' + nativelsbstring
         if os.path.isdir(nativedir):
             destdir = os.path.join(output_sstate_cache, fixedlsbstring)
-            bb.utils.mkdirhier(destdir)
-
-            dirlist = os.listdir(nativedir)
-            for i in dirlist:
-                src = os.path.join(nativedir, i)
-                dest = os.path.join(destdir, i)
-                os.rename(src, dest)
+            for root, _, files in os.walk(nativedir):
+                for fn in files:
+                    src = os.path.join(root, fn)
+                    dest = os.path.join(destdir, os.path.relpath(src, nativedir))
+                    if os.path.exists(dest):
+                        # Already exists, and it'll be the same file, so just delete it
+                        os.unlink(src)
+                    else:
+                        bb.utils.mkdirhier(os.path.dirname(dest))
+                        shutil.move(src, dest)
 
 def check_sstate_task_list(d, targets, filteroutfile, cmdprefix='', cwd=None, logfile=None):
     import subprocess

index a075a09..18013564 100644
@@ -490,14 +490,16 @@ SDK_PRE_INSTALL_COMMAND_task-populate-sdk-ext = "${sdk_ext_preinst}"
 sdk_ext_postinst() {
 	printf "\nExtracting buildtools...\n"
 	cd $target_sdk_dir
-	printf "buildtools\ny" | ./*buildtools-nativesdk-standalone* > /dev/null || ( printf 'ERROR: buildtools installation failed\n' ; exit 1 )
+	env_setup_script="$target_sdk_dir/environment-setup-${REAL_MULTIMACH_TARGET_SYS}"
+	printf "buildtools\ny" | ./*buildtools-nativesdk-standalone* > buildtools.log || { printf 'ERROR: buildtools installation failed:\n' ; cat buildtools.log ; echo "printf 'ERROR: this SDK was not fully installed and needs reinstalling\n'" >> $env_setup_script ; exit 1 ; }
 
 	# Delete the buildtools tar file since it won't be used again
 	rm ./*buildtools-nativesdk-standalone*.sh -f
+	# We don't need the log either since it succeeded
+	rm -f buildtools.log
 
 	# Make sure when the user sets up the environment, they also get
 	# the buildtools-tarball tools in their path.
-	env_setup_script="$target_sdk_dir/environment-setup-${REAL_MULTIMACH_TARGET_SYS}"
 	echo ". $target_sdk_dir/buildtools/environment-setup*" >> $env_setup_script
 
 	# Allow bitbake environment setup to be ran as part of this sdk.

index 18013564..2464acb 100644
@@ -452,8 +452,12 @@ python copy_buildsystem () {
                 f.write('%s\t%s\n' % (chksum, os.path.relpath(fn, baseoutpath)))
 }
 
-def extsdk_get_buildtools_filename(d):
-    return '*-buildtools-nativesdk-standalone-*.sh'
+def get_current_buildtools(d):
+    """Get the file name of the current buildtools installer"""
+    import glob
+    btfiles = glob.glob(os.path.join(d.getVar('SDK_DEPLOY', True), '*-buildtools-nativesdk-standalone-*.sh'))
+    btfiles.sort(key=os.path.getctime)
+    return os.path.basename(btfiles[-1])
 
 install_tools() {
 	install -d ${SDK_OUTPUT}/${SDKPATHNATIVE}${bindir_nativesdk}

@@ -462,8 +466,7 @@ install_tools() {
 	touch ${SDK_OUTPUT}/${SDKPATH}/.devtoolbase
 
 	# find latest buildtools-tarball and install it
-	buildtools_path=`ls -t1 ${SDK_DEPLOY}/${@extsdk_get_buildtools_filename(d)} | head -n1`
-	install $buildtools_path ${SDK_OUTPUT}/${SDKPATH}
+	install ${SDK_DEPLOY}/${SDK_BUILDTOOLS_INSTALLER} ${SDK_OUTPUT}/${SDKPATH}
 
 	install -m 0644 ${COREBASE}/meta/files/ext-sdk-prepare.py ${SDK_OUTPUT}/${SDKPATH}
 }

@@ -491,10 +494,10 @@ sdk_ext_postinst() {
 	printf "\nExtracting buildtools...\n"
 	cd $target_sdk_dir
 	env_setup_script="$target_sdk_dir/environment-setup-${REAL_MULTIMACH_TARGET_SYS}"
-	printf "buildtools\ny" | ./*buildtools-nativesdk-standalone* > buildtools.log || { printf 'ERROR: buildtools installation failed:\n' ; cat buildtools.log ; echo "printf 'ERROR: this SDK was not fully installed and needs reinstalling\n'" >> $env_setup_script ; exit 1 ; }
+	printf "buildtools\ny" | ./${SDK_BUILDTOOLS_INSTALLER} > buildtools.log || { printf 'ERROR: buildtools installation failed:\n' ; cat buildtools.log ; echo "printf 'ERROR: this SDK was not fully installed and needs reinstalling\n'" >> $env_setup_script ; exit 1 ; }
 
 	# Delete the buildtools tar file since it won't be used again
-	rm ./*buildtools-nativesdk-standalone*.sh -f
+	rm -f ./${SDK_BUILDTOOLS_INSTALLER}
 	# We don't need the log either since it succeeded
 	rm -f buildtools.log
 

@@ -538,6 +541,8 @@ fakeroot python do_populate_sdk_ext() {
         bb.fatal('The extensible SDK can currently only be built for the same architecture as the machine being built on - SDK_ARCH is set to %s (likely via setting SDKMACHINE) which is different from the architecture of the build machine (%s). Unable to continue.' % (d.getVar('SDK_ARCH', True), d.getVar('BUILD_ARCH', True)))
 
     d.setVar('SDK_INSTALL_TARGETS', get_sdk_install_targets(d))
+    buildtools_fn = get_current_buildtools(d)
+    d.setVar('SDK_BUILDTOOLS_INSTALLER', buildtools_fn)
 
     bb.build.exec_func("do_populate_sdk", d)
 }

index 9279310..4345104 100644
@@ -47,7 +47,8 @@ relocate=1
 savescripts=0
 verbose=0
 publish=0
-while getopts ":yd:npDRS" OPT; do
+listcontents=0
+while getopts ":yd:npDRSl" OPT; do
 	case $OPT in
 	y)
 		answer="Y"

@@ -72,6 +73,9 @@ while getopts ":yd:npDRS" OPT; do
 	S)
 		savescripts=1
 		;;
+	l)
+		listcontents=1
+		;;
 	*)
 		echo "Usage: $(basename $0) [-y] [-d <dir>]"
 		echo "  -y         Automatic yes to all prompts"

@@ -83,11 +87,18 @@ while getopts ":yd:npDRS" OPT; do
 		echo "  -S         Save relocation scripts"
 		echo "  -R         Do not relocate executables"
 		echo "  -D         use set -x to see what is going on"
+		echo "  -l         list files that will be extracted"
 		exit 1
 		;;
 	esac
 done
 
+payload_offset=$(($(grep -na -m1 "^MARKER:$" $0|cut -d':' -f1) + 1))
+if [ "$listcontents" = "1" ] ; then
+    tail -n +$payload_offset $0| tar tvJ || exit 1
+    exit
+fi
+
 titlestr="@SDK_TITLE@ installer version @SDK_VERSION@"
 printf "%s\n" "$titlestr"
 printf "%${#titlestr}s\n" | tr " " "="

@@ -187,8 +198,6 @@ if [ ! -x $target_sdk_dir -o ! -w $target_sdk_dir -o ! -r $target_sdk_dir ]; the
 	$SUDO_EXEC mkdir -p $target_sdk_dir >/dev/null 2>&1
 fi
 
-payload_offset=$(($(grep -na -m1 "^MARKER:$" $0|cut -d':' -f1) + 1))
-
 printf "Extracting SDK..."
 tail -n +$payload_offset $0| $SUDO_EXEC tar xJ -C $target_sdk_dir --checkpoint=.2500 $EXTRA_TAR_OPTIONS || exit 1
 echo "done"

index 2464acb..f1ae7c1 100644
@@ -459,6 +459,36 @@ def get_current_buildtools(d):
     btfiles.sort(key=os.path.getctime)
     return os.path.basename(btfiles[-1])
 
+def get_sdk_required_utilities(buildtools_fn, d):
+    """Find required utilities that aren't provided by the buildtools"""
+    sanity_required_utilities = (d.getVar('SANITY_REQUIRED_UTILITIES', True) or '').split()
+    sanity_required_utilities.append(d.expand('${BUILD_PREFIX}gcc'))
+    sanity_required_utilities.append(d.expand('${BUILD_PREFIX}g++'))
+    buildtools_installer = os.path.join(d.getVar('SDK_DEPLOY', True), buildtools_fn)
+    filelist, _ = bb.process.run('%s -l' % buildtools_installer)
+    localdata = bb.data.createCopy(d)
+    localdata.setVar('SDKPATH', '.')
+    sdkpathnative = localdata.getVar('SDKPATHNATIVE', True)
+    sdkbindirs = [localdata.getVar('bindir_nativesdk', True),
+                  localdata.getVar('sbindir_nativesdk', True),
+                  localdata.getVar('base_bindir_nativesdk', True),
+                  localdata.getVar('base_sbindir_nativesdk', True)]
+    for line in filelist.splitlines():
+        splitline = line.split()
+        if len(splitline) > 5:
+            fn = splitline[5]
+            if not fn.startswith('./'):
+                fn = './%s' % fn
+            if fn.startswith(sdkpathnative):
+                relpth = '/' + os.path.relpath(fn, sdkpathnative)
+                for bindir in sdkbindirs:
+                    if relpth.startswith(bindir):
+                        relpth = os.path.relpath(relpth, bindir)
+                        if relpth in sanity_required_utilities:
+                            sanity_required_utilities.remove(relpth)
+                        break
+    return ' '.join(sanity_required_utilities)
+
 install_tools() {
 	install -d ${SDK_OUTPUT}/${SDKPATHNATIVE}${bindir_nativesdk}
 	lnr ${SDK_OUTPUT}/${SDKPATH}/${scriptrelpath}/devtool ${SDK_OUTPUT}/${SDKPATHNATIVE}${bindir_nativesdk}/devtool

@@ -472,13 +502,38 @@ install_tools() {
 }
 do_populate_sdk_ext[file-checksums] += "${COREBASE}/meta/files/ext-sdk-prepare.py:True"
 
-# Since bitbake won't run as root it doesn't make sense to try and install
-# the extensible sdk as root.
 sdk_ext_preinst() {
+	# Since bitbake won't run as root it doesn't make sense to try and install
+	# the extensible sdk as root.
 	if [ "`id -u`" = "0" ]; then
 		echo "ERROR: The extensible sdk cannot be installed as root."
 		exit 1
 	fi
+	if ! command -v locale > /dev/null; then
+		echo "ERROR: The installer requires the locale command, please install it first"
+		exit 1
+	fi
+        # Check setting of LC_ALL set above
+	canonicalised_locale=`echo $LC_ALL | sed 's/UTF-8/utf8/'`
+	if ! locale -a | grep -q $canonicalised_locale ; then
+		echo "ERROR: the installer requires the $LC_ALL locale to be installed (but not selected), please install it first"
+		exit 1
+	fi
+	# The relocation script used by buildtools installer requires python
+	if ! command -v python > /dev/null; then
+		echo "ERROR: The installer requires python, please install it first"
+		exit 1
+	fi
+	missing_utils=""
+	for util in ${SDK_REQUIRED_UTILITIES}; do
+		if ! command -v $util > /dev/null; then
+			missing_utils="$missing_utils $util"
+		fi
+	done
+	if [ -n "$missing_utils" ] ; then
+		echo "ERROR: the SDK requires the following missing utilities, please install them: $missing_utils"
+		exit 1
+	fi
 	SDK_EXTENSIBLE="1"
 	if [ "$publish" = "1" ] ; then
 		EXTRA_TAR_OPTIONS="$EXTRA_TAR_OPTIONS --exclude=ext-sdk-prepare.py"

@@ -542,6 +597,7 @@ fakeroot python do_populate_sdk_ext() {
 
     d.setVar('SDK_INSTALL_TARGETS', get_sdk_install_targets(d))
     buildtools_fn = get_current_buildtools(d)
+    d.setVar('SDK_REQUIRED_UTILITIES', get_sdk_required_utilities(buildtools_fn, d))
     d.setVar('SDK_BUILDTOOLS_INSTALLER', buildtools_fn)
 
     bb.build.exec_func("do_populate_sdk", d)

index 4345104..66c017f 100644
@@ -143,6 +143,16 @@ if [ "$SDK_EXTENSIBLE" = "1" ]; then
 		     "characters such as spaces, @, \$ or +. Abort!"
 		exit 1
 	fi
+	# The build system doesn't work well with /tmp on NFS
+	fs_dev_path="$target_sdk_dir"
+	while [ ! -d "$fs_dev_path" ] ; do
+		fs_dev_path=`dirname $fs_dev_path`
+        done
+	fs_dev_type=`stat -f -c '%t' "$fs_dev_path"`
+	if [ "$fsdevtype" = "6969" ] ; then
+		echo "The target directory path $target_sdk_dir is on NFS, this is not possible. Abort!"
+		exit 1
+	fi
 else
 	if [ -n "$(echo $target_sdk_dir|grep ' ')" ]; then
 		echo "The target directory path ($target_sdk_dir) contains spaces. Abort!"

index f1ae7c1..b52ad41 100644
@@ -578,7 +578,7 @@ sdk_ext_postinst() {
 		# current working directory when first ran, nor will it set $1 when
 		# sourcing a script. That is why this has to look so ugly.
 		LOGFILE="$target_sdk_dir/preparing_build_system.log"
-		sh -c ". buildtools/environment-setup* > $LOGFILE && cd $target_sdk_dir/`dirname ${oe_init_build_env_path}` && set $target_sdk_dir && . $target_sdk_dir/${oe_init_build_env_path} $target_sdk_dir >> $LOGFILE && python $target_sdk_dir/ext-sdk-prepare.py $LOGFILE '${SDK_INSTALL_TARGETS}'" || { echo "ERROR: SDK preparation failed: see $LOGFILE for a slightly more detailed log"; echo "printf 'ERROR: this SDK was not fully installed and needs reinstalling\n'" >> $env_setup_script ; exit 1 ; }
+		sh -c ". buildtools/environment-setup* > $LOGFILE && cd $target_sdk_dir/`dirname ${oe_init_build_env_path}` && set $target_sdk_dir && . $target_sdk_dir/${oe_init_build_env_path} $target_sdk_dir >> $LOGFILE && python $target_sdk_dir/ext-sdk-prepare.py $LOGFILE '${SDK_INSTALL_TARGETS}'" || { echo "printf 'ERROR: this SDK was not fully installed and needs reinstalling\n'" >> $env_setup_script ; exit 1 ; }
 		rm $target_sdk_dir/ext-sdk-prepare.py
 	fi
 	echo done

index 8b15982..78c1d16 100644
@@ -59,7 +59,7 @@ def main():
                 for line in f:
                     logf.write(line)
         if ret:
-            print('ERROR: SDK preparation failed: see %s' % logfile)
+            print('ERROR: SDK preparation failed: error log written to %s' % logfile)
             return ret
 
 if __name__ == "__main__":