scripts/install-buildtools: improvements
PATCH v2 1/6 scripts/install-buildtools: improvements
* Install directory defaults to scripts/../buildtools e.g. --directory is set by default
This avoids the user having to type in their sudo password to install in /opt/poky/<installer-version>
* Use "." rather than "source" for sourcing the environment script as not all distros (e.g. Debian) have "source" by default.
* Add buildtools/ to .gitignore
* Fix typos in example usage (--install-version -> --installer-version)
[YOCTO #13832]
--- .gitignore | 1 + scripts/install-buildtools | 13 +++++++++---- 2 files changed, 10 insertions(+), 4 deletions(-)
.gitignore +1 -0
@@ -9,6 +9,7 @@ pstage/ scripts/oe-git-proxy-socks sources/ meta-*/ +buildtools/ !meta-skeleton !meta-selftest hob-image-*.bb
scripts/install-buildtools +9 -4
@@ -17,7 +17,7 @@ # $ install-buildtools \ # --base-url http://downloads.yoctoproject.org/releases/yocto \ # --release yocto-3.1_M2 \ -# --install-version 3.0+snapshot +# --installer-version 3.0+snapshot # --build-date 202000122 # # Example usage (standard buildtools from release):
@@ -29,7 +29,7 @@ # $ install-buildtools --without-extended-buildtools \ # --base-url http://downloads.yoctoproject.org/releases/yocto \ # --release yocto-3.0.2 \ -# --install-version 3.0.2 +# --installer-version 3.0.2 # import argparse
@@ -59,6 +59,7 @@ if not bitbakepath: PROGNAME = 'install-buildtools' logger = scriptutils.logger_create(PROGNAME, stream=sys.stdout) +DEFAULT_INSTALL_DIR: str = os.path.join(os.path.split(scripts_path)[0],'buildtools') DEFAULT_BASE_URL: str = 'http://downloads.yoctoproject.org/releases/yocto' DEFAULT_RELEASE: str = 'yocto-3.1_M2' DEFAULT_INSTALLER_VERSION: str = '3.0+snapshot'
@@ -66,6 +67,7 @@ DEFAULT_BUILDDATE: str = "20200122" def main(): + global DEFAULT_INSTALL_DIR global DEFAULT_BASE_URL global DEFAULT_RELEASE global DEFAULT_INSTALLER_VERSION
@@ -73,6 +75,7 @@ def main(): filename: str = "" release: str = "" buildtools_url: str = "" + install_dir: str = "" parser = argparse.ArgumentParser( description="Buildtools installation helper",
@@ -87,6 +90,7 @@ def main(): '(optional)\nRequires --url', action='store') parser.add_argument('-d', '--directory', + default=DEFAULT_INSTALL_DIR, help='directory where buildtools SDK will be installed (optional)', action='store') parser.add_argument('-r', '--release',
@@ -216,12 +220,12 @@ def main(): st = os.stat(tmpbuildtools) os.chmod(tmpbuildtools, st.st_mode | stat.S_IEXEC) logger.debug(os.stat(tmpbuildtools)) - install_dir = "/opt/poky/%s" % args.installer_version if args.directory: install_dir = args.directory ret = subprocess.call("%s -d %s -y" % (tmpbuildtools, install_dir), shell=True) else: + install_dir = "/opt/poky/%s" % args.installer_version ret = subprocess.call("%s -y" % tmpbuildtools, shell=True) if ret != 0: logger.error("Could not run buildtools installer")
@@ -238,7 +242,8 @@ def main(): tool = 'gcc' else: tool = 'tar' - proc = subprocess.run("source %s/environment-setup-x86_64-pokysdk-linux && which %s" % + logger.debug("install_dir: %s" % install_dir) + proc = subprocess.run(". %s/environment-setup-x86_64-pokysdk-linux && which %s" % (install_dir, tool), shell=True, stdout=subprocess.PIPE) which_tool = proc.stdout.decode("utf-8")
Signature · 1 line
2.24.0
PATCH v2 2/6 oe-buildenv-internal: python 3.5 as min version
Python 3.4 is EOL:
https://www.python.org/downloads/release/python-3410/
The last supported distro was probably CentOS-7, which has python36 available from epel-7 or scl (as rh-python36) [1]
[1] https://www.softwarecollections.org/en/scls/rhscl/rh-python36/
--- scripts/oe-buildenv-internal | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-)
scripts/oe-buildenv-internal +4 -4
@@ -32,12 +32,12 @@ fi # We potentially have code that doesn't parse correctly with older versions # of Python, and rather than fixing that and being eternally vigilant for # any other new feature use, just check the version here. -py_v34_check=$(python3 -c 'import sys; print(sys.version_info >= (3,4,0))') -if [ "$py_v34_check" != "True" ]; then - echo >&2 "BitBake requires Python 3.4.0 or later as 'python3'" +py_v35_check=$(python3 -c 'import sys; print(sys.version_info >= (3,5,0))') +if [ "$py_v35_check" != "True" ]; then + echo >&2 "BitBake requires Python 3.5.0 or later as 'python3'" return 1 fi -unset py_v34_check +unset py_v35_check if [ -z "$BDIR" ]; then if [ -z "$1" ]; then
Signature · 1 line
2.24.0
PATCH v2 3/6 sanity.bbclass: recommend using install-buildtools
In reply to Tim OrlingFor old tar version (< 1.28), recommend usingscripts/install-buildtools
Drop check for tar version 1.24. Dubious extra value.
--- meta/classes/sanity.bbclass | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-)
meta/classes/sanity.bbclass +1 -3
@@ -532,10 +532,8 @@ def check_tar_version(sanity_data): except subprocess.CalledProcessError as e: return "Unable to execute tar --version, exit code %d\n%s\n" % (e.returncode, e.output) version = result.split()[3] - if LooseVersion(version) < LooseVersion("1.24"): - return "Your version of tar is older than 1.24 and has bugs which will break builds. Please install a newer version of tar (1.28+).\n" if LooseVersion(version) < LooseVersion("1.28"): - return "Your version of tar is older than 1.28 and does not have the support needed to enable reproducible builds. Please install a newer version of tar (you could use the projects buildtools-tarball from our last release).\n" + return "Your version of tar is older than 1.28 and does not have the support needed to enable reproducible builds. Please install a newer version of tar (you could use the projects buildtools-tarball from our last release or use scripts/install-buildtools).\n" return None # We use git parameters and functionality only found in 1.7.8 or later
Signature · 1 line
2.24.0
PATCH v2 4/6 lib/oe/utils.py: add get_host_compiler_version()
In reply to Tim OrlingAdd helper function to get the host compiler and version. Do not assume compiler is gcc.
NOTE: cannot set env to d.getVar("PATH") as that does not contain the session PATH which was set by environment-setup-... which breaks the install-buildtools use-case
--- meta/lib/oe/utils.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+)
meta/lib/oe/utils.py +31 -0
@@ -373,6 +373,37 @@ def format_pkg_list(pkg_dict, ret_format=None): return output_str + +# Helper function to get the host compiler version +# Do not assume the compiler is gcc +def get_host_compiler_version(d, taskcontextonly=False): + import re, subprocess + + if taskcontextonly and d.getVar('BB_WORKERCONTEXT') != '1': + return + + compiler = d.getVar("BUILD_CC") + # Get rid of ccache since it is not present when parsing. + if compiler.startswith('ccache '): + compiler = compiler[7:] + try: + env = os.environ.copy() + # datastore PATH does not contain session PATH as set by environment-setup-... + # this breaks the install-buildtools use-case + # env["PATH"] = d.getVar("PATH") + output = subprocess.check_output("%s --version" % compiler, \ + shell=True, env=env, stderr=subprocess.STDOUT).decode("utf-8") + except subprocess.CalledProcessError as e: + bb.fatal("Error running %s --version: %s" % (compiler, e.output.decode("utf-8"))) + + match = re.match(r".* (\d+\.\d+)\.\d+.*", output.split('\n')[0]) + if not match: + bb.fatal("Can't get compiler version from %s --version output" % compiler) + + version = match.group(1) + return compiler, version + + def host_gcc_version(d, taskcontextonly=False): import re, subprocess
Signature · 1 line
2.24.0
PATCH v2 5/6 sanity.bbclass: add test for gcc < 5.0
In reply to Tim OrlingIt is known that the version of gcc in CentOS-7 (4.8.5) causes builds to fail.
Add a test for BUILD_CC == 'gcc' and gcc < 5.0 and recommend using scripts/install-buildtools or user built buildtools-extended-tarball.
Use the new get_host_compiler_version function from lib/oe/utils.py
NOTE: another solution is to install devtoolset-6+ from scl [1], but this is a rather large install (> 1 Gb) and fairly invasive.
[1] https://www.softwarecollections.org/en/scls/rhscl/devtoolset-6/
--- meta/classes/sanity.bbclass | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-)
meta/classes/sanity.bbclass +21 -1
@@ -521,6 +521,25 @@ def check_wsl(d): return "OpenEmbedded doesn't work under WSL at this time, sorry" return None +# The gcc version in CentOS-7 (4.8.5) is known to be a problem. +# Require at least gcc version 5.0. +# +# This can be fixed on CentOS-7 with devtoolset-6+ +# https://www.softwarecollections.org/en/scls/rhscl/devtoolset-6/ +# +# A less invasive fix is with scripts/install-buildtools (or with user +# built buildtools-extended-tarball) +# +def check_gcc_version(sanity_data): + from distutils.version import LooseVersion + import subprocess + + build_cc, version = oe.utils.get_host_compiler_version(sanity_data) + if build_cc.strip() == "gcc": + if LooseVersion(version) < LooseVersion("5.0"): + return "Your version of gcc is older than 5.0 and will break builds. Please install a newer version of gcc (you could use the project's buildtools-extended-tarball or use scripts/install-buildtools).\n" + return None + # Tar version 1.24 and onwards handle overwriting symlinks correctly # but earlier versions do not; this needs to work properly for sstate # Version 1.28 is needed so opkg-build works correctly when reproducibile builds are enabled
@@ -533,7 +552,7 @@ def check_tar_version(sanity_data): return "Unable to execute tar --version, exit code %d\n%s\n" % (e.returncode, e.output) version = result.split()[3] if LooseVersion(version) < LooseVersion("1.28"): - return "Your version of tar is older than 1.28 and does not have the support needed to enable reproducible builds. Please install a newer version of tar (you could use the projects buildtools-tarball from our last release or use scripts/install-buildtools).\n" + return "Your version of tar is older than 1.28 and does not have the support needed to enable reproducible builds. Please install a newer version of tar (you could use the project's buildtools-tarball from our last release or use scripts/install-buildtools).\n" return None # We use git parameters and functionality only found in 1.7.8 or later
@@ -632,6 +651,7 @@ def check_sanity_version_change(status, d): except ImportError as e: status.addresult('Your Python 3 is not a full install. Please install the module %s (see the Getting Started guide for further information).\n' % e.name) + status.addresult(check_gcc_version(d)) status.addresult(check_make_version(d)) status.addresult(check_patch_version(d)) status.addresult(check_tar_version(d))
Signature · 1 line
2.24.0
PATCH v2 6/6 install-buildtools: bump default to yocto-3.1_M3, fixes
In reply to Tim OrlingAdd ability to check md5sum (yocto-3.1_M2 and before) or sha256 (yocto-3.1_M3 and beyond).
Make regex for path in checksum file optional, since for yocto-3.1_M3 the format is <checksum> <filename>, but prior releases was <checksum> <path><filename>
--- scripts/install-buildtools | 51 ++++++++++++++++++++++---------------- 1 file changed, 30 insertions(+), 21 deletions(-)
scripts/install-buildtools +30 -21
@@ -11,14 +11,14 @@ # Example usage (extended buildtools from milestone): # (1) using --url and --filename # $ install-buildtools \ -# --url http://downloads.yoctoproject.org/releases/yocto/milestones/yocto-3.1_M2/buildtools \ -# --filename x86_64-buildtools-extended-nativesdk-standalone-3.0+snapshot-20200122.sh +# --url http://downloads.yoctoproject.org/releases/yocto/milestones/yocto-3.1_M3/buildtools \ +# --filename x86_64-buildtools-extended-nativesdk-standalone-3.0+snapshot-20200315.sh # (2) using --base-url, --release, --installer-version and --build-date # $ install-buildtools \ # --base-url http://downloads.yoctoproject.org/releases/yocto \ -# --release yocto-3.1_M2 \ +# --release yocto-3.1_M3 \ # --installer-version 3.0+snapshot -# --build-date 202000122 +# --build-date 202000315 # # Example usage (standard buildtools from release): # (3) using --url and --filename
@@ -61,9 +61,9 @@ logger = scriptutils.logger_create(PROGNAME, stream=sys.stdout) DEFAULT_INSTALL_DIR: str = os.path.join(os.path.split(scripts_path)[0],'buildtools') DEFAULT_BASE_URL: str = 'http://downloads.yoctoproject.org/releases/yocto' -DEFAULT_RELEASE: str = 'yocto-3.1_M2' +DEFAULT_RELEASE: str = 'yocto-3.1_M3' DEFAULT_INSTALLER_VERSION: str = '3.0+snapshot' -DEFAULT_BUILDDATE: str = "20200122" +DEFAULT_BUILDDATE: str = "20200315" def main():
@@ -189,31 +189,40 @@ def main(): if args.check: import bb logger.info("Fetching buildtools installer checksum") - check_url = "{}.md5sum".format(buildtools_url) - checksum_filename = "%s.md5sum" % filename - tmpbuildtools_checksum = os.path.join(tmpsdk_dir, checksum_filename) - ret = subprocess.call("wget -q -O %s %s" % - (tmpbuildtools_checksum, check_url), shell=True) - if ret != 0: - logger.error("Could not download file from %s" % check_url) - return ret - regex = re.compile(r"^(?P<md5sum>[0-9a-f]+)\s\s(?P<path>.*/)(?P<filename>.*)$") + checksum_type = "" + for checksum_type in ["md5sum", "sha256"]: + check_url = "{}.{}".format(buildtools_url, checksum_type) + checksum_filename = "{}.{}".format(filename, checksum_type) + tmpbuildtools_checksum = os.path.join(tmpsdk_dir, checksum_filename) + ret = subprocess.call("wget -q -O %s %s" % + (tmpbuildtools_checksum, check_url), shell=True) + if ret == 0: + break + else: + if ret != 0: + logger.error("Could not download file from %s" % check_url) + return ret + regex = re.compile(r"^(?P<checksum>[0-9a-f]+)\s\s(?P<path>.*/)?(?P<filename>.*)$") with open(tmpbuildtools_checksum, 'rb') as f: original = f.read() m = re.search(regex, original.decode("utf-8")) - logger.debug("md5sum: %s" % m.group('md5sum')) + logger.debug("checksum regex match: %s" % m) + logger.debug("checksum: %s" % m.group('checksum')) logger.debug("path: %s" % m.group('path')) logger.debug("filename: %s" % m.group('filename')) if filename != m.group('filename'): logger.error("Filename does not match name in checksum") return 1 - md5sum = m.group('md5sum') - md5value = bb.utils.md5_file(tmpbuildtools) - if md5sum == md5value: - logger.info("Checksum success") + checksum = m.group('checksum') + if checksum_type == "md5sum": + checksum_value = bb.utils.md5_file(tmpbuildtools) + else: + checksum_value = bb.utils.sha256_file(tmpbuildtools) + if checksum == checksum_value: + logger.info("Checksum success") else: logger.error("Checksum %s expected. Actual checksum is %s." % - (md5sum, md5value)) + (checksum, checksum_value)) # Make installer executable logger.info("Making installer executable")
Signature · 1 line
2.24.0