Skip to contentSkip to conversation

toolchain-shar-extract.sh: check required tool before extracting SDK

PATCH In review Ming Liu 2 participants 4 messages Oct 23, 2024 – Oct 27, 2024

lore
Share & tools
lore

Ming Liu

author
#0 lore raw

PATCH v2 1/2 toolchain-shar-extract.sh: check required tool before extracting SDK

To extract the SDK archive, the proper tools need to be present on system, check unzip for zip archive type, check xz for tar.xz archive type.

---
 meta/files/toolchain-shar-extract.sh | 8 ++++++++
 1 file changed, 8 insertions(+)
meta/files/toolchain-shar-extract.sh +8 -0
index 3b4647fca7..9b72499ccf 100644
@@ -245,6 +245,10 @@ fi
 
 printf "Extracting SDK..."
 if [ @SDK_ARCHIVE_TYPE@ = "zip" ]; then
+    if [ -z "$(which unzip)" ]; then
+        echo "Aborted, unzip is required to extract the SDK archive, please make sure it's installed on your system!"
+        exit 1
+    fi
     tail -n +$payload_offset "$0" > sdk.zip
     if $SUDO_EXEC unzip $EXTRA_TAR_OPTIONS sdk.zip -d $target_sdk_dir;then
         rm sdk.zip
@@ -252,6 +256,10 @@ if [ @SDK_ARCHIVE_TYPE@ = "zip" ]; then
         rm sdk.zip && exit 1
     fi
 else
+    if [ -z "$(which xz)" ]; then
+        echo "Aborted, xz is required to extract the SDK archive, please make sure it's installed on your system!"
+        exit 1
+    fi
     tail -n +$payload_offset "$0"| $SUDO_EXEC tar mxJ -C $target_sdk_dir --checkpoint=.2500 $EXTRA_TAR_OPTIONS || exit 1
 fi
 echo "done"
Signature · 1 line
2.43.0
author
#1 lore raw

PATCH v2 2/2 meta: support tar.zst SDK_ARCHIVE_TYPE

zst is much faster than the default xz, test result on my Precision 5680 machine:

Installing a tar.xz SDK takes 37 seconds while tar.zst only takes 17 seconds.

Let's introduce support for tar.zst.

Also add a sanity check for supported archive types.

---
 meta/classes-recipe/populate_sdk_base.bbclass | 12 ++++++++++--
 meta/files/toolchain-shar-extract.sh          |  6 ++++++
 2 files changed, 16 insertions(+), 2 deletions(-)
meta/classes-recipe/populate_sdk_base.bbclass +10 -2
index 16013d5872..031c65f05f 100644
@@ -84,6 +84,7 @@ SDK_XZ_OPTIONS ?= "${XZ_DEFAULTS} ${SDK_XZ_COMPRESSION_LEVEL}"
 SDK_ZIP_OPTIONS ?= "-y"
 SDK_7ZIP_OPTIONS ?= "-mx=9 -mm=BZip2"
 SDK_7ZIP_TYPE ?= "7z"
+SDK_ZSTD_COMPRESSION_LEVEL = "-17"
 
 # To support different sdk type according to SDK_ARCHIVE_TYPE, now support zip and tar.xz
 python () {
@@ -95,9 +96,16 @@ python () {
     elif d.getVar('SDK_ARCHIVE_TYPE') == '7zip':
        d.setVar('SDK_ARCHIVE_DEPENDS', 'p7zip-native')
        d.setVar('SDK_ARCHIVE_CMD', 'cd ${SDK_OUTPUT}/${SDKPATH}; 7za a -r ${SDK_7ZIP_OPTIONS} ${SDKDEPLOYDIR}/${TOOLCHAIN_OUTPUTNAME}.${SDK_7ZIP_TYPE} .')
-    else:
+    elif d.getVar('SDK_ARCHIVE_TYPE') == 'tar.zst':
+       d.setVar('SDK_ARCHIVE_DEPENDS', 'zstd-native')
+       d.setVar('SDK_ARCHIVE_CMD',
+         'cd ${SDK_OUTPUT}/${SDKPATH}; tar ${SDKTAROPTS} -cf - . | zstd -f -k -T0 -c ${SDK_ZSTD_COMPRESSION_LEVEL} > ${SDKDEPLOYDIR}/${TOOLCHAIN_OUTPUTNAME}.${SDK_ARCHIVE_TYPE}')
+    elif d.getVar('SDK_ARCHIVE_TYPE') == 'tar.xz':
        d.setVar('SDK_ARCHIVE_DEPENDS', 'xz-native')
-       d.setVar('SDK_ARCHIVE_CMD', 'cd ${SDK_OUTPUT}/${SDKPATH}; tar ${SDKTAROPTS} -cf - . | xz ${SDK_XZ_OPTIONS} > ${SDKDEPLOYDIR}/${TOOLCHAIN_OUTPUTNAME}.${SDK_ARCHIVE_TYPE}')
+       d.setVar('SDK_ARCHIVE_CMD',
+         'cd ${SDK_OUTPUT}/${SDKPATH}; tar ${SDKTAROPTS} -cf - . | xz ${SDK_XZ_OPTIONS} > ${SDKDEPLOYDIR}/${TOOLCHAIN_OUTPUTNAME}.${SDK_ARCHIVE_TYPE}')
+    else:
+        bb.fatal("Invalid SDK_ARCHIVE_TYPE: %s, the supported SDK archive types are: zip, 7z, tar.xz, tar.zst" % d.getVar('SDK_ARCHIVE_TYPE'))
 }
 
 SDK_RDEPENDS = "${TOOLCHAIN_TARGET_TASK} ${TOOLCHAIN_HOST_TASK}"
meta/files/toolchain-shar-extract.sh +6 -0
index 9b72499ccf..208afaa4ae 100644
@@ -255,6 +255,12 @@ if [ @SDK_ARCHIVE_TYPE@ = "zip" ]; then
     else
         rm sdk.zip && exit 1
     fi
+elif [ @SDK_ARCHIVE_TYPE@ = "tar.zst" ]; then
+    if [ -z "$(which zstd)" ]; then
+        echo "Aborted, zstd is required to extract the SDK archive, please make sure it's installed on your system!"
+        exit 1
+    fi
+    tail -n +$payload_offset "$0"| zstd -T0 -dc | $SUDO_EXEC tar mx -C $target_sdk_dir --checkpoint=.2500 $EXTRA_TAR_OPTIONS || exit 1
 else
     if [ -z "$(which xz)" ]; then
         echo "Aborted, xz is required to extract the SDK archive, please make sure it's installed on your system!"
Signature · 1 line
2.43.0
#2 lore raw
In reply to Ming Liu
Quoted 18 lines · Ming Liu via lists.openembedded.org

To extract the SDK archive, the proper tools need to be present on system, check unzip for zip archive type, check xz for tar.xz archive type.

---
meta/files/toolchain-shar-extract.sh | 8 ++++++++

1 file changed, 8 insertions(+)
meta/files/toolchain-shar-extract.sh +0 -0
index 3b4647fca7..9b72499ccf 100644
@@ -245,6 +245,10 @@ fi
 

printf "Extracting SDK..."
if [ @SDK_ARCHIVE_TYPE@ = "zip" ]; then

meta/files/toolchain-shar-extract.sh fi · line 251

Quoted 4 lines
meta/files/toolchain-shar-extract.sh +4 -0
+    if [ -z "$(which unzip)" ]; then
+        echo "Aborted, unzip is required to extract the SDK archive, please make sure it's installed on your system!"
+        exit 1
+    fi

Unfortunately ‘which’ is not POSIX or even universally installed on Linux systems.

https://hynek.me/til/which-not-posix/

Using command -v is the portable alternative.

Ross

author
#3 lore raw

Hi, Ross:

Thanks for the reply, will make a V3!

Ross Burton <[email protected]> 於 2024年10月24日 週四 下午12:59寫道:

Quoted 37 lines

Review

Quoted 11 lines · Ming Liu via lists.openembedded.org

To extract the SDK archive, the proper tools need to be present on system, check unzip for zip archive type, check xz for tar.xz archive type.

---
meta/files/toolchain-shar-extract.sh | 8 ++++++++

1 file changed, 8 insertions(+)
(quoted diff) +0 -0

b/meta/files/toolchain-shar-extract.sh

meta/files/toolchain-shar-extract.sh

Quoted 9 lines

index 3b4647fca7..9b72499ccf 100644

meta/files/toolchain-shar-extract.sh +0 -0
@@ -245,6 +245,10 @@ fi
 

printf "Extracting SDK..."
if [ @SDK_ARCHIVE_TYPE@ = "zip" ]; then

meta/files/toolchain-shar-extract.sh +2 -0
+    if [ -z "$(which unzip)" ]; then
+        echo "Aborted, unzip is required to extract the SDK archive,

please make sure it's installed on your system!"

meta/files/toolchain-shar-extract.sh

Quoted 2 lines
meta/files/toolchain-shar-extract.sh +2 -0
+        exit 1
+    fi

Unfortunately ‘which’ is not POSIX or even universally installed on Linux systems.

https://hynek.me/til/which-not-posix/

Using command -v is the portable alternative.

Ross