Skip to contentSkip to conversation

sstate-diff-machines.sh: Return non-zero return code when there was some failure detected

PATCH New Martin Jansa 1 participant 2 messages Jun 25, 2014

lore
Share & tools
lore
#0 lore raw

PATCH 1/2 sstate-diff-machines.sh: Return non-zero return code when there was some failure detected

* add --analyze option, which is useful when using from jenkins job where you don't want to read output just to dectect how bad it was
* I was always using something like this inside jenkins job, but better to share it in original script

---
 scripts/sstate-diff-machines.sh | 85 ++++++++++++++++++++++++++++++++++++-----
 1 file changed, 75 insertions(+), 10 deletions(-)
scripts/sstate-diff-machines.sh +75 -10
index 860be73..056aa0a 100755
@@ -1,7 +1,8 @@
-#!/bin/sh
+#!/bin/bash
 
-# Used to compare sstate checksums between MACHINES
-# Execute script and compare generated list.M files
+# Used to compare sstate checksums between MACHINES.
+# Execute script and compare generated list.M files.
+# Using bash to have PIPESTATUS variable.
 
 # It's also usefull to keep older sstate checksums
 # to be able to find out why something is rebuilding
@@ -27,6 +28,7 @@ machines=
 targets=
 default_machines="qemuarm qemux86 qemux86-64"
 default_targets="core-image-base"
+analyze="N"
 
 usage () {
   cat << EOF
@@ -48,6 +50,12 @@ Options:
   --targets=<targets>
         List of targets separated by space, will use the environment variable TARGETS if it is not specified.
         Default value is "core-image-base".
+
+  --analyze
+        Show the differences between MACHINEs. It assumes:
+        * First 2 MACHINEs in --machines parameter have the same TUNE_PKGARCH
+        * Third optional MACHINE has different TUNE_PKGARCH - only native and allarch recipes are compared).
+        * Next MACHINEs are ignored
 EOF
 }
 
@@ -72,6 +80,10 @@ while [ -n "$1" ]; do
       targets=`echo $1 | sed -e 's#^--targets="*\([^"]*\)"*#\1#'`
       shift
         ;;
+    --analyze)
+      analyze="Y"
+      shift
+        ;;
     --help|-h)
       usage
       exit 0
@@ -94,14 +106,67 @@ done
 [ -n "$targets" ] || targets=$default_targets
 
 OUTPUT=${tmpdir}/sstate-diff/`date "+%s"`
+declare -i RESULT=0
 
 for M in ${machines}; do
-  find ${tmpdir}/stamps/ -name \*sigdata\* | xargs rm -f
+  [ -d ${tmpdir}/stamps/ ] && find ${tmpdir}/stamps/ -name \*sigdata\* | xargs rm -f
   mkdir -p ${OUTPUT}/${M}
-  export MACHINE=${M}; bitbake -S none ${targets} | tee -a ${OUTPUT}/${M}/log;
-  cp -ra ${tmpdir}/stamps/* ${OUTPUT}/${M}
-  find ${OUTPUT}/${M} -name \*sigdata\* | sed "s#${OUTPUT}/${M}/##g" | sort > ${OUTPUT}/${M}/list
-  M_UNDERSCORE=`echo ${M} | sed 's/-/_/g'`
-  sed "s/${M_UNDERSCORE}/MACHINE/g; s/${M}/MACHINE/g" ${OUTPUT}/${M}/list | sort > ${OUTPUT}/${M}/list.M
-  find ${tmpdir}/stamps/ -name \*sigdata\* | xargs rm -f
+  export MACHINE=${M}
+  bitbake -S none ${targets} 2>&1 | tee -a ${OUTPUT}/${M}/log;
+  RESULT+=${PIPESTATUS[0]}
+  if ls ${tmpdir}/stamps/* >/dev/null 2>/dev/null ; then
+    cp -ra ${tmpdir}/stamps/* ${OUTPUT}/${M}
+    find ${OUTPUT}/${M} -name \*sigdata\* | sed "s#${OUTPUT}/${M}/##g" | sort > ${OUTPUT}/${M}/list
+    M_UNDERSCORE=`echo ${M} | sed 's/-/_/g'`
+    sed "s/${M_UNDERSCORE}/MACHINE/g; s/${M}/MACHINE/g" ${OUTPUT}/${M}/list | sort > ${OUTPUT}/${M}/list.M
+    find ${tmpdir}/stamps/ -name \*sigdata\* | xargs rm -f
+  else
+    printf "ERROR: no sigdata files were generated for MACHINE $M in ${tmpdir}/stamps\n";
+  fi
 done
+
+function compareSignatures() {
+  MACHINE1=$1
+  MACHINE2=$2
+  PATTERN="$3"
+  PRE_PATTERN=""
+  [ -n "${PATTERN}" ] || PRE_PATTERN="-v"
+  [ -n "${PATTERN}" ] || PATTERN="MACHINE"
+  for TASK in do_configure.sigdata do_populate_sysroot.sigdata do_package_write_ipk.sigdata; do
+    printf "\n\n === Comparing signatures for task ${TASK} between ${MACHINE1} and ${MACHINE2} ===\n" | tee -a ${OUTPUT}/signatures.${MACHINE2}.${TASK}.log
+    diff ${OUTPUT}/${MACHINE1}/list.M ${OUTPUT}/${MACHINE2}/list.M | grep ${PRE_PATTERN} "${PATTERN}" | grep ${TASK} > ${OUTPUT}/signatures.${MACHINE2}.${TASK}
+    for i in `cat ${OUTPUT}/signatures.${MACHINE2}.${TASK} | sed 's#[^/]*/\([^/]*\)/.*#\1#g' | sort -u | xargs`; do
+      [ -e ${OUTPUT}/${MACHINE1}/*/$i/*${TASK}* ] || echo "INFO: ${i} task ${TASK} doesn't exist in ${MACHINE1}" >&2
+      [ -e ${OUTPUT}/${MACHINE1}/*/$i/*${TASK}* ] || continue
+      [ -e ${OUTPUT}/${MACHINE2}/*/$i/*${TASK}* ] || echo "INFO: ${i} task ${TASK} doesn't exist in ${MACHINE2}" >&2
+      [ -e ${OUTPUT}/${MACHINE2}/*/$i/*${TASK}* ] || continue
+      printf "ERROR: $i different signature for task ${TASK} between ${MACHINE1} and ${MACHINE2}\n";
+      bitbake-diffsigs ${OUTPUT}/${MACHINE1}/*/$i/*${TASK}* ${OUTPUT}/${MACHINE2}/*/$i/*${TASK}*;
+      echo "$i" >> ${OUTPUT}/failed-recipes.log
+      echo
+    done | tee -a ${OUTPUT}/signatures.${MACHINE2}.${TASK}.log
+    # don't create empty files
+    ERRORS=`grep "^ERROR.*" ${OUTPUT}/signatures.${MACHINE2}.${TASK}.log | wc -l`
+    if [ "${ERRORS}" != "0" ] ; then
+      echo "ERROR: ${ERRORS} errors found in ${OUTPUT}/signatures.${MACHINE2}.${TASK}.log"
+      RESULT+=${ERRORS}
+    fi
+  done
+}
+
+function compareMachines() {
+  [ "$#" -ge 2 ] && compareSignatures $1 $2
+  [ "$#" -ge 3 ] && compareSignatures $1 $3 "\(^< all\)\|\(^< x86_64-linux\)\|\(^< i586-linux\)"
+}
+
+if [ "${analyze}" = "Y" ] ; then
+  compareMachines ${machines}
+fi
+
+if [ "${RESULT}" != "0" -a -f ${OUTPUT}/failed-recipes.log ] ; then
+  cat ${OUTPUT}/failed-recipes.log | sort -u >${OUTPUT}/failed-recipes.log.u && mv ${OUTPUT}/failed-recipes.log.u ${OUTPUT}/failed-recipes.log
+  echo "ERROR: ${RESULT} issues were found in these recipes: `cat ${OUTPUT}/failed-recipes.log | xargs`"
+fi
+
+echo "INFO: Output written in: ${OUTPUT}"
+exit ${RESULT}
Signature · 1 line
2.0.0
#1 lore raw

PATCH 2/2 test-dependencies.sh: Return non-zero return code when there was some failure detected

* this is useful when using from jenkins job where you don't want to read output just to dectect how bad it was
* add .log suffix to all files, so they can be easily downloaded from http servers without default mimetype set to something useful
* add recipes failed in step 1 to steps 2 and 3 to generate standalone logs for them

---
 scripts/test-dependencies.sh | 52 +++++++++++++++++++++++++++++++++-----------
 1 file changed, 39 insertions(+), 13 deletions(-)
scripts/test-dependencies.sh +39 -13
index d3212c4..6ebfd3e 100755
@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
 
 # Author: Martin Jansa <[email protected]>
 #
@@ -6,7 +6,7 @@
 
 # Used to detect missing dependencies or automagically
 # enabled dependencies which aren't explicitly enabled
-# or disabled.
+# or disabled. Using bash to have PIPESTATUS variable.
 
 # It does 3 builds of <target>
 # 1st to populate sstate-cache directory and sysroot
@@ -131,6 +131,7 @@ done
 echo "$buildtype" | grep -v '^[1234c ]*$' && echo_error "Invalid buildtype \"$buildtype\", only some combination of 1, 2, 3, 4, c separated by space is allowed"
 
 OUTPUT_BASE=test-dependencies/`date "+%s"`
+declare -i RESULT=0
 
 build_all() {
   echo "===== 1st build to populate sstate-cache directory and sysroot ====="
@@ -138,6 +139,9 @@ build_all() {
   mkdir -p ${OUTPUT1}
   echo "Logs will be stored in ${OUTPUT1} directory"
   bitbake -k $targets 2>&1 | tee -a ${OUTPUT1}/complete.log
+  RESULT+=${PIPESTATUS[0]}
+  grep "ERROR: Task.*failed" ${OUTPUT1}/complete.log > ${OUTPUT1}/failed-tasks.log
+  cat ${OUTPUT1}/failed-tasks.log | sed 's@.*/@@g; s@_.*@@g; s@\..*@@g' | sort -u > ${OUTPUT1}/failed-recipes.log
 }
 
 build_every_recipe() {
@@ -162,12 +166,24 @@ build_every_recipe() {
     rm -rf $tmpdir/deploy $tmpdir/pkgdata $tmpdir/sstate-control $tmpdir/stamps $tmpdir/sysroots $tmpdir/work $tmpdir/work-shared 2>/dev/null
   fi
   i=1
-  count=`cat $recipes | wc -l`
-  for recipe in `cat $recipes`; do
+  count=`cat $recipes ${OUTPUT1}/failed-recipes.log | sort -u | wc -l`
+  for recipe in `cat $recipes ${OUTPUT1}/failed-recipes.log | sort -u`; do
     echo "Building recipe: ${recipe} ($i/$count)"
-    bitbake -c cleansstate ${recipe} > ${OUTPUTB}/log.${recipe} 2>&1;
-    bitbake ${recipe} >> ${OUTPUTB}/log.${recipe} 2>&1;
-    grep "ERROR: Task.*failed" ${OUTPUTB}/log.${recipe} && mv ${OUTPUTB}/log.${recipe} ${OUTPUTB}/failed/${recipe} || mv ${OUTPUTB}/log.${recipe} ${OUTPUTB}/ok/${recipe}
+    declare -i RECIPE_RESULT=0
+    bitbake -c cleansstate ${recipe} > ${OUTPUTB}/${recipe}.log 2>&1;
+    RECIPE_RESULT+=$?
+    bitbake ${recipe} >> ${OUTPUTB}/${recipe}.log 2>&1;
+    RECIPE_RESULT+=$?
+    if [ "${RECIPE_RESULT}" != "0" ] ; then
+      RESULT+=${RECIPE_RESULT}
+      mv ${OUTPUTB}/${recipe}.log ${OUTPUTB}/failed/
+      grep "ERROR: Task.*failed"  ${OUTPUTB}/failed/${recipe}.log | tee -a ${OUTPUTB}/failed-tasks.log
+      grep "ERROR: Task.*failed"  ${OUTPUTB}/failed/${recipe}.log | sed 's@.*/@@g; s@_.*@@g; s@\..*@@g' >> ${OUTPUTB}/failed-recipes.log
+      # and append also ${recipe} in case the failed task was from some dependency
+      echo ${recipe} >> ${OUTPUTB}/failed-recipes.log
+    else
+      mv ${OUTPUTB}/${recipe}.log ${OUTPUTB}/ok/
+    fi
     if [ "${TYPE}" != "2" ] ; then
       rm -rf $tmpdir/deploy $tmpdir/pkgdata $tmpdir/sstate-control $tmpdir/stamps $tmpdir/sysroots $tmpdir/work $tmpdir/work-shared 2>/dev/null
     fi
@@ -184,8 +200,6 @@ build_every_recipe() {
     dest=`echo $f | sed "s#$tmpdir/work/##g; s#/#_#g"`
     cp $f ${OUTPUTB}/do_package/$dest
   done
-  grep "ERROR: Task.*failed" ${OUTPUTB}/failed/* 2>/dev/null
-  ls -1 ${OUTPUTB}/failed/* >> ${OUTPUT_BASE}/failed.recipes 2>/dev/null
 }
 
 compare_deps() {
@@ -209,8 +223,10 @@ compare_deps() {
   find ${OUTPUT_MAX}/packages/ -name latest | sed "s#${OUTPUT_MAX}/##g" | while read pkg; do
     max_pkg=${OUTPUT_MAX}/${pkg}
     min_pkg=${OUTPUT_MIN}/${pkg}
+    recipe=`echo "${pkg}" | sed 's#/.*##g'`
     if [ ! -f "${min_pkg}" ] ; then
       echo "ERROR: ${min_pkg} doesn't exist" | tee -a ${OUTPUT_FILE}
+      echo ${recipe} >> ${OUTPUTC}/failed-recipes.log
       continue
     fi
     # strip version information in parenthesis
@@ -231,17 +247,18 @@ compare_deps() {
       if [ -n "${missing_deps}" ] ; then
         echo # to get rid of dots on last line
         echo "WARN: ${pkg} lost dependency on ${missing_deps}" | tee -a ${OUTPUT_FILE}
+        echo ${recipe} >> ${OUTPUTC}/failed-recipes.log
       fi
     fi
     i=`expr $i + 1`
   done
   echo # to get rid of dots on last line
   echo "Found differences: "
-  grep "^WARN: " ${OUTPUT_FILE} | tee ${OUTPUT_FILE}.warn
+  grep "^WARN: " ${OUTPUT_FILE} | tee ${OUTPUT_FILE}.warn.log
   echo "Found errors: "
-  grep "^ERROR: " ${OUTPUT_FILE} | tee ${OUTPUT_FILE}.error
-  # useful for reexecuting this script with only small subset of recipes with known issues
-  sed 's#.*[ /]packages/\([^/]*\)/\([^/]*\)/.*#\2#g' ${OUTPUT_FILE}.warn ${OUTPUT_FILE}.error | sort -u >> ${OUTPUT_BASE}/failed.recipes
+  grep "^ERROR: " ${OUTPUT_FILE} | tee ${OUTPUT_FILE}.error.log
+  RESULT+=`cat ${OUTPUT_FILE}.warn.log | wc -l`
+  RESULT+=`cat ${OUTPUT_FILE}.error.log | wc -l`
 }
 
 for TYPE in $buildtype; do
@@ -254,3 +271,12 @@ for TYPE in $buildtype; do
     *) echo_error "Invalid buildtype \"$TYPE\""
   esac
 done
+
+cat ${OUTPUT_BASE}/*/failed-recipes.log | sort -u > ${OUTPUT_BASE}/failed-recipes.log
+
+if [ "${RESULT}" != "0" ] ; then
+  echo "ERROR: ${RESULT} issues were found in these recipes: `cat ${OUTPUT_BASE}/failed-recipes.log | xargs`"
+fi
+
+echo "INFO: Output written in: ${OUTPUT_BASE}"
+exit ${RESULT}
Signature · 1 line
2.0.0