Skip to contentSkip to conversation

sstate-cache-management.sh fixes [YOCTO #2897]

PATCH Applied Martin Jansa 4 participants 12 messages Sep 2, 2012 – Sep 4, 2012

lore
Share & tools
lore
#0 lore raw

cover sstate-cache-management.sh fixes [YOCTO #2897]

Fixes some issues reported in YOCTO #2897

The following changes since commit c2109b765b24a7ffe4781257ad3fe4641a3b2a49:

sstate: Ensure master.list exists if it doesn't already (2012-08-30 22:44:52 -0700)

are available in the git repository at:

git://git.openembedded.org/openembedded-core-contrib jansa/pull
http://cgit.openembedded.org/cgit.cgi/openembedded-core-contrib/log/?h=jansa/pull

Martin Jansa (4):
sstate-cache-management.sh: fix regexp in checksum grep sstate-cache-management.sh: fix regexp in AVAILTUNES grep sstate-cache-management.sh: look in meta* above oe-core dir when
looking for available tunes
sstate-cache-management.sh: don't hardcode available machines only to
qemu*

 scripts/sstate-cache-management.sh | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)
Signature · 1 line
1.7.12
#1 lore raw

PATCH 1/4 sstate-cache-management.sh: fix regexp in checksum grep

* using checksum without recipe/task name doesn't seem like good idea IMHO

---
 scripts/sstate-cache-management.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
scripts/sstate-cache-management.sh +1 -1
index a1f7c89..2b3100c 100755
@@ -292,7 +292,7 @@ rm_by_stamps (){
 
   echo -n "Figuring out the files which will be removed ... "
   for i in $all_sums; do
-      grep ".*-${i}_*" $cache_list >>$keep_list
+      grep ".*-${i}_.*" $cache_list >>$keep_list
   done
   echo "Done"
 
Signature · 1 line
1.7.12
#2 lore raw

PATCH 2/4 sstate-cache-management.sh: fix regexp in AVAILTUNES grep

In reply to Martin Jansa

* it was also picking line
openembedded-core/meta/conf/machine/include/README:AVAILTUNES - This is a list of all of the tuning definitions currently filling all_archs with wrong entries

---
 scripts/sstate-cache-management.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
scripts/sstate-cache-management.sh +1 -1
index 2b3100c..0f774a3 100755
@@ -167,7 +167,7 @@ remove_duplicated () {
   topdir=$(dirname $(dirname $(readlink -e $0)))
   tunedirs="`find $topdir/meta* $layers -path '*/meta*/conf/machine/include'`"
   [ -n "$tunedirs" ] || echo_error "Can't find the tune directory"
-  all_archs=`grep -r -h "^AVAILTUNES " $tunedirs | sed -e 's/.*=//' -e 's/\"//g'`
+  all_archs=`grep -r -h "^AVAILTUNES .*=" $tunedirs | sed -e 's/.*=//' -e 's/\"//g'`
   # Add the qemu and native archs
   # Use the "_" to substitute "-", e.g., x86-64 to x86_64
   # Sort to remove the duplicated ones
Signature · 1 line
1.7.12
#3 lore raw

PATCH 3/4 sstate-cache-management.sh: look in meta* above oe-core dir when looking for available tunes

In reply to Martin Jansa

* many layouts have other layers on the same level as oe-core checkout

---
 scripts/sstate-cache-management.sh | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)
scripts/sstate-cache-management.sh +4 -2
index 0f774a3..e569487 100755
@@ -153,6 +153,7 @@ gen_rmlist (){
 remove_duplicated () {
 
   local topdir
+  local oe_core_dir
   local tunedirs
   local all_archs
   local ava_archs
@@ -164,8 +165,9 @@ remove_duplicated () {
 
   # Find out the archs in all the layers
   echo -n "Figuring out the archs in the layers ... "
-  topdir=$(dirname $(dirname $(readlink -e $0)))
-  tunedirs="`find $topdir/meta* $layers -path '*/meta*/conf/machine/include'`"
+  oe_core_dir=$(dirname $(dirname $(readlink -e $0)))
+  topdir=$(dirname $oe_core_dir)
+  tunedirs="`find $topdir/meta* ${oe_core_dir}/meta* $layers -path '*/meta*/conf/machine/include'`"
   [ -n "$tunedirs" ] || echo_error "Can't find the tune directory"
   all_archs=`grep -r -h "^AVAILTUNES .*=" $tunedirs | sed -e 's/.*=//' -e 's/\"//g'`
   # Add the qemu and native archs
Signature · 1 line
1.7.12
#7 lore raw

PATCH 4/4 sstate-cache-management.sh: don't hardcode available machines only to qemu*

In reply to Martin Jansa

* find them dynamically in layers like AVAILTUNES

---
 scripts/sstate-cache-management.sh | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)
scripts/sstate-cache-management.sh +3 -1
index e569487..491df5f 100755
@@ -156,6 +156,7 @@ remove_duplicated () {
   local oe_core_dir
   local tunedirs
   local all_archs
+  local all_machines
   local ava_archs
   local arch
   local file_names
@@ -169,11 +170,12 @@ remove_duplicated () {
   topdir=$(dirname $oe_core_dir)
   tunedirs="`find $topdir/meta* ${oe_core_dir}/meta* $layers -path '*/meta*/conf/machine/include'`"
   [ -n "$tunedirs" ] || echo_error "Can't find the tune directory"
+  all_machines="`find $topdir/meta* ${oe_core_dir}/meta* $layers -path '*/meta*/conf/machine/*' -name '*.conf' | sed -e 's/.*\///' -e 's/.conf$//'`"
   all_archs=`grep -r -h "^AVAILTUNES .*=" $tunedirs | sed -e 's/.*=//' -e 's/\"//g'`
   # Add the qemu and native archs
   # Use the "_" to substitute "-", e.g., x86-64 to x86_64
   # Sort to remove the duplicated ones
-  all_archs=$(echo $all_archs qemuarm qemux86 qemumips qemuppc qemux86_64 $(uname -m) \
+  all_archs=$(echo $all_archs $all_machines $(uname -m) \
           | sed -e 's/-/_/g' -e 's/ /\n/g' | sort -u)
   echo "Done"
 
Signature · 1 line
1.7.12
#4 lore raw

Re: [PATCH 3/4] sstate-cache-management.sh: look in meta* above oe-core dir when looking for available tunes

In reply to Martin Jansa
Quoted 1 line · On Sunday 02 September 2012 13:13:20 Martin Jansa wrote:

* many layouts have other layers on the same level as oe-core checkout

This is beginning to indicate that this script really needs to actually know something about your configuration rather than trying to guess it.

Cheers,
Paul

Signature · 2 lines
Paul Eggleton
Intel Open Source Technology Centre
#5 lore raw

Re: [PATCH 3/4] sstate-cache-management.sh: look in meta* above oe-core dir when looking for available tunes

Quoted 5 lines · Paul Eggleton
Quoted 1 line · On Sunday 02 September 2012 13:13:20 Martin Jansa wrote:

* many layouts have other layers on the same level as oe-core checkout

This is beginning to indicate that this script really needs to actually know something about your configuration rather than trying to guess it.

You can pass more layers to script with:

--extra-layer=<layer1>,<layer2>...<layern>
Specify the layer which will be used for searching the archs, it will search the meta and meta-* layers in the top dir by default, and will search meta, meta-*, <layer1>, <layer2>, ...<layern> when specified. Use "," as the separator.

But nobody says what is "top dir", in my bblayers.conf it's the directory with oe-core and meta-* checkouts and I think this is common layout if you're not using some combo tool to merge all layers to oe-core checkout.

So this change only makes default search path a bit more usefull and then it filteres available arch anyway.

Cheers,

Signature · 1 line
Martin 'JaMa' Jansa     jabber: [email protected]
#6 lore raw

Re: [PATCH 3/4] sstate-cache-management.sh: look in meta* above oe-core dir when looking for available tunes

Quoted 14 lines · On Monday 03 September 2012 09:23:17 Martin Jansa wrote:
Quoted 5 lines · Paul Eggleton
Quoted 1 line · On Sunday 02 September 2012 13:13:20 Martin Jansa wrote:

* many layouts have other layers on the same level as oe-core checkout

This is beginning to indicate that this script really needs to actually know something about your configuration rather than trying to guess it.

You can pass more layers to script with:

--extra-layer=<layer1>,<layer2>...<layern>
Specify the layer which will be used for searching the archs, it will search the meta and meta-* layers in the top dir by default, and will search meta, meta-*, <layer1>, <layer2>, ...<layern> when specified. Use "," as the separator.

Quoted 7 lines

But nobody says what is "top dir", in my bblayers.conf it's the directory with oe-core and meta-* checkouts and I think this is common layout if you're not using some combo tool to merge all layers to oe-core checkout.

So this change only makes default search path a bit more usefull and then it filteres available arch anyway.

I don't dispute that your change is an improvement, I just think it would be a good future improvement to pick up the configuration from bitbake rather than working in the way it currently does.

Cheers,
Paul

Signature · 2 lines
Paul Eggleton
Intel Open Source Technology Centre
#8 lore raw

PATCH 5/9 sstate-cache-management.sh: add option to enable debug output

In reply to Martin Jansa
---
 scripts/sstate-cache-management.sh | 15 +++++++++++++++
 1 file changed, 15 insertions(+)
scripts/sstate-cache-management.sh +15 -0
index 491df5f..0cfff77 100755
@@ -22,6 +22,7 @@ confirm=
 fsym=
 total_deleted=0
 verbose=
+debug=0
 
 usage () {
   cat << EOF
@@ -69,6 +70,9 @@ Options:
   -v, --verbose
         explain what is being done
 
+  -d, --debug
+        show debug info, repeat for more debug info
+
 EOF
 }
 
@@ -215,13 +219,16 @@ remove_duplicated () {
           for arch in $ava_archs; do
               grep -h "/$fn-$arch-" $list_suffix >>$fn_tmp
           done
+          [ $debug -gt 1 ] && echo "Available files for $fn with suffix $suffix:" && cat $fn_tmp
           # Use the modification time
           to_del=$(ls -t $(cat $fn_tmp) | sed -n '1!p')
+          [ $debug -gt 2 ] && echo "Considering to delete: $to_del"
           # The sstate file which is downloaded from the SSTATE_MIRROR is
           # put in SSTATE_DIR, and there is a symlink in SSTATE_DIR/??/ to
           # it, so filter it out from the remove list if it should not be
           # removed.
           to_keep=$(ls -t $(cat $fn_tmp) | sed -n '1p')
+          [ $debug -gt 2 ] && echo "Considering to keep: $to_keep"
           for k in $to_keep; do
               if [ -L "$k" ]; then
                   # The symlink's destination
@@ -235,9 +242,11 @@ remove_duplicated () {
               fi
           done
           rm -f $fn_tmp
+          [ $debug -gt 2 ] && echo "Decided to delete: $to_del"
           gen_rmlist $rm_list "$to_del"
       done
       [ ! -s "$rm_list" ] || deleted=`cat $rm_list | wc -l`
+      [ -s "$rm_list" -a $debug -gt 0 ] && cat $rm_list
       echo "($deleted files will be removed)"
       let total_deleted=$total_deleted+$deleted
   done
@@ -306,6 +315,7 @@ rm_by_stamps (){
       gen_rmlist $rm_list "$to_del"
       let total_deleted=(`cat $rm_list | wc -w`)
       if [ $total_deleted -gt 0 ]; then
+          [ $debug -gt 0 ] && cat $rm_list
           read_confirm
           if [ "$confirm" = "y" -o "$confirm" = "Y" ]; then
               echo "Removing sstate cache files ... ($total_deleted files)"
@@ -374,6 +384,11 @@ while [ -n "$1" ]; do
       verbose="-v"
       shift
         ;;
+    --debug)
+      debug=`expr $debug + 1`
+      echo "Debug level $debug"
+      shift
+        ;;
     --help|-h)
       usage
       exit 0
Signature · 1 line
1.7.12
#9 lore raw

PATCH 6/9 sstate-cache-management.sh: fix remove_duplicated when multiple archs were built

* The problem is that you're looking for duplicates across all available archs, so armv4t and armv7a populate_sysroot are considered duplicate and only last one is kept, but to rebuild from sstate-cache you need both!

---
 scripts/sstate-cache-management.sh | 52 ++++++++++++++++++++------------------
 1 file changed, 27 insertions(+), 25 deletions(-)
scripts/sstate-cache-management.sh +27 -25
index 0cfff77..f519555 100755
@@ -217,33 +217,35 @@ remove_duplicated () {
       for fn in $file_names; do
           [ -z "$verbose" ] || echo "Analyzing $fn-xxx_$suffix.tgz"
           for arch in $ava_archs; do
-              grep -h "/$fn-$arch-" $list_suffix >>$fn_tmp
-          done
-          [ $debug -gt 1 ] && echo "Available files for $fn with suffix $suffix:" && cat $fn_tmp
-          # Use the modification time
-          to_del=$(ls -t $(cat $fn_tmp) | sed -n '1!p')
-          [ $debug -gt 2 ] && echo "Considering to delete: $to_del"
-          # The sstate file which is downloaded from the SSTATE_MIRROR is
-          # put in SSTATE_DIR, and there is a symlink in SSTATE_DIR/??/ to
-          # it, so filter it out from the remove list if it should not be
-          # removed.
-          to_keep=$(ls -t $(cat $fn_tmp) | sed -n '1p')
-          [ $debug -gt 2 ] && echo "Considering to keep: $to_keep"
-          for k in $to_keep; do
-              if [ -L "$k" ]; then
-                  # The symlink's destination
-                  k_dest="`readlink -e $k`"
-                  # Maybe it is the one in cache_dir
-                  k_maybe="$cache_dir/${k##/*/}"
-                  # Remove it from the remove list if they are the same.
-                  if [ "$k_dest" = "$k_maybe" ]; then
-                      to_del="`echo $to_del | sed 's#'\"$k_maybe\"'##g'`"
-                  fi
+              grep -h "/$fn-$arch-" $list_suffix >$fn_tmp
+              if [ -s $fn_tmp ] ; then
+                  [ $debug -gt 1 ] && echo "Available files for $fn-$arch- with suffix $suffix:" && cat $fn_tmp
+                  # Use the modification time
+                  to_del=$(ls -t $(cat $fn_tmp) | sed -n '1!p')
+                  [ $debug -gt 2 ] && echo "Considering to delete: $to_del"
+                  # The sstate file which is downloaded from the SSTATE_MIRROR is
+                  # put in SSTATE_DIR, and there is a symlink in SSTATE_DIR/??/ to
+                  # it, so filter it out from the remove list if it should not be
+                  # removed.
+                  to_keep=$(ls -t $(cat $fn_tmp) | sed -n '1p')
+                  [ $debug -gt 2 ] && echo "Considering to keep: $to_keep"
+                  for k in $to_keep; do
+                      if [ -L "$k" ]; then
+                          # The symlink's destination
+                          k_dest="`readlink -e $k`"
+                          # Maybe it is the one in cache_dir
+                          k_maybe="$cache_dir/${k##/*/}"
+                          # Remove it from the remove list if they are the same.
+                          if [ "$k_dest" = "$k_maybe" ]; then
+                              to_del="`echo $to_del | sed 's#'\"$k_maybe\"'##g'`"
+                          fi
+                      fi
+                  done
+                  rm -f $fn_tmp
+                  [ $debug -gt 2 ] && echo "Decided to delete: $to_del"
+                  gen_rmlist $rm_list "$to_del"
               fi
           done
-          rm -f $fn_tmp
-          [ $debug -gt 2 ] && echo "Decided to delete: $to_del"
-          gen_rmlist $rm_list "$to_del"
       done
       [ ! -s "$rm_list" ] || deleted=`cat $rm_list | wc -l`
       [ -s "$rm_list" -a $debug -gt 0 ] && cat $rm_list
Signature · 1 line
1.7.12
#11 lore raw
In reply to Martin Jansa
Quoted 11 lines · Martin Jansa

Fixes some issues reported in YOCTO #2897

The following changes since commit c2109b765b24a7ffe4781257ad3fe4641a3b2a49:

sstate: Ensure master.list exists if it doesn't already (2012-08-30 22:44:52 -0700)

are available in the git repository at:

git://git.openembedded.org/openembedded-core-contrib jansa/pull
http://cgit.openembedded.org/cgit.cgi/openembedded-core-contrib/log/?h=jansa/pull

Quoted 7 lines

Martin Jansa (4):
sstate-cache-management.sh: fix regexp in checksum grep sstate-cache-management.sh: fix regexp in AVAILTUNES grep sstate-cache-management.sh: look in meta* above oe-core dir when
looking for available tunes
sstate-cache-management.sh: don't hardcode available machines only to
qemu*

Merged to master, thanks (along with 5/4 and 6/4 :).

I do agree with Paul, we need to find a better way to do some of this but that is something for the future...

Cheers,

Richard

Applied
#10 lore raw

Re: [PATCH 5/9] sstate-cache-management.sh: add option to enable debug output

In reply to Martin Jansa
Quoted 67 lines · Martin Jansa
---
  scripts/sstate-cache-management.sh | 15 +++++++++++++++
  1 file changed, 15 insertions(+)
scripts/sstate-cache-management.sh +10 -0
index 491df5f..0cfff77 100755
@@ -22,6 +22,7 @@ confirm=
  fsym=
  total_deleted=0
  verbose=
+debug=0
 
  usage () {
    cat << EOF
@@ -69,6 +70,9 @@ Options:
    -v, --verbose
          explain what is being done
 
+  -d, --debug
+        show debug info, repeat for more debug info
+
  EOF
  }
 
@@ -215,13 +219,16 @@ remove_duplicated () {
            for arch in $ava_archs; do
                grep -h "/$fn-$arch-" $list_suffix >>$fn_tmp
            done
+          [ $debug -gt 1 ] && echo "Available files for $fn with suffix $suffix:" && cat $fn_tmp
            # Use the modification time
            to_del=$(ls -t $(cat $fn_tmp) | sed -n '1!p')
+          [ $debug -gt 2 ] && echo "Considering to delete: $to_del"
            # The sstate file which is downloaded from the SSTATE_MIRROR is
            # put in SSTATE_DIR, and there is a symlink in SSTATE_DIR/??/ to
            # it, so filter it out from the remove list if it should not be
            # removed.
            to_keep=$(ls -t $(cat $fn_tmp) | sed -n '1p')
+          [ $debug -gt 2 ] && echo "Considering to keep: $to_keep"
            for k in $to_keep; do
                if [ -L "$k" ]; then
                    # The symlink's destination
@@ -235,9 +242,11 @@ remove_duplicated () {
                fi
            done
            rm -f $fn_tmp
+          [ $debug -gt 2 ] && echo "Decided to delete: $to_del"
            gen_rmlist $rm_list "$to_del"
        done
        [ ! -s "$rm_list" ] || deleted=`cat $rm_list | wc -l`
+      [ -s "$rm_list" -a $debug -gt 0 ] && cat $rm_list
        echo "($deleted files will be removed)"
        let total_deleted=$total_deleted+$deleted
    done
@@ -306,6 +315,7 @@ rm_by_stamps (){
        gen_rmlist $rm_list "$to_del"
        let total_deleted=(`cat $rm_list | wc -w`)
        if [ $total_deleted -gt 0 ]; then
+          [ $debug -gt 0 ] && cat $rm_list
            read_confirm
            if [ "$confirm" = "y" -o "$confirm" = "Y" ]; then
                echo "Removing sstate cache files ... ($total_deleted files)"
@@ -374,6 +384,11 @@ while [ -n "$1" ]; do
        verbose="-v"

scripts/sstate-cache-management.sh while [ -n "$1" ]; do · line 394

Quoted 10 lines
scripts/sstate-cache-management.sh +5 -0
        shift
          ;;
+    --debug)
+      debug=`expr $debug + 1`
+      echo "Debug level $debug"
+      shift
+        ;;
      --help|-h)
        usage
        exit 0

Merged into OE-Core

Thanks
Sau!