Skip to contentSkip to conversation

classes/sstate: Update output hash

PATCH Superseded Joshua Watt 3 participants 5 messages Jan 15, 2019 – Jan 21, 2019

lore
Share & tools
lore
#0 lore raw

PATCH 1/1 classes/sstate: Update output hash

Updates the output hash calculation for determining if tasks are equivalent. The new algorithm does the following based on feedback:
1) All files are printed in a single line tabular format
2) Prints the file type and mode in a user-friendly ls-like format
3) Includes the file owner and group (by name, not ID). These are only included if the task is run under pseudo, since that is the only time they can be consistently determined.
4) File size is included for regular files

---
 meta/classes/sstate.bbclass | 91 +++++++++++++++++++++++++++++++------
 1 file changed, 76 insertions(+), 15 deletions(-)
meta/classes/sstate.bbclass +76 -15
index 482ffa83f98..a103a759825 100644
@@ -784,6 +784,8 @@ python sstate_sign_package () {
 def OEOuthashBasic(path, sigfile, task, d):
     import hashlib
     import stat
+    import pwd
+    import grp
 
     def update_hash(s):
         s = s.encode('utf-8')
@@ -793,6 +795,7 @@ def OEOuthashBasic(path, sigfile, task, d):
 
     h = hashlib.sha256()
     prev_dir = os.getcwd()
+    include_owners = os.environ.get('PSEUDO_DISABLED') == '0'
 
     try:
         os.chdir(path)
@@ -807,34 +810,92 @@ def OEOuthashBasic(path, sigfile, task, d):
         update_hash("task=%s\n" % task)
 
         for root, dirs, files in os.walk('.', topdown=True):
-            # Sort directories and files to ensure consistent ordering
+            # Sort directories to ensure consistent ordering when recursing
             dirs.sort()
             files.sort()
 
-            for f in files:
-                path = os.path.join(root, f)
+            def process(path):
                 s = os.lstat(path)
 
-                # Hash file path
-                update_hash(path + '\n')
+                if stat.S_ISDIR(s.st_mode):
+                    update_hash('d')
+                elif stat.S_ISCHR(s.st_mode):
+                    update_hash('c')
+                elif stat.S_ISBLK(s.st_mode):
+                    update_hash('b')
+                elif stat.S_ISSOCK(s.st_mode):
+                    update_hash('s')
+                elif stat.S_ISLNK(s.st_mode):
+                    update_hash('l')
+                elif stat.S_ISFIFO(s.st_mode):
+                    update_hash('p')
+                else:
+                    update_hash('-')
+
+                def add_perm(mask, on, off='-'):
+                    if mask & s.st_mode:
+                        update_hash(on)
+                    else:
+                        update_hash(off)
+
+                add_perm(stat.S_IRUSR, 'r')
+                add_perm(stat.S_IWUSR, 'w')
+                if stat.S_ISUID & s.st_mode:
+                    add_perm(stat.S_IXUSR, 's', 'S')
+                else:
+                    add_perm(stat.S_IXUSR, 'x')
 
-                # Hash file mode
-                update_hash("\tmode=0x%x\n" % stat.S_IMODE(s.st_mode))
-                update_hash("\ttype=0x%x\n" % stat.S_IFMT(s.st_mode))
+                add_perm(stat.S_IRGRP, 'r')
+                add_perm(stat.S_IWGRP, 'w')
+                if stat.S_ISGID & s.st_mode:
+                    add_perm(stat.S_IXGRP, 's', 'S')
+                else:
+                    add_perm(stat.S_IXGRP, 'x')
 
-                if stat.S_ISBLK(s.st_mode) or stat.S_ISBLK(s.st_mode):
-                    # Hash device major and minor
-                    update_hash("\tdev=%d,%d\n" % (os.major(s.st_rdev), os.minor(s.st_rdev)))
-                elif stat.S_ISLNK(s.st_mode):
-                    # Hash symbolic link
-                    update_hash("\tsymlink=%s\n" % os.readlink(path))
+                add_perm(stat.S_IROTH, 'r')
+                add_perm(stat.S_IWOTH, 'w')
+                if stat.S_ISVTX & s.st_mode:
+                    update_hash('t')
+                else:
+                    add_perm(stat.S_IXOTH, 'x')
+
+                if include_owners:
+                    #update_hash(" %5d" % s.st_uid)
+                    #update_hash(" %5d" % s.st_gid)
+                    update_hash(" %10s" % pwd.getpwuid(s.st_uid).pw_name)
+                    update_hash(" %10s" % grp.getgrgid(s.st_gid).gr_name)
+
+                if stat.S_ISBLK(s.st_mode) or stat.S_ISCHR(s.st_mode):
+                    update_hash(" %9s" % ("%d.%d" % (os.major(s.st_rdev), os.minor(s.st_rdev))))
                 else:
+                    update_hash(" " * 10)
+
+                if stat.S_ISREG(s.st_mode):
+                    update_hash(" %10d" % s.st_size)
+                else:
+                    update_hash(" " * 11)
+
+                update_hash(" %s" % path)
+
+                if stat.S_ISLNK(s.st_mode):
+                    update_hash(" -> %s" % os.readlink(path))
+
+                if stat.S_ISREG(s.st_mode):
                     fh = hashlib.sha256()
                     # Hash file contents
                     with open(path, 'rb') as d:
                         for chunk in iter(lambda: d.read(4096), b""):
                             fh.update(chunk)
-                    update_hash("\tdigest=%s\n" % fh.hexdigest())
+                    update_hash(" %s" % fh.hexdigest())
+
+                update_hash("\n")
+
+            # Process this directory and all its child files
+            process(root)
+            for f in files:
+                if f == 'fixmepath':
+                    continue
+                process(os.path.join(root, f))
     finally:
         os.chdir(prev_dir)
 
Signature · 1 line
2.20.1
#1 lore raw
Quoted 134 lines · Joshua Watt

Updates the output hash calculation for determining if tasks are equivalent. The new algorithm does the following based on feedback:
1) All files are printed in a single line tabular format
2) Prints the file type and mode in a user-friendly ls-like format
3) Includes the file owner and group (by name, not ID). These are only included if the task is run under pseudo, since that is the only time they can be consistently determined.
4) File size is included for regular files

---
  meta/classes/sstate.bbclass | 91 +++++++++++++++++++++++++++++++------
  1 file changed, 76 insertions(+), 15 deletions(-)
meta/classes/sstate.bbclass +69 -15
index 482ffa83f98..a103a759825 100644
@@ -784,6 +784,8 @@ python sstate_sign_package () {
  def OEOuthashBasic(path, sigfile, task, d):
      import hashlib
      import stat
+    import pwd
+    import grp
 
      def update_hash(s):
          s = s.encode('utf-8')
@@ -793,6 +795,7 @@ def OEOuthashBasic(path, sigfile, task, d):
 
      h = hashlib.sha256()
      prev_dir = os.getcwd()
+    include_owners = os.environ.get('PSEUDO_DISABLED') == '0'
 
      try:
          os.chdir(path)
@@ -807,34 +810,92 @@ def OEOuthashBasic(path, sigfile, task, d):
          update_hash("task=%s\n" % task)
 
          for root, dirs, files in os.walk('.', topdown=True):
-            # Sort directories and files to ensure consistent ordering
+            # Sort directories to ensure consistent ordering when recursing
              dirs.sort()
              files.sort()
 
-            for f in files:
-                path = os.path.join(root, f)
+            def process(path):
                  s = os.lstat(path)
 
-                # Hash file path
-                update_hash(path + '\n')
+                if stat.S_ISDIR(s.st_mode):
+                    update_hash('d')
+                elif stat.S_ISCHR(s.st_mode):
+                    update_hash('c')
+                elif stat.S_ISBLK(s.st_mode):
+                    update_hash('b')
+                elif stat.S_ISSOCK(s.st_mode):
+                    update_hash('s')
+                elif stat.S_ISLNK(s.st_mode):
+                    update_hash('l')
+                elif stat.S_ISFIFO(s.st_mode):
+                    update_hash('p')
+                else:
+                    update_hash('-')
+
+                def add_perm(mask, on, off='-'):
+                    if mask & s.st_mode:
+                        update_hash(on)
+                    else:
+                        update_hash(off)
+
+                add_perm(stat.S_IRUSR, 'r')
+                add_perm(stat.S_IWUSR, 'w')
+                if stat.S_ISUID & s.st_mode:
+                    add_perm(stat.S_IXUSR, 's', 'S')
+                else:
+                    add_perm(stat.S_IXUSR, 'x')
 
-                # Hash file mode
-                update_hash("\tmode=0x%x\n" % stat.S_IMODE(s.st_mode))
-                update_hash("\ttype=0x%x\n" % stat.S_IFMT(s.st_mode))
+                add_perm(stat.S_IRGRP, 'r')
+                add_perm(stat.S_IWGRP, 'w')
+                if stat.S_ISGID & s.st_mode:
+                    add_perm(stat.S_IXGRP, 's', 'S')
+                else:
+                    add_perm(stat.S_IXGRP, 'x')
 
-                if stat.S_ISBLK(s.st_mode) or stat.S_ISBLK(s.st_mode):
-                    # Hash device major and minor
-                    update_hash("\tdev=%d,%d\n" % (os.major(s.st_rdev), os.minor(s.st_rdev)))
-                elif stat.S_ISLNK(s.st_mode):
-                    # Hash symbolic link
-                    update_hash("\tsymlink=%s\n" % os.readlink(path))
+                add_perm(stat.S_IROTH, 'r')
+                add_perm(stat.S_IWOTH, 'w')
+                if stat.S_ISVTX & s.st_mode:
+                    update_hash('t')
+                else:
+                    add_perm(stat.S_IXOTH, 'x')
+
+                if include_owners:
+                    #update_hash(" %5d" % s.st_uid)
+                    #update_hash(" %5d" % s.st_gid)
+                    update_hash(" %10s" % pwd.getpwuid(s.st_uid).pw_name)
+                    update_hash(" %10s" % grp.getgrgid(s.st_gid).gr_name)
+
+                if stat.S_ISBLK(s.st_mode) or stat.S_ISCHR(s.st_mode):
+                    update_hash(" %9s" % ("%d.%d" % (os.major(s.st_rdev), os.minor(s.st_rdev))))
                  else:
+                    update_hash(" " * 10)
+
+                if stat.S_ISREG(s.st_mode):
+                    update_hash(" %10d" % s.st_size)
+                else:
+                    update_hash(" " * 11)
+
+                update_hash(" %s" % path)
+
+                if stat.S_ISLNK(s.st_mode):
+                    update_hash(" -> %s" % os.readlink(path))
+
+                if stat.S_ISREG(s.st_mode):
                      fh = hashlib.sha256()
                      # Hash file contents
                      with open(path, 'rb') as d:
                          for chunk in iter(lambda: d.read(4096), b""):
                              fh.update(chunk)
-                    update_hash("\tdigest=%s\n" % fh.hexdigest())
+                    update_hash(" %s" % fh.hexdigest())
+
+                update_hash("\n")

meta/classes/sstate.bbclass def OEOuthashBasic(path, sigfile, task, d): · line 901

Quoted 10 lines
meta/classes/sstate.bbclass +7 -0
+
+            # Process this directory and all its child files
+            process(root)
+            for f in files:
+                if f == 'fixmepath':
+                    continue
+                process(os.path.join(root, f))
      finally:
          os.chdir(prev_dir)
 

Thanks for working on this Joshua.

It looks really nice, this is an example of the busybox depsig.do_package I get with this patch applied:

drwxrwxr-x       root       root                      .
drwxr-xr-x       root       root                      ./package
drwxr-xr-x       root       root                      ./package/bin
lrwxrwxrwx       root       root                      ./package/bin/busybox -> busybox.nosuid
-rwxr-xr-x       root       root               551388 ./package/bin/busybox.nosuid b50144c6a810bf92cbd442fd6f55794b6cdc8625a46f55c2e9d86ad22d75134a
-rwsr-xr-x       root       root                50860 ./package/bin/busybox.suid eb7af7e8f9e4a5bf6be7fb5ac16064ccd8f35e9890661134c5d73efbeb6e1d44
lrwxrwxrwx       root       root                      ./package/bin/sh -> busybox.nosuid

How about prepending the hashes first, printing just spaces or maybe doing '000...' for symlinks and directories ?

My 2 cents..
/Jacob

#2 lore raw
In reply to Joshua Watt
Quoted 95 lines · Joshua Watt

Updates the output hash calculation for determining if tasks are equivalent. The new algorithm does the following based on feedback:
1) All files are printed in a single line tabular format
2) Prints the file type and mode in a user-friendly ls-like format
3) Includes the file owner and group (by name, not ID). These are only included if the task is run under pseudo, since that is the only time they can be consistently determined.
4) File size is included for regular files

---
  meta/classes/sstate.bbclass | 91 +++++++++++++++++++++++++++++++------
  1 file changed, 76 insertions(+), 15 deletions(-)
meta/classes/sstate.bbclass +38 -14
index 482ffa83f98..a103a759825 100644
@@ -784,6 +784,8 @@ python sstate_sign_package () {
  def OEOuthashBasic(path, sigfile, task, d):
      import hashlib
      import stat
+    import pwd
+    import grp
 
      def update_hash(s):
          s = s.encode('utf-8')
@@ -793,6 +795,7 @@ def OEOuthashBasic(path, sigfile, task, d):
 
      h = hashlib.sha256()
      prev_dir = os.getcwd()
+    include_owners = os.environ.get('PSEUDO_DISABLED') == '0'
 
      try:
          os.chdir(path)
@@ -807,34 +810,92 @@ def OEOuthashBasic(path, sigfile, task, d):
          update_hash("task=%s\n" % task)
 
          for root, dirs, files in os.walk('.', topdown=True):
-            # Sort directories and files to ensure consistent ordering
+            # Sort directories to ensure consistent ordering when recursing
              dirs.sort()
              files.sort()
 
-            for f in files:
-                path = os.path.join(root, f)
+            def process(path):
                  s = os.lstat(path)
 
-                # Hash file path
-                update_hash(path + '\n')
+                if stat.S_ISDIR(s.st_mode):
+                    update_hash('d')
+                elif stat.S_ISCHR(s.st_mode):
+                    update_hash('c')
+                elif stat.S_ISBLK(s.st_mode):
+                    update_hash('b')
+                elif stat.S_ISSOCK(s.st_mode):
+                    update_hash('s')
+                elif stat.S_ISLNK(s.st_mode):
+                    update_hash('l')
+                elif stat.S_ISFIFO(s.st_mode):
+                    update_hash('p')
+                else:
+                    update_hash('-')
+
+                def add_perm(mask, on, off='-'):
+                    if mask & s.st_mode:
+                        update_hash(on)
+                    else:
+                        update_hash(off)
+
+                add_perm(stat.S_IRUSR, 'r')
+                add_perm(stat.S_IWUSR, 'w')
+                if stat.S_ISUID & s.st_mode:
+                    add_perm(stat.S_IXUSR, 's', 'S')
+                else:
+                    add_perm(stat.S_IXUSR, 'x')
 
-                # Hash file mode
-                update_hash("\tmode=0x%x\n" % stat.S_IMODE(s.st_mode))
-                update_hash("\ttype=0x%x\n" % stat.S_IFMT(s.st_mode))
+                add_perm(stat.S_IRGRP, 'r')
+                add_perm(stat.S_IWGRP, 'w')
+                if stat.S_ISGID & s.st_mode:
+                    add_perm(stat.S_IXGRP, 's', 'S')
+                else:
+                    add_perm(stat.S_IXGRP, 'x')
 
-                if stat.S_ISBLK(s.st_mode) or stat.S_ISBLK(s.st_mode):
-                    # Hash device major and minor
-                    update_hash("\tdev=%d,%d\n" % (os.major(s.st_rdev), os.minor(s.st_rdev)))
-                elif stat.S_ISLNK(s.st_mode):
-                    # Hash symbolic link
-                    update_hash("\tsymlink=%s\n" % os.readlink(path))

meta/classes/sstate.bbclass def OEOuthashBasic(path, sigfile, task, d): · line 864

Quoted 10 lines
meta/classes/sstate.bbclass +10 -0
+                add_perm(stat.S_IROTH, 'r')
+                add_perm(stat.S_IWOTH, 'w')
+                if stat.S_ISVTX & s.st_mode:
+                    update_hash('t')
+                else:
+                    add_perm(stat.S_IXOTH, 'x')
+
+                if include_owners:
+                    #update_hash(" %5d" % s.st_uid)
+                    #update_hash(" %5d" % s.st_gid)

left-overs from development ?

Quoted 39 lines
meta/classes/sstate.bbclass +28 -1
+                    update_hash(" %10s" % pwd.getpwuid(s.st_uid).pw_name)
+                    update_hash(" %10s" % grp.getgrgid(s.st_gid).gr_name)
+
+                if stat.S_ISBLK(s.st_mode) or stat.S_ISCHR(s.st_mode):
+                    update_hash(" %9s" % ("%d.%d" % (os.major(s.st_rdev), os.minor(s.st_rdev))))
                  else:
+                    update_hash(" " * 10)
+
+                if stat.S_ISREG(s.st_mode):
+                    update_hash(" %10d" % s.st_size)
+                else:
+                    update_hash(" " * 11)
+
+                update_hash(" %s" % path)
+
+                if stat.S_ISLNK(s.st_mode):
+                    update_hash(" -> %s" % os.readlink(path))
+
+                if stat.S_ISREG(s.st_mode):
                      fh = hashlib.sha256()
                      # Hash file contents
                      with open(path, 'rb') as d:
                          for chunk in iter(lambda: d.read(4096), b""):
                              fh.update(chunk)
-                    update_hash("\tdigest=%s\n" % fh.hexdigest())
+                    update_hash(" %s" % fh.hexdigest())
+
+                update_hash("\n")
+
+            # Process this directory and all its child files
+            process(root)
+            for f in files:
+                if f == 'fixmepath':
+                    continue
+                process(os.path.join(root, f))
      finally:
          os.chdir(prev_dir)
 
#3 lore raw
In reply to Joshua Watt
Quoted 10 lines · Joshua Watt

Updates the output hash calculation for determining if tasks are equivalent. The new algorithm does the following based on feedback:
1) All files are printed in a single line tabular format
2) Prints the file type and mode in a user-friendly ls-like format
3) Includes the file owner and group (by name, not ID). These are
only
included if the task is run under pseudo, since that is the only time they can be consistently determined.
4) File size is included for regular files

Quoted 5 lines
---
 meta/classes/sstate.bbclass | 91 +++++++++++++++++++++++++++++++--
----
 1 file changed, 76 insertions(+), 15 deletions(-)

I've not really looked at the code itself yet but could we turn this into a function in lib/oe/sstate*.py somewhere please?

I'd really like to start getting some of the python code into strong function libraries where it makes sense and this lends itself toa library function.

Cheers,

Richard

#4 lore raw

PATCH v2 1/1 classes/sstate: Update output hash

In reply to Joshua Watt

Updates the output hash calculation for determining if tasks are equivalent. The new algorithm does the following based on feedback:
1) The output hash function was moved to the OE library.
2) All files are printed in a single line tabular format
3) Prints the file type and mode in a user-friendly ls-like format
4) Includes the file owner and group (by name, not ID). These are only included if the task is run under pseudo since that is the only time they can be consistently determined.
5) File size is included for regular files

---
 meta/classes/sstate.bbclass |  65 +----------------
 meta/lib/oe/sstatesig.py    | 139 +++++++++++++++++++++++++++++++++++-
 2 files changed, 139 insertions(+), 65 deletions(-)
meta/classes/sstate.bbclass +3 -62
index 763fce07f95..2f0bbd2d7df 100644
@@ -83,9 +83,9 @@ SSTATE_SIG_PASSPHRASE ?= ""
 # Whether to verify the GnUPG signatures when extracting sstate archives
 SSTATE_VERIFY_SIG ?= "0"
 
-SSTATE_HASHEQUIV_METHOD ?= "OEOuthashBasic"
-SSTATE_HASHEQUIV_METHOD[doc] = "The function used to calculate the output hash \
-    for a task, which in turn is used to determine equivalency. \
+SSTATE_HASHEQUIV_METHOD ?= "oe.sstatesig.OEOuthashBasic"
+SSTATE_HASHEQUIV_METHOD[doc] = "The fully-qualified function used to calculate \
+    the output hash for a task, which in turn is used to determine equivalency. \
     "
 
 SSTATE_HASHEQUIV_SERVER ?= ""
@@ -782,65 +782,6 @@ python sstate_sign_package () {
                            d.getVar('SSTATE_SIG_PASSPHRASE'), armor=False)
 }
 
-def OEOuthashBasic(path, sigfile, task, d):
-    import hashlib
-    import stat
-
-    def update_hash(s):
-        s = s.encode('utf-8')
-        h.update(s)
-        if sigfile:
-            sigfile.write(s)
-
-    h = hashlib.sha256()
-    prev_dir = os.getcwd()
-
-    try:
-        os.chdir(path)
-
-        update_hash("OEOuthashBasic\n")
-
-        # It is only currently useful to get equivalent hashes for things that
-        # can be restored from sstate. Since the sstate object is named using
-        # SSTATE_PKGSPEC and the task name, those should be included in the
-        # output hash calculation.
-        update_hash("SSTATE_PKGSPEC=%s\n" % d.getVar('SSTATE_PKGSPEC'))
-        update_hash("task=%s\n" % task)
-
-        for root, dirs, files in os.walk('.', topdown=True):
-            # Sort directories and files to ensure consistent ordering
-            dirs.sort()
-            files.sort()
-
-            for f in files:
-                path = os.path.join(root, f)
-                s = os.lstat(path)
-
-                # Hash file path
-                update_hash(path + '\n')
-
-                # Hash file mode
-                update_hash("\tmode=0x%x\n" % stat.S_IMODE(s.st_mode))
-                update_hash("\ttype=0x%x\n" % stat.S_IFMT(s.st_mode))
-
-                if stat.S_ISBLK(s.st_mode) or stat.S_ISBLK(s.st_mode):
-                    # Hash device major and minor
-                    update_hash("\tdev=%d,%d\n" % (os.major(s.st_rdev), os.minor(s.st_rdev)))
-                elif stat.S_ISLNK(s.st_mode):
-                    # Hash symbolic link
-                    update_hash("\tsymlink=%s\n" % os.readlink(path))
-                else:
-                    fh = hashlib.sha256()
-                    # Hash file contents
-                    with open(path, 'rb') as d:
-                        for chunk in iter(lambda: d.read(4096), b""):
-                            fh.update(chunk)
-                    update_hash("\tdigest=%s\n" % fh.hexdigest())
-    finally:
-        os.chdir(prev_dir)
-
-    return h.hexdigest()
-
 python sstate_report_unihash() {
     report_unihash = getattr(bb.parse.siggen, 'report_unihash', None)
 
meta/lib/oe/sstatesig.py +136 -3

164 lines not on this page · raw message

Signature · 1 line
2.20.1