Skip to contentSkip to conversation

classes/reproducible: Fix error when no git HEAD

PATCH New Joshua Watt 1 participant 3 messages Jul 20, 2020

lore
Share & tools
lore
#0 lore raw

cover classes/reproducible: Fix error when no git HEAD

Fixes a bug where getting the source date epoch would fail if there was no git HEAD (such as when 'subdir=' is used).

Also takes the opportunity to move many of the anonymous python functions to library code to reducing the parsing burden

Joshua Watt (2):
classes/reproducible: Move to library code
lib/oe/reproducible: Fix error when no git HEAD

 meta/classes/reproducible_build.bbclass |  90 +-------------------
 meta/lib/oe/reproducible.py             | 104 ++++++++++++++++++++++++
 2 files changed, 107 insertions(+), 87 deletions(-)
 create mode 100644 meta/lib/oe/reproducible.py
Signature · 1 line
2.27.0
#1 lore raw

PATCH 1/2 classes/reproducible: Move to library code

Moves most of the python code used for dealing with the source date epoch to library code.

---
 meta/classes/reproducible_build.bbclass | 90 +----------------------
 meta/lib/oe/reproducible.py             | 95 +++++++++++++++++++++++++
 2 files changed, 98 insertions(+), 87 deletions(-)
 create mode 100644 meta/lib/oe/reproducible.py
meta/classes/reproducible_build.bbclass +3 -87
index 8da40f656a..2f3bd90b07 100644
@@ -70,100 +70,16 @@ do_deploy_source_date_epoch[sstate-plaindirs] = "${SDE_DEPLOYDIR}"
 addtask do_deploy_source_date_epoch_setscene
 addtask do_deploy_source_date_epoch before do_configure after do_patch
 
-def get_source_date_epoch_from_known_files(d, sourcedir):
-    source_date_epoch = None
-    newest_file = None
-    known_files = set(["NEWS", "ChangeLog", "Changelog", "CHANGES"])
-    for file in known_files:
-        filepath = os.path.join(sourcedir, file)
-        if os.path.isfile(filepath):
-            mtime = int(os.lstat(filepath).st_mtime)
-            # There may be more than one "known_file" present, if so, use the youngest one
-            if not source_date_epoch or mtime > source_date_epoch:
-                source_date_epoch = mtime
-                newest_file = filepath
-    if newest_file:
-        bb.debug(1, "SOURCE_DATE_EPOCH taken from: %s" % newest_file)
-    return source_date_epoch
-
-def find_git_folder(d, sourcedir):
-    # First guess: WORKDIR/git
-    # This is the default git fetcher unpack path
-    workdir = d.getVar('WORKDIR')
-    gitpath = os.path.join(workdir, "git/.git")
-    if os.path.isdir(gitpath):
-        return gitpath
-
-    # Second guess: ${S}
-    gitpath = os.path.join(sourcedir, ".git")
-    if os.path.isdir(gitpath):
-        return gitpath
-
-    # Perhaps there was a subpath or destsuffix specified.
-    # Go looking in the WORKDIR
-    exclude = set(["build", "image", "license-destdir", "patches", "pseudo",
-                   "recipe-sysroot", "recipe-sysroot-native", "sysroot-destdir", "temp"])
-    for root, dirs, files in os.walk(workdir, topdown=True):
-        dirs[:] = [d for d in dirs if d not in exclude]
-        if '.git' in dirs:
-            return root
-
-    bb.warn("Failed to find a git repository in WORKDIR: %s" % workdir)
-    return None
-
-def get_source_date_epoch_from_git(d, sourcedir):
-    source_date_epoch = None
-    if "git://" in d.getVar('SRC_URI'):
-        gitpath = find_git_folder(d, sourcedir)
-        if gitpath:
-            import subprocess
-            source_date_epoch = int(subprocess.check_output(['git','log','-1','--pretty=%ct'], cwd=gitpath))
-            bb.debug(1, "git repository: %s" % gitpath)
-    return source_date_epoch
-
-def get_source_date_epoch_from_youngest_file(d, sourcedir):
-    if sourcedir == d.getVar('WORKDIR'):
-       # These sources are almost certainly not from a tarball
-       return None
-
-    # Do it the hard way: check all files and find the youngest one...
-    source_date_epoch = None
-    newest_file = None
-    for root, dirs, files in os.walk(sourcedir, topdown=True):
-        files = [f for f in files if not f[0] == '.']
-
-        for fname in files:
-            filename = os.path.join(root, fname)
-            try:
-                mtime = int(os.lstat(filename).st_mtime)
-            except ValueError:
-                mtime = 0
-            if not source_date_epoch or mtime > source_date_epoch:
-                source_date_epoch = mtime
-                newest_file = filename
-
-    if newest_file:
-        bb.debug(1, "Newest file found: %s" % newest_file)
-    return source_date_epoch
-
-def fixed_source_date_epoch():
-    bb.debug(1, "No tarball or git repo found to determine SOURCE_DATE_EPOCH")
-    return 0
-
 python create_source_date_epoch_stamp() {
+    import oe.reproducible
+
     epochfile = d.getVar('SDE_FILE')
     # If it exists we need to regenerate as the sources may have changed
     if os.path.isfile(epochfile):
         bb.debug(1, "Deleting existing SOURCE_DATE_EPOCH from: %s" % epochfile)
         os.remove(epochfile)
 
-    sourcedir = d.getVar('S')
-    source_date_epoch = (
-        get_source_date_epoch_from_git(d, sourcedir) or
-        get_source_date_epoch_from_known_files(d, sourcedir) or
-        get_source_date_epoch_from_youngest_file(d, sourcedir) or
-        fixed_source_date_epoch()       # Last resort
-    )
+    source_date_epoch = oe.reproducible.get_source_date_epoch(d, d.getVar('S'))
 
     bb.debug(1, "SOURCE_DATE_EPOCH: %d" % source_date_epoch)
     bb.utils.mkdirhier(d.getVar('SDE_DIR'))
meta/lib/oe/reproducible.py new file +95 -0

95 lines not on this page · raw message

Signature · 1 line
2.27.0
#2 lore raw

PATCH 2/2 lib/oe/reproducible: Fix error when no git HEAD

In reply to Joshua Watt

Fixes an error that occurs when attempting to get the timestamp of the latest commit when there is no HEAD in the git repository. The easiest way to trigger this condition is to use the 'subdir=' option when specifying a 'git://' SRC_URI.

---
 meta/lib/oe/reproducible.py | 25 +++++++++++++++++--------
 1 file changed, 17 insertions(+), 8 deletions(-)
meta/lib/oe/reproducible.py +17 -8
index f80a85ddef..f4f58dd952 100644
@@ -47,14 +47,23 @@ def find_git_folder(d, sourcedir):
     return None
 
 def get_source_date_epoch_from_git(d, sourcedir):
-    source_date_epoch = None
-    if "git://" in d.getVar('SRC_URI'):
-        gitpath = find_git_folder(d, sourcedir)
-        if gitpath:
-            import subprocess
-            source_date_epoch = int(subprocess.check_output(['git','log','-1','--pretty=%ct'], cwd=gitpath))
-            bb.debug(1, "git repository: %s" % gitpath)
-    return source_date_epoch
+    if not "git://" in d.getVar('SRC_URI'):
+        return None
+
+    gitpath = find_git_folder(d, sourcedir)
+    if not gitpath:
+        return None
+
+    # Check that the repository has a valid HEAD; it may not if subdir is used
+    # in SRC_URI
+    p = subprocess.run(['git', 'rev-parse', 'HEAD'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, cwd=gitpath)
+    if p.returncode != 0:
+        bb.debug(1, "%s does not have a valid HEAD: %s" % (gitpath, p.stdout.decode('utf-8')))
+        return None
+
+    bb.debug(1, "git repository: %s" % gitpath)
+    p = subprocess.run(['git','log','-1','--pretty=%ct'], check=True, stdout=subprocess.PIPE, cwd=gitpath)
+    return int(p.stdout.decode('utf-8'))
 
 def get_source_date_epoch_from_youngest_file(d, sourcedir):
     if sourcedir == d.getVar('WORKDIR'):
Signature · 1 line
2.27.0