classes/reproducible: Fix error when no git HEAD
cover classes/reproducible: Fix error when no git HEAD
Fixes a bug where getting the source date epoch would fail if there was no git HEAD (such as when 'subdir=' is used).
Also takes the opportunity to move many of the anonymous python functions to library code to reducing the parsing burden
Joshua Watt (2):
classes/reproducible: Move to library code
lib/oe/reproducible: Fix error when no git HEAD
meta/classes/reproducible_build.bbclass | 90 +------------------- meta/lib/oe/reproducible.py | 104 ++++++++++++++++++++++++ 2 files changed, 107 insertions(+), 87 deletions(-) create mode 100644 meta/lib/oe/reproducible.py
Signature · 1 line
2.27.0
PATCH 1/2 classes/reproducible: Move to library code
Moves most of the python code used for dealing with the source date epoch to library code.
--- meta/classes/reproducible_build.bbclass | 90 +---------------------- meta/lib/oe/reproducible.py | 95 +++++++++++++++++++++++++ 2 files changed, 98 insertions(+), 87 deletions(-) create mode 100644 meta/lib/oe/reproducible.py
meta/classes/reproducible_build.bbclass +3 -87
@@ -70,100 +70,16 @@ do_deploy_source_date_epoch[sstate-plaindirs] = "${SDE_DEPLOYDIR}" addtask do_deploy_source_date_epoch_setscene addtask do_deploy_source_date_epoch before do_configure after do_patch -def get_source_date_epoch_from_known_files(d, sourcedir): - source_date_epoch = None - newest_file = None - known_files = set(["NEWS", "ChangeLog", "Changelog", "CHANGES"]) - for file in known_files: - filepath = os.path.join(sourcedir, file) - if os.path.isfile(filepath): - mtime = int(os.lstat(filepath).st_mtime) - # There may be more than one "known_file" present, if so, use the youngest one - if not source_date_epoch or mtime > source_date_epoch: - source_date_epoch = mtime - newest_file = filepath - if newest_file: - bb.debug(1, "SOURCE_DATE_EPOCH taken from: %s" % newest_file) - return source_date_epoch - -def find_git_folder(d, sourcedir): - # First guess: WORKDIR/git - # This is the default git fetcher unpack path - workdir = d.getVar('WORKDIR') - gitpath = os.path.join(workdir, "git/.git") - if os.path.isdir(gitpath): - return gitpath - - # Second guess: ${S} - gitpath = os.path.join(sourcedir, ".git") - if os.path.isdir(gitpath): - return gitpath - - # Perhaps there was a subpath or destsuffix specified. - # Go looking in the WORKDIR - exclude = set(["build", "image", "license-destdir", "patches", "pseudo", - "recipe-sysroot", "recipe-sysroot-native", "sysroot-destdir", "temp"]) - for root, dirs, files in os.walk(workdir, topdown=True): - dirs[:] = [d for d in dirs if d not in exclude] - if '.git' in dirs: - return root - - bb.warn("Failed to find a git repository in WORKDIR: %s" % workdir) - return None - -def get_source_date_epoch_from_git(d, sourcedir): - source_date_epoch = None - if "git://" in d.getVar('SRC_URI'): - gitpath = find_git_folder(d, sourcedir) - if gitpath: - import subprocess - source_date_epoch = int(subprocess.check_output(['git','log','-1','--pretty=%ct'], cwd=gitpath)) - bb.debug(1, "git repository: %s" % gitpath) - return source_date_epoch - -def get_source_date_epoch_from_youngest_file(d, sourcedir): - if sourcedir == d.getVar('WORKDIR'): - # These sources are almost certainly not from a tarball - return None - - # Do it the hard way: check all files and find the youngest one... - source_date_epoch = None - newest_file = None - for root, dirs, files in os.walk(sourcedir, topdown=True): - files = [f for f in files if not f[0] == '.'] - - for fname in files: - filename = os.path.join(root, fname) - try: - mtime = int(os.lstat(filename).st_mtime) - except ValueError: - mtime = 0 - if not source_date_epoch or mtime > source_date_epoch: - source_date_epoch = mtime - newest_file = filename - - if newest_file: - bb.debug(1, "Newest file found: %s" % newest_file) - return source_date_epoch - -def fixed_source_date_epoch(): - bb.debug(1, "No tarball or git repo found to determine SOURCE_DATE_EPOCH") - return 0 - python create_source_date_epoch_stamp() { + import oe.reproducible + epochfile = d.getVar('SDE_FILE') # If it exists we need to regenerate as the sources may have changed if os.path.isfile(epochfile): bb.debug(1, "Deleting existing SOURCE_DATE_EPOCH from: %s" % epochfile) os.remove(epochfile) - sourcedir = d.getVar('S') - source_date_epoch = ( - get_source_date_epoch_from_git(d, sourcedir) or - get_source_date_epoch_from_known_files(d, sourcedir) or - get_source_date_epoch_from_youngest_file(d, sourcedir) or - fixed_source_date_epoch() # Last resort - ) + source_date_epoch = oe.reproducible.get_source_date_epoch(d, d.getVar('S')) bb.debug(1, "SOURCE_DATE_EPOCH: %d" % source_date_epoch) bb.utils.mkdirhier(d.getVar('SDE_DIR'))
meta/lib/oe/reproducible.py new file +95 -0
95 lines not on this page · raw message
Signature · 1 line
2.27.0
PATCH 2/2 lib/oe/reproducible: Fix error when no git HEAD
In reply to Joshua WattFixes an error that occurs when attempting to get the timestamp of the latest commit when there is no HEAD in the git repository. The easiest way to trigger this condition is to use the 'subdir=' option when specifying a 'git://' SRC_URI.
--- meta/lib/oe/reproducible.py | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-)
meta/lib/oe/reproducible.py +17 -8
@@ -47,14 +47,23 @@ def find_git_folder(d, sourcedir): return None def get_source_date_epoch_from_git(d, sourcedir): - source_date_epoch = None - if "git://" in d.getVar('SRC_URI'): - gitpath = find_git_folder(d, sourcedir) - if gitpath: - import subprocess - source_date_epoch = int(subprocess.check_output(['git','log','-1','--pretty=%ct'], cwd=gitpath)) - bb.debug(1, "git repository: %s" % gitpath) - return source_date_epoch + if not "git://" in d.getVar('SRC_URI'): + return None + + gitpath = find_git_folder(d, sourcedir) + if not gitpath: + return None + + # Check that the repository has a valid HEAD; it may not if subdir is used + # in SRC_URI + p = subprocess.run(['git', 'rev-parse', 'HEAD'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, cwd=gitpath) + if p.returncode != 0: + bb.debug(1, "%s does not have a valid HEAD: %s" % (gitpath, p.stdout.decode('utf-8'))) + return None + + bb.debug(1, "git repository: %s" % gitpath) + p = subprocess.run(['git','log','-1','--pretty=%ct'], check=True, stdout=subprocess.PIPE, cwd=gitpath) + return int(p.stdout.decode('utf-8')) def get_source_date_epoch_from_youngest_file(d, sourcedir): if sourcedir == d.getVar('WORKDIR'):
Signature · 1 line
2.27.0