1.44 merge request
cover 1.44 merge request
Clean A-full build.
Please merge these chages to 1.44
The following changes since commit 1a79651c518abc35b99005c137ab7e82a99c75b0:
fetch2: Ensure cached url data is matched to a datastore (2019-11-13 21:59:58 +0000)
are available in the git repository at:
git://git.openembedded.org/bitbake-contrib stable/1.44-next
http://cgit.openembedded.org/bitbake-contrib/log/?h=stable/1.44-next
Christopher Larson (2):
fetch2/git: fetch shallow revs when needed
tests/fetch: add test for fetching shallow revs
Gavin Li (1):
prserv: fix ResourceWarning due to unclosed socket
Peter Kjellerstedt (1):
cooker: Remove a left-over comment about expanded_data
Richard Purdie (2):
fetch2/clearcase: Fix warnings from python 3.8
runqueue: Fix hash equivalence duplicate tasks running
Ross Burton (2):
tests: add test for the hashing functions
utils: also use mmap for SHA256 and SHA1, for performance
Volker Vogelhuber (1):
fetch2/hg: Fix various runtime issues
lib/bb/cooker.py | 4 ---- lib/bb/fetch2/clearcase.py | 6 +++--- lib/bb/fetch2/git.py | 23 ++++++++++++++++------- lib/bb/fetch2/hg.py | 27 +++++++++++++++++---------- lib/bb/runqueue.py | 14 ++++++++++---- lib/bb/tests/fetch.py | 20 ++++++++++++++++++++ lib/bb/tests/utils.py | 26 ++++++++++++++++++++++++++ lib/bb/utils.py | 34 ++++++++++++++-------------------- lib/prserv/serv.py | 1 + 9 files changed, 107 insertions(+), 48 deletions(-)
Signature · 1 line
2.7.4
PATCH 1/9 fetch2/git: fetch shallow revs when needed
From: Christopher Larson <[email protected]>
When bitbake determines if a git clone needs updating, it only checks for the needed srcrevs, not the revs listed in BB_GIT_SHALLOW_REVS, which will fail if using shallow and the needed rev was added to the upstream git repo after a previous fetch. Ensure that we also check for shallow revs.
[YOCTO #13586]
--- lib/bb/fetch2/git.py | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-)
lib/bb/fetch2/git.py +16 -7
@@ -292,11 +292,21 @@ class Git(FetchMethod): def clonedir_need_update(self, ud, d): if not os.path.exists(ud.clonedir): return True + if ud.shallow and ud.write_shallow_tarballs and self.clonedir_need_shallow_revs(ud, d): + return True for name in ud.names: if not self._contains_ref(ud, d, name, ud.clonedir): return True return False + def clonedir_need_shallow_revs(self, ud, d): + for rev in ud.shallow_revs: + try: + runfetchcmd('%s rev-parse -q --verify %s' % (ud.basecmd, rev), d, quiet=True, workdir=ud.clonedir) + except bb.fetch2.FetchError: + return rev + return None + def shallow_tarball_need_update(self, ud): return ud.shallow and ud.write_shallow_tarballs and not os.path.exists(ud.fullshallow)
@@ -339,13 +349,7 @@ class Git(FetchMethod): runfetchcmd(clone_cmd, d, log=progresshandler) # Update the checkout if needed - needupdate = False - for name in ud.names: - if not self._contains_ref(ud, d, name, ud.clonedir): - needupdate = True - break - - if needupdate: + if self.clonedir_need_update(ud, d): output = runfetchcmd("%s remote" % ud.basecmd, d, quiet=True, workdir=ud.clonedir) if "origin" in output: runfetchcmd("%s remote rm origin" % ud.basecmd, d, workdir=ud.clonedir)
@@ -369,6 +373,11 @@ class Git(FetchMethod): if not self._contains_ref(ud, d, name, ud.clonedir): raise bb.fetch2.FetchError("Unable to find revision %s in branch %s even from upstream" % (ud.revisions[name], ud.branches[name])) + if ud.shallow and ud.write_shallow_tarballs: + missing_rev = self.clonedir_need_shallow_revs(ud, d) + if missing_rev: + raise bb.fetch2.FetchError("Unable to find revision %s even from upstream" % missing_rev) + def build_mirror_data(self, ud, d): if ud.shallow and ud.write_shallow_tarballs: if not os.path.exists(ud.fullshallow):
Signature · 1 line
2.7.4
PATCH 2/9 tests/fetch: add test for fetching shallow revs
In reply to Armin KusterFrom: Christopher Larson <[email protected]>
[YOCTO #13586]
--- lib/bb/tests/fetch.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+)
lib/bb/tests/fetch.py +20 -0
@@ -1863,6 +1863,26 @@ class GitShallowTest(FetcherTest): with self.assertRaises(bb.fetch2.FetchError): self.fetch() + def test_shallow_fetch_missing_revs(self): + self.add_empty_file('a') + self.add_empty_file('b') + fetcher, ud = self.fetch(self.d.getVar('SRC_URI')) + self.git('tag v0.0 master', cwd=self.srcdir) + self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0') + self.d.setVar('BB_GIT_SHALLOW_REVS', 'v0.0') + self.fetch_shallow() + + def test_shallow_fetch_missing_revs_fails(self): + self.add_empty_file('a') + self.add_empty_file('b') + fetcher, ud = self.fetch(self.d.getVar('SRC_URI')) + self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0') + self.d.setVar('BB_GIT_SHALLOW_REVS', 'v0.0') + + with self.assertRaises(bb.fetch2.FetchError), self.assertLogs("BitBake.Fetcher", level="ERROR") as cm: + self.fetch_shallow() + self.assertIn("Unable to find revision v0.0 even from upstream", cm.output[0]) + @skipIfNoNetwork() def test_bitbake(self): self.git('remote add --mirror=fetch origin git://github.com/openembedded/bitbake', cwd=self.srcdir)
Signature · 1 line
2.7.4
PATCH 3/9 fetch2/hg: Fix various runtime issues
In reply to Armin KusterFrom: Volker Vogelhuber <[email protected]>
Fix mercurial fetching after breakage from changes to the core fetcher. Fix username and password usage and setting moddir needed by setup_revisions.
--- lib/bb/fetch2/hg.py | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-)
lib/bb/fetch2/hg.py +17 -10
@@ -54,13 +54,6 @@ class Hg(FetchMethod): else: ud.proto = "hg" - ud.setup_revisions(d) - - if 'rev' in ud.parm: - ud.revision = ud.parm['rev'] - elif not ud.revision: - ud.revision = self.latest_revision(ud, d) - # Create paths to mercurial checkouts hgsrcname = '%s_%s_%s' % (ud.module.replace('/', '.'), \ ud.host, ud.path.replace('/', '.'))
@@ -74,6 +67,13 @@ class Hg(FetchMethod): ud.localfile = ud.moddir ud.basecmd = d.getVar("FETCHCMD_hg") or "/usr/bin/env hg" + ud.setup_revisions(d) + + if 'rev' in ud.parm: + ud.revision = ud.parm['rev'] + elif not ud.revision: + ud.revision = self.latest_revision(ud, d) + ud.write_tarballs = d.getVar("BB_GENERATE_MIRROR_TARBALLS") def need_update(self, ud, d):
@@ -139,7 +139,7 @@ class Hg(FetchMethod): cmd = "%s --config auth.default.prefix=* --config auth.default.username=%s --config auth.default.password=%s --config \"auth.default.schemes=%s\" pull" % (ud.basecmd, ud.user, ud.pswd, proto) else: cmd = "%s pull" % (ud.basecmd) - elif command == "update": + elif command == "update" or command == "up": if ud.user and ud.pswd: cmd = "%s --config auth.default.prefix=* --config auth.default.username=%s --config auth.default.password=%s --config \"auth.default.schemes=%s\" update -C %s" % (ud.basecmd, ud.user, ud.pswd, proto, " ".join(options)) else:
@@ -247,12 +247,19 @@ class Hg(FetchMethod): scmdata = ud.parm.get("scmdata", "") if scmdata != "nokeep": + proto = ud.parm.get('protocol', 'http') if not os.access(os.path.join(codir, '.hg'), os.R_OK): logger.debug(2, "Unpack: creating new hg repository in '" + codir + "'") runfetchcmd("%s init %s" % (ud.basecmd, codir), d) logger.debug(2, "Unpack: updating source in '" + codir + "'") - runfetchcmd("%s pull %s" % (ud.basecmd, ud.moddir), d, workdir=codir) - runfetchcmd("%s up -C %s" % (ud.basecmd, revflag), d, workdir=codir) + if ud.user and ud.pswd: + runfetchcmd("%s --config auth.default.prefix=* --config auth.default.username=%s --config auth.default.password=%s --config \"auth.default.schemes=%s\" pull %s" % (ud.basecmd, ud.user, ud.pswd, proto, ud.moddir), d, workdir=codir) + else: + runfetchcmd("%s pull %s" % (ud.basecmd, ud.moddir), d, workdir=codir) + if ud.user and ud.pswd: + runfetchcmd("%s --config auth.default.prefix=* --config auth.default.username=%s --config auth.default.password=%s --config \"auth.default.schemes=%s\" up -C %s" % (ud.basecmd, ud.user, ud.pswd, proto, revflag), d, workdir=codir) + else: + runfetchcmd("%s up -C %s" % (ud.basecmd, revflag), d, workdir=codir) else: logger.debug(2, "Unpack: extracting source to '" + codir + "'") runfetchcmd("%s archive -t files %s %s" % (ud.basecmd, revflag, codir), d, workdir=ud.moddir)
Signature · 1 line
2.7.4
PATCH 4/9 prserv: fix ResourceWarning due to unclosed socket
In reply to Armin KusterFrom: Gavin Li <[email protected]>
With PRSERV_HOST = "localhost:0", this message would occasionally pop up during the initial cache read:
WARNING: /home/matic/ambayocto/poky/bitbake/lib/bb/cache.py:446: ResourceWarning: unclosed <socket.socket fd=10, family=AddressFamily.AF_INET, type=SocketKind.SOCK_STREAM, proto=0, laddr=('127.0.0.1', 45655)>
value = pickled.load()
The file location stated is irrelevant; it just happens to be wherever CPython decides to run the garbage collector. The issue is that after we fork off a PRServer, self.socket is also duplicated. The parent side of it also needs to be closed.
--- lib/prserv/serv.py | 1 + 1 file changed, 1 insertion(+)
lib/prserv/serv.py +1 -0
@@ -243,6 +243,7 @@ class PRServer(SimpleXMLRPCServer): try: pid = os.fork() if pid > 0: + self.socket.close() # avoid ResourceWarning in parent return pid except OSError as e: raise Exception("%s [%d]" % (e.strerror, e.errno))
Signature · 1 line
2.7.4
PATCH 5/9 cooker: Remove a left-over comment about expanded_data
In reply to Armin KusterFrom: Peter Kjellerstedt <[email protected]>
This should have been removed together with expanded_data in commit e3694e73 (cooker/command: Drop expanded_data).
--- lib/bb/cooker.py | 4 ---- 1 file changed, 4 deletions(-)
lib/bb/cooker.py +0 -4
@@ -371,10 +371,6 @@ class BBCooker: self.data.setVar('BB_CMDLINE', self.ui_cmdline) - # - # Copy of the data store which has been expanded. - # Used for firing events and accessing variables where expansion needs to be accounted for - # if CookerFeatures.BASEDATASTORE_TRACKING in self.featureset: self.disableDataTracking()
Signature · 1 line
2.7.4
PATCH 6/9 tests: add test for the hashing functions
In reply to Armin KusterFrom: Ross Burton <[email protected]>
Add a basic test for bb.utils.md5_file() etc.
--- lib/bb/tests/utils.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+)
lib/bb/tests/utils.py +26 -0
@@ -103,6 +103,32 @@ class Path(unittest.TestCase): result = bb.utils._check_unsafe_delete_path(arg1) self.assertEqual(result, correctresult, '_check_unsafe_delete_path("%s") != %s' % (arg1, correctresult)) +class Checksum(unittest.TestCase): + filler = b"Shiver me timbers square-rigged spike Gold Road galleon bilge water boatswain wherry jack pirate. Mizzenmast rum lad Privateer jack salmagundi hang the jib piracy Pieces of Eight Corsair. Parrel marooned black spot yawl provost quarterdeck cable no prey, no pay spirits lateen sail." + + def test_md5(self): + import hashlib + with tempfile.NamedTemporaryFile() as f: + f.write(self.filler) + f.flush() + checksum = bb.utils.md5_file(f.name) + self.assertEqual(checksum, "bd572cd5de30a785f4efcb6eaf5089e3") + + def test_sha1(self): + import hashlib + with tempfile.NamedTemporaryFile() as f: + f.write(self.filler) + f.flush() + checksum = bb.utils.sha1_file(f.name) + self.assertEqual(checksum, "249eb8fd654732ea836d5e702d7aa567898eca71") + + def test_sha256(self): + import hashlib + with tempfile.NamedTemporaryFile() as f: + f.write(self.filler) + f.flush() + checksum = bb.utils.sha256_file(f.name) + self.assertEqual(checksum, "fcfbae8bf6b721dbb9d2dc6a9334a58f2031a9a9b302999243f99da4d7f12d0f") class EditMetadataFile(unittest.TestCase): _origfile = """
Signature · 1 line
2.7.4
PATCH 7/9 utils: also use mmap for SHA256 and SHA1, for performance
In reply to Armin KusterFrom: Ross Burton <[email protected]>
md5_file() uses a mmap() window to improve performance when hashing files, so refactor the code and do the same for SHA1 and SHA256.
--- lib/bb/utils.py | 34 ++++++++++++++-------------------- 1 file changed, 14 insertions(+), 20 deletions(-)
lib/bb/utils.py +14 -20
@@ -520,22 +520,26 @@ def unlockfile(lf): fcntl.flock(lf.fileno(), fcntl.LOCK_UN) lf.close() -def md5_file(filename): - """ - Return the hex string representation of the MD5 checksum of filename. - """ - import hashlib, mmap +def _hasher(method, filename): + import mmap with open(filename, "rb") as f: - m = hashlib.md5() try: with mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ) as mm: for chunk in iter(lambda: mm.read(8192), b''): - m.update(chunk) + method.update(chunk) except ValueError: # You can't mmap() an empty file so silence this exception pass - return m.hexdigest() + return method.hexdigest() + + +def md5_file(filename): + """ + Return the hex string representation of the MD5 checksum of filename. + """ + import hashlib + return _hasher(hashlib.md5(), filename) def sha256_file(filename): """
@@ -543,24 +547,14 @@ def sha256_file(filename): filename. """ import hashlib - - s = hashlib.sha256() - with open(filename, "rb") as f: - for line in f: - s.update(line) - return s.hexdigest() + return _hasher(hashlib.sha256(), filename) def sha1_file(filename): """ Return the hex string representation of the SHA1 checksum of the filename """ import hashlib - - s = hashlib.sha1() - with open(filename, "rb") as f: - for line in f: - s.update(line) - return s.hexdigest() + return _hasher(hashlib.sha1(), filename) def preserved_envvars_exported(): """Variables which are taken from the environment and placed in and exported
Signature · 1 line
2.7.4
PATCH 8/9 fetch2/clearcase: Fix warnings from python 3.8
In reply to Armin KusterFrom: Richard Purdie <[email protected]>
bitbake/lib/bb/fetch2/clearcase.py:148: SyntaxWarning: "is" with a literal. Did you mean "=="?
if command is 'mkview':bitbake/lib/bb/fetch2/clearcase.py:155: SyntaxWarning: "is" with a literal. Did you mean "=="?
elif command is 'rmview':bitbake/lib/bb/fetch2/clearcase.py:159: SyntaxWarning: "is" with a literal. Did you mean "=="?
elif command is 'setcs':
Python 3.8 is quite correct and we so mean "==" here, fix it to avoid the warnings.
--- lib/bb/fetch2/clearcase.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-)
lib/bb/fetch2/clearcase.py +3 -3
@@ -145,18 +145,18 @@ class ClearCase(FetchMethod): basecmd = "%s %s" % (ud.basecmd, command) - if command is 'mkview': + if command == 'mkview': if not "rcleartool" in ud.basecmd: # Cleartool needs a -snapshot view options.append("-snapshot") options.append("-tag %s" % ud.viewname) options.append(ud.viewdir) - elif command is 'rmview': + elif command == 'rmview': options.append("-force") options.append("%s" % ud.viewdir) - elif command is 'setcs': + elif command == 'setcs': options.append("-overwrite") options.append(ud.configspecfile)
Signature · 1 line
2.7.4
PATCH 9/9 runqueue: Fix hash equivalence duplicate tasks running
In reply to Armin KusterFrom: Richard Purdie <[email protected]>
The key problem is that currently running setscene tasks are not accounted for when processing task migrations. This means can allow two of the same task to execute at the same time with unpredictable effects.
This change allows us to stop doing that and refactor the code slightly to make it clearer that these conditions don't arrive even with deferred tasks.
--- lib/bb/runqueue.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-)
lib/bb/runqueue.py +10 -4
@@ -2303,16 +2303,22 @@ class RunQueueExecute: for tid in changed: if tid not in self.rqdata.runq_setscene_tids: continue + if tid not in self.pending_migrations: + self.pending_migrations.add(tid) + + for tid in self.pending_migrations.copy(): if tid in self.runq_running: + # Too late, task already running, not much we can do now + self.pending_migrations.remove(tid) continue - if tid in self.scenequeue_covered: + + if tid in self.scenequeue_covered or tid in self.sq_live: + # Already ran this setscene task or it running # Potentially risky, should we report this hash as a match? logger.info("Already covered setscene for %s so ignoring rehash" % (tid)) + self.pending_migrations.remove(tid) continue - if tid not in self.pending_migrations: - self.pending_migrations.add(tid) - for tid in self.pending_migrations.copy(): valid = True # Check no tasks this covers are running for dep in self.sqdata.sq_covered_tasks[tid]:
Signature · 1 line
2.7.4