Skip to contentSkip to conversation

1.44 merge request

PATCH New Armin Kuster 1 participant 10 messages Nov 24, 2019

lore
Share & tools
lore
#0 lore raw

cover 1.44 merge request

Clean A-full build.

Please merge these chages to 1.44

The following changes since commit 1a79651c518abc35b99005c137ab7e82a99c75b0:

fetch2: Ensure cached url data is matched to a datastore (2019-11-13 21:59:58 +0000)

are available in the git repository at:

git://git.openembedded.org/bitbake-contrib stable/1.44-next
http://cgit.openembedded.org/bitbake-contrib/log/?h=stable/1.44-next

Christopher Larson (2):
fetch2/git: fetch shallow revs when needed
tests/fetch: add test for fetching shallow revs

Gavin Li (1):
prserv: fix ResourceWarning due to unclosed socket

Peter Kjellerstedt (1):
cooker: Remove a left-over comment about expanded_data

Richard Purdie (2):
fetch2/clearcase: Fix warnings from python 3.8
runqueue: Fix hash equivalence duplicate tasks running

Ross Burton (2):
tests: add test for the hashing functions
utils: also use mmap for SHA256 and SHA1, for performance

Volker Vogelhuber (1):
fetch2/hg: Fix various runtime issues

 lib/bb/cooker.py           |  4 ----
 lib/bb/fetch2/clearcase.py |  6 +++---
 lib/bb/fetch2/git.py       | 23 ++++++++++++++++-------
 lib/bb/fetch2/hg.py        | 27 +++++++++++++++++----------
 lib/bb/runqueue.py         | 14 ++++++++++----
 lib/bb/tests/fetch.py      | 20 ++++++++++++++++++++
 lib/bb/tests/utils.py      | 26 ++++++++++++++++++++++++++
 lib/bb/utils.py            | 34 ++++++++++++++--------------------
 lib/prserv/serv.py         |  1 +
 9 files changed, 107 insertions(+), 48 deletions(-)
Signature · 1 line
2.7.4
#1 lore raw

PATCH 1/9 fetch2/git: fetch shallow revs when needed

From: Christopher Larson <[email protected]>

When bitbake determines if a git clone needs updating, it only checks for the needed srcrevs, not the revs listed in BB_GIT_SHALLOW_REVS, which will fail if using shallow and the needed rev was added to the upstream git repo after a previous fetch. Ensure that we also check for shallow revs.

[YOCTO #13586]

---
 lib/bb/fetch2/git.py | 23 ++++++++++++++++-------
 1 file changed, 16 insertions(+), 7 deletions(-)
lib/bb/fetch2/git.py +16 -7
index 2d1d2ca..fa41b07 100644
@@ -292,11 +292,21 @@ class Git(FetchMethod):
     def clonedir_need_update(self, ud, d):
         if not os.path.exists(ud.clonedir):
             return True
+        if ud.shallow and ud.write_shallow_tarballs and self.clonedir_need_shallow_revs(ud, d):
+            return True
         for name in ud.names:
             if not self._contains_ref(ud, d, name, ud.clonedir):
                 return True
         return False
 
+    def clonedir_need_shallow_revs(self, ud, d):
+        for rev in ud.shallow_revs:
+            try:
+                runfetchcmd('%s rev-parse -q --verify %s' % (ud.basecmd, rev), d, quiet=True, workdir=ud.clonedir)
+            except bb.fetch2.FetchError:
+                return rev
+        return None
+
     def shallow_tarball_need_update(self, ud):
         return ud.shallow and ud.write_shallow_tarballs and not os.path.exists(ud.fullshallow)
 
@@ -339,13 +349,7 @@ class Git(FetchMethod):
             runfetchcmd(clone_cmd, d, log=progresshandler)
 
         # Update the checkout if needed
-        needupdate = False
-        for name in ud.names:
-            if not self._contains_ref(ud, d, name, ud.clonedir):
-                needupdate = True
-                break
-
-        if needupdate:
+        if self.clonedir_need_update(ud, d):
             output = runfetchcmd("%s remote" % ud.basecmd, d, quiet=True, workdir=ud.clonedir)
             if "origin" in output:
               runfetchcmd("%s remote rm origin" % ud.basecmd, d, workdir=ud.clonedir)
@@ -369,6 +373,11 @@ class Git(FetchMethod):
             if not self._contains_ref(ud, d, name, ud.clonedir):
                 raise bb.fetch2.FetchError("Unable to find revision %s in branch %s even from upstream" % (ud.revisions[name], ud.branches[name]))
 
+        if ud.shallow and ud.write_shallow_tarballs:
+            missing_rev = self.clonedir_need_shallow_revs(ud, d)
+            if missing_rev:
+                raise bb.fetch2.FetchError("Unable to find revision %s even from upstream" % missing_rev)
+
     def build_mirror_data(self, ud, d):
         if ud.shallow and ud.write_shallow_tarballs:
             if not os.path.exists(ud.fullshallow):
Signature · 1 line
2.7.4
#2 lore raw

PATCH 2/9 tests/fetch: add test for fetching shallow revs

In reply to Armin Kuster

From: Christopher Larson <[email protected]>

[YOCTO #13586]

---
 lib/bb/tests/fetch.py | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)
lib/bb/tests/fetch.py +20 -0
index a0b656b..83fad3f 100644
@@ -1863,6 +1863,26 @@ class GitShallowTest(FetcherTest):
         with self.assertRaises(bb.fetch2.FetchError):
             self.fetch()
 
+    def test_shallow_fetch_missing_revs(self):
+        self.add_empty_file('a')
+        self.add_empty_file('b')
+        fetcher, ud = self.fetch(self.d.getVar('SRC_URI'))
+        self.git('tag v0.0 master', cwd=self.srcdir)
+        self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0')
+        self.d.setVar('BB_GIT_SHALLOW_REVS', 'v0.0')
+        self.fetch_shallow()
+
+    def test_shallow_fetch_missing_revs_fails(self):
+        self.add_empty_file('a')
+        self.add_empty_file('b')
+        fetcher, ud = self.fetch(self.d.getVar('SRC_URI'))
+        self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0')
+        self.d.setVar('BB_GIT_SHALLOW_REVS', 'v0.0')
+
+        with self.assertRaises(bb.fetch2.FetchError), self.assertLogs("BitBake.Fetcher", level="ERROR") as cm:
+            self.fetch_shallow()
+        self.assertIn("Unable to find revision v0.0 even from upstream", cm.output[0])
+
     @skipIfNoNetwork()
     def test_bitbake(self):
         self.git('remote add --mirror=fetch origin git://github.com/openembedded/bitbake', cwd=self.srcdir)
Signature · 1 line
2.7.4
#3 lore raw

PATCH 3/9 fetch2/hg: Fix various runtime issues

In reply to Armin Kuster

From: Volker Vogelhuber <[email protected]>

Fix mercurial fetching after breakage from changes to the core fetcher. Fix username and password usage and setting moddir needed by setup_revisions.

---
 lib/bb/fetch2/hg.py | 27 +++++++++++++++++----------
 1 file changed, 17 insertions(+), 10 deletions(-)
lib/bb/fetch2/hg.py +17 -10
index 15d729e..e21115d 100644
@@ -54,13 +54,6 @@ class Hg(FetchMethod):
         else:
             ud.proto = "hg"
 
-        ud.setup_revisions(d)
-
-        if 'rev' in ud.parm:
-            ud.revision = ud.parm['rev']
-        elif not ud.revision:
-            ud.revision = self.latest_revision(ud, d)
-
         # Create paths to mercurial checkouts
         hgsrcname = '%s_%s_%s' % (ud.module.replace('/', '.'), \
                             ud.host, ud.path.replace('/', '.'))
@@ -74,6 +67,13 @@ class Hg(FetchMethod):
         ud.localfile = ud.moddir
         ud.basecmd = d.getVar("FETCHCMD_hg") or "/usr/bin/env hg"
 
+        ud.setup_revisions(d)
+
+        if 'rev' in ud.parm:
+            ud.revision = ud.parm['rev']
+        elif not ud.revision:
+            ud.revision = self.latest_revision(ud, d)
+
         ud.write_tarballs = d.getVar("BB_GENERATE_MIRROR_TARBALLS")
 
     def need_update(self, ud, d):
@@ -139,7 +139,7 @@ class Hg(FetchMethod):
                 cmd = "%s --config auth.default.prefix=* --config auth.default.username=%s --config auth.default.password=%s --config \"auth.default.schemes=%s\" pull" % (ud.basecmd, ud.user, ud.pswd, proto)
             else:
                 cmd = "%s pull" % (ud.basecmd)
-        elif command == "update":
+        elif command == "update" or command == "up":
             if ud.user and ud.pswd:
                 cmd = "%s --config auth.default.prefix=* --config auth.default.username=%s --config auth.default.password=%s --config \"auth.default.schemes=%s\" update -C %s" % (ud.basecmd, ud.user, ud.pswd, proto, " ".join(options))
             else:
@@ -247,12 +247,19 @@ class Hg(FetchMethod):
 
         scmdata = ud.parm.get("scmdata", "")
         if scmdata != "nokeep":
+            proto = ud.parm.get('protocol', 'http')
             if not os.access(os.path.join(codir, '.hg'), os.R_OK):
                 logger.debug(2, "Unpack: creating new hg repository in '" + codir + "'")
                 runfetchcmd("%s init %s" % (ud.basecmd, codir), d)
             logger.debug(2, "Unpack: updating source in '" + codir + "'")
-            runfetchcmd("%s pull %s" % (ud.basecmd, ud.moddir), d, workdir=codir)
-            runfetchcmd("%s up -C %s" % (ud.basecmd, revflag), d, workdir=codir)
+            if ud.user and ud.pswd:
+                runfetchcmd("%s --config auth.default.prefix=* --config auth.default.username=%s --config auth.default.password=%s --config \"auth.default.schemes=%s\" pull %s" % (ud.basecmd, ud.user, ud.pswd, proto, ud.moddir), d, workdir=codir)
+            else:
+                runfetchcmd("%s pull %s" % (ud.basecmd, ud.moddir), d, workdir=codir)
+            if ud.user and ud.pswd:
+                runfetchcmd("%s --config auth.default.prefix=* --config auth.default.username=%s --config auth.default.password=%s --config \"auth.default.schemes=%s\" up -C %s" % (ud.basecmd, ud.user, ud.pswd, proto, revflag), d, workdir=codir)
+            else:
+                runfetchcmd("%s up -C %s" % (ud.basecmd, revflag), d, workdir=codir)
         else:
             logger.debug(2, "Unpack: extracting source to '" + codir + "'")
             runfetchcmd("%s archive -t files %s %s" % (ud.basecmd, revflag, codir), d, workdir=ud.moddir)
Signature · 1 line
2.7.4
#4 lore raw

PATCH 4/9 prserv: fix ResourceWarning due to unclosed socket

In reply to Armin Kuster

From: Gavin Li <[email protected]>

With PRSERV_HOST = "localhost:0", this message would occasionally pop up during the initial cache read:

WARNING: /home/matic/ambayocto/poky/bitbake/lib/bb/cache.py:446: ResourceWarning: unclosed <socket.socket fd=10, family=AddressFamily.AF_INET, type=SocketKind.SOCK_STREAM, proto=0, laddr=('127.0.0.1', 45655)>
value = pickled.load()

The file location stated is irrelevant; it just happens to be wherever CPython decides to run the garbage collector. The issue is that after we fork off a PRServer, self.socket is also duplicated. The parent side of it also needs to be closed.

---
 lib/prserv/serv.py | 1 +
 1 file changed, 1 insertion(+)
lib/prserv/serv.py +1 -0
index be3acec..1d9148b 100644
@@ -243,6 +243,7 @@ class PRServer(SimpleXMLRPCServer):
         try:
             pid = os.fork()
             if pid > 0:
+                self.socket.close() # avoid ResourceWarning in parent
                 return pid
         except OSError as e:
             raise Exception("%s [%d]" % (e.strerror, e.errno))
Signature · 1 line
2.7.4
#5 lore raw

PATCH 5/9 cooker: Remove a left-over comment about expanded_data

In reply to Armin Kuster

From: Peter Kjellerstedt <[email protected]>

This should have been removed together with expanded_data in commit e3694e73 (cooker/command: Drop expanded_data).

---
 lib/bb/cooker.py | 4 ----
 1 file changed, 4 deletions(-)
lib/bb/cooker.py +0 -4
index 20ef04d..e6442bf 100644
@@ -371,10 +371,6 @@ class BBCooker:
 
         self.data.setVar('BB_CMDLINE', self.ui_cmdline)
 
-        #
-        # Copy of the data store which has been expanded.
-        # Used for firing events and accessing variables where expansion needs to be accounted for
-        #
         if CookerFeatures.BASEDATASTORE_TRACKING in self.featureset:
             self.disableDataTracking()
 
Signature · 1 line
2.7.4
#6 lore raw

PATCH 6/9 tests: add test for the hashing functions

In reply to Armin Kuster

From: Ross Burton <[email protected]>

Add a basic test for bb.utils.md5_file() etc.

---
 lib/bb/tests/utils.py | 26 ++++++++++++++++++++++++++
 1 file changed, 26 insertions(+)
lib/bb/tests/utils.py +26 -0
index f4adf1d..5c910b4 100644
@@ -103,6 +103,32 @@ class Path(unittest.TestCase):
             result = bb.utils._check_unsafe_delete_path(arg1)
             self.assertEqual(result, correctresult, '_check_unsafe_delete_path("%s") != %s' % (arg1, correctresult))
 
+class Checksum(unittest.TestCase):
+    filler = b"Shiver me timbers square-rigged spike Gold Road galleon bilge water boatswain wherry jack pirate. Mizzenmast rum lad Privateer jack salmagundi hang the jib piracy Pieces of Eight Corsair. Parrel marooned black spot yawl provost quarterdeck cable no prey, no pay spirits lateen sail."
+
+    def test_md5(self):
+        import hashlib
+        with tempfile.NamedTemporaryFile() as f:
+            f.write(self.filler)
+            f.flush()
+            checksum = bb.utils.md5_file(f.name)
+            self.assertEqual(checksum, "bd572cd5de30a785f4efcb6eaf5089e3")
+
+    def test_sha1(self):
+        import hashlib
+        with tempfile.NamedTemporaryFile() as f:
+            f.write(self.filler)
+            f.flush()
+            checksum = bb.utils.sha1_file(f.name)
+            self.assertEqual(checksum, "249eb8fd654732ea836d5e702d7aa567898eca71")
+
+    def test_sha256(self):
+        import hashlib
+        with tempfile.NamedTemporaryFile() as f:
+            f.write(self.filler)
+            f.flush()
+            checksum = bb.utils.sha256_file(f.name)
+            self.assertEqual(checksum, "fcfbae8bf6b721dbb9d2dc6a9334a58f2031a9a9b302999243f99da4d7f12d0f")
 
 class EditMetadataFile(unittest.TestCase):
     _origfile = """
Signature · 1 line
2.7.4
#7 lore raw

PATCH 7/9 utils: also use mmap for SHA256 and SHA1, for performance

In reply to Armin Kuster

From: Ross Burton <[email protected]>

md5_file() uses a mmap() window to improve performance when hashing files, so refactor the code and do the same for SHA1 and SHA256.

---
 lib/bb/utils.py | 34 ++++++++++++++--------------------
 1 file changed, 14 insertions(+), 20 deletions(-)
lib/bb/utils.py +14 -20
index d035949..8d40bcd 100644
@@ -520,22 +520,26 @@ def unlockfile(lf):
     fcntl.flock(lf.fileno(), fcntl.LOCK_UN)
     lf.close()
 
-def md5_file(filename):
-    """
-    Return the hex string representation of the MD5 checksum of filename.
-    """
-    import hashlib, mmap
+def _hasher(method, filename):
+    import mmap
 
     with open(filename, "rb") as f:
-        m = hashlib.md5()
         try:
             with mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ) as mm:
                 for chunk in iter(lambda: mm.read(8192), b''):
-                    m.update(chunk)
+                    method.update(chunk)
         except ValueError:
             # You can't mmap() an empty file so silence this exception
             pass
-    return m.hexdigest()
+    return method.hexdigest()
+
+
+def md5_file(filename):
+    """
+    Return the hex string representation of the MD5 checksum of filename.
+    """
+    import hashlib
+    return _hasher(hashlib.md5(), filename)
 
 def sha256_file(filename):
     """
@@ -543,24 +547,14 @@ def sha256_file(filename):
     filename.
     """
     import hashlib
-
-    s = hashlib.sha256()
-    with open(filename, "rb") as f:
-        for line in f:
-            s.update(line)
-    return s.hexdigest()
+    return _hasher(hashlib.sha256(), filename)
 
 def sha1_file(filename):
     """
     Return the hex string representation of the SHA1 checksum of the filename
     """
     import hashlib
-
-    s = hashlib.sha1()
-    with open(filename, "rb") as f:
-        for line in f:
-            s.update(line)
-    return s.hexdigest()
+    return _hasher(hashlib.sha1(), filename)
 
 def preserved_envvars_exported():
     """Variables which are taken from the environment and placed in and exported
Signature · 1 line
2.7.4
#8 lore raw

PATCH 8/9 fetch2/clearcase: Fix warnings from python 3.8

In reply to Armin Kuster

From: Richard Purdie <[email protected]>

bitbake/lib/bb/fetch2/clearcase.py:148: SyntaxWarning: "is" with a literal. Did you mean "=="?
if command is 'mkview':
bitbake/lib/bb/fetch2/clearcase.py:155: SyntaxWarning: "is" with a literal. Did you mean "=="?
elif command is 'rmview':
bitbake/lib/bb/fetch2/clearcase.py:159: SyntaxWarning: "is" with a literal. Did you mean "=="?
elif command is 'setcs':

Python 3.8 is quite correct and we so mean "==" here, fix it to avoid the warnings.

---
 lib/bb/fetch2/clearcase.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)
lib/bb/fetch2/clearcase.py +3 -3
index 3dd93ad..e2934ef 100644
@@ -145,18 +145,18 @@ class ClearCase(FetchMethod):
 
         basecmd = "%s %s" % (ud.basecmd, command)
 
-        if command is 'mkview':
+        if command == 'mkview':
             if not "rcleartool" in ud.basecmd:
                 # Cleartool needs a -snapshot view
                 options.append("-snapshot")
             options.append("-tag %s" % ud.viewname)
             options.append(ud.viewdir)
 
-        elif command is 'rmview':
+        elif command == 'rmview':
             options.append("-force")
             options.append("%s" % ud.viewdir)
 
-        elif command is 'setcs':
+        elif command == 'setcs':
             options.append("-overwrite")
             options.append(ud.configspecfile)
 
Signature · 1 line
2.7.4
#9 lore raw

PATCH 9/9 runqueue: Fix hash equivalence duplicate tasks running

In reply to Armin Kuster

From: Richard Purdie <[email protected]>

The key problem is that currently running setscene tasks are not accounted for when processing task migrations. This means can allow two of the same task to execute at the same time with unpredictable effects.

This change allows us to stop doing that and refactor the code slightly to make it clearer that these conditions don't arrive even with deferred tasks.

---
 lib/bb/runqueue.py | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)
lib/bb/runqueue.py +10 -4
index 1804943..02a9b91 100644
@@ -2303,16 +2303,22 @@ class RunQueueExecute:
         for tid in changed:
             if tid not in self.rqdata.runq_setscene_tids:
                 continue
+            if tid not in self.pending_migrations:
+                self.pending_migrations.add(tid)
+
+        for tid in self.pending_migrations.copy():
             if tid in self.runq_running:
+                # Too late, task already running, not much we can do now
+                self.pending_migrations.remove(tid)
                 continue
-            if tid in self.scenequeue_covered:
+
+            if tid in self.scenequeue_covered or tid in self.sq_live:
+                # Already ran this setscene task or it running
                 # Potentially risky, should we report this hash as a match?
                 logger.info("Already covered setscene for %s so ignoring rehash" % (tid))
+                self.pending_migrations.remove(tid)
                 continue
-            if tid not in self.pending_migrations:
-                self.pending_migrations.add(tid)
 
-        for tid in self.pending_migrations.copy():
             valid = True
             # Check no tasks this covers are running
             for dep in self.sqdata.sq_covered_tasks[tid]:
Signature · 1 line
2.7.4