hashserv: Merge divergent output hashes
PATCH 1/1 hashserv: Merge divergent output hashes
Instructs the hash equivalence server to merge diverging output hashes to the same unihash when one is reported that unifies them. The primary use for this is -cross and -native recipes where the outputs will never match on different host architectures, but the unihashes need to converge.
Signed-off-by Joshua Watt <[email protected]>
--- bitbake/lib/hashserv/__init__.py | 1 + bitbake/lib/hashserv/server.py | 31 +++++++++++++++++++++++-- bitbake/lib/hashserv/tests.py | 39 ++++++++++++++++++++++++++++++++ 3 files changed, 69 insertions(+), 2 deletions(-)
bitbake/lib/hashserv/__init__.py +1 -0
@@ -48,6 +48,7 @@ def setup_database(database, sync=True): # Create new indexes cursor.execute('CREATE INDEX IF NOT EXISTS taskhash_lookup_v2 ON tasks_v2 (method, taskhash, created)') cursor.execute('CREATE INDEX IF NOT EXISTS outhash_lookup_v2 ON tasks_v2 (method, outhash)') + cursor.execute('CREATE INDEX IF NOT EXISTS unihash_lookup_v2 ON tasks_v2 (method, unihash)') return db
bitbake/lib/hashserv/server.py +29 -2
@@ -267,9 +267,36 @@ class ServerClient(object): # If a row matching the outhash was found, the unihash for # the new taskhash should be the same as that one. # Otherwise the caller provided unihash is used. - unihash = data['unihash'] - if row is not None: + if row is not None and data['unihash'] != row['unihash']: + # Update unihashes to ensure all branches are converging on + # the same unihash. This is generally a bad thing because + # it means that builds are not reproducible, but it + # occasionally is unavoidable such as in the case of -cross + # and -native tasks for different build host architectures. + # For example, take the following taskhashes that produce + # the given outhashes and unihashes: + # + # taskhash outhash unihash + # A Z 1 + # B Y 2 + # C Y 3 -> 2 + # B Z 2 -> 1 + # + # In this case, the B task isn't being built reproducibly, + # but for the second B the server sees it matches the + # outhash of A, and reports that the unihash should be + # changed from 2 to 1. + # + # The inconsistency is that there are still entries that + # refer to unihash 2. These should be remapped to unihash + # 1, since the latest entry shows that these are + # equivalent. + cursor.execute('''UPDATE tasks_v2 SET unihash=:new_unihash WHERE method=:method AND unihash=:old_unihash''', + {'method': data['method'], 'new_unihash': row['unihash'], 'old_unihash': data['unihash']}) + unihash = row['unihash'] + else: + unihash = data['unihash'] insert_data = { 'method': data['method'],
bitbake/lib/hashserv/tests.py +39 -0
@@ -99,6 +99,45 @@ class TestHashEquivalenceServer(object): result = self.client.get_unihash(self.METHOD, taskhash) self.assertEqual(result, unihash) + def test_hash_merging(self): + # Verify that unihashes are correctly merged together when diverging + # hashes are found. Uses the following table which describes how each + # task is reported: + # + # taskhash outhash unihash + # A Z A + # B Y B + # C Y C -> B + # B Z B -> A + # + A_taskhash = "A" + B_taskhash = "B" + C_taskhash = "C" + Z_outhash = "Z" + Y_outhash = "Y" + + result = self.client.report_unihash(A_taskhash, self.METHOD, Z_outhash, A_taskhash) + self.assertEqual(result['unihash'], A_taskhash, 'Server reported bad unihash change for task A') + + result = self.client.report_unihash(B_taskhash, self.METHOD, Y_outhash, B_taskhash) + self.assertEqual(result['unihash'], B_taskhash, 'Server reported bad unihash change for task A') + + result = self.client.report_unihash(C_taskhash, self.METHOD, Y_outhash, C_taskhash) + self.assertEqual(result['unihash'], B_taskhash, 'Server reported bad unihash change for task C') + + # Report a second B with the Z outhash. It should be change to A's unihash + result = self.client.report_unihash(B_taskhash, self.METHOD, Z_outhash, B_taskhash) + self.assertEqual(result['unihash'], A_taskhash, 'Server reported bad unihash change for task B') + + # The unihash for C should also be A's unihash + result = self.client.get_unihash(self.METHOD, C_taskhash) + self.assertEqual(result, A_taskhash, 'Server returned bad unihash for task C') + + # The reported unihash for B should also be A. NOTE: this *should* + # return the first B reported because it is older + result = self.client.get_unihash(self.METHOD, B_taskhash) + self.assertEqual(result, A_taskhash, 'Server returned bad unihash for task B') + def test_stress(self): def query_server(failures): client = Client(self.server.address)
Signature · 1 line
2.23.0