Skip to contentSkip to conversation

hashserv: tighten unihash handling

PATCH New Anders Heimer 1 participant 3 messages Jun 5

lore
Share & tools
lore
#0 lore raw

cover hashserv: tighten unihash handling

This series tightens unihash handling so hashserv accepts and stores only valid lowercase 64-character hexadecimal unihash values, and BitBake treats BB_UNIHASH as literal hash data when reading it from the datastore.

The first patch updates existing hashserv test fixtures to use valid 64-character unihash-shaped values.

The second patch adds unihash validation and focused coverage for hashserv report paths and BitBake siggen handling. Invalid unihashes reported to hashserv, or returned by hashserv to BitBake, are rejected before use.

Validation:
- py_compile on touched Python files
- focused hashserv/siggen tests
- broader hashserv and siggen regression tests

Anders Heimer (2):
hashserv/tests: use valid 64-character unihashes
hashserv: validate unihash values

 lib/bb/siggen.py         |   9 +-
 lib/bb/tests/siggen.py   |  47 ++++++++++
 lib/hashserv/__init__.py |   7 ++
 lib/hashserv/server.py   |  27 +++++-
 lib/hashserv/tests.py    | 194 ++++++++++++++++++++++++++++-----------
 5 files changed, 224 insertions(+), 60 deletions(-)
Signature · 1 line
2.34.1
#1 lore raw

PATCH 1/2 hashserv/tests: use valid 64-character unihashes

---
 lib/hashserv/tests.py | 164 ++++++++++++++++++++++++++++--------------
 1 file changed, 110 insertions(+), 54 deletions(-)
lib/hashserv/tests.py +110 -54
index 124d8aa00..83ce0c5ca 100644
@@ -28,6 +28,7 @@ from pathlib import Path
 THIS_DIR = Path(__file__).parent
 BIN_DIR = THIS_DIR.parent.parent / "bin"
 
+
 def server_prefunc(server, idx):
     logging.basicConfig(level=logging.DEBUG, filename='bbhashserv-%d.log' % idx, filemode='w',
                         format='%(levelname)s %(filename)s:%(lineno)d %(message)s')
@@ -124,7 +125,7 @@ class HashEquivalenceTestSetup(object):
         # Simple test that hashes can be created
         taskhash = '35788efcb8dfb0a02659d81cf2bfd695fb30faf9'
         outhash = '2765d4a5884be49b28601445c2760c5f21e7e5c0ee2b7e3fce98fd7e5970796f'
-        unihash = 'f46d3fbb439bd9b921095da657a4de906510d2cd'
+        unihash = 'a69ec97f5af2e21e1a1f9cc8896965515d5559425666f734e245a3d40cee33d9'
 
         self.assertClientGetHash(client, taskhash, None)
 
@@ -178,7 +179,7 @@ class HashEquivalenceCommonTests(object):
         # assigned the same unihash
         taskhash = '53b8dce672cb6d0c73170be43f540460bfc347b4'
         outhash = '5a9cb1649625f0bf41fc7791b635cd9c2d7118c7f021ba87dcd03f72b67ce7a8'
-        unihash = 'f37918cc02eb5a520b1aff86faacbc0a38124646'
+        unihash = '46edb5140d2613049332d0bf3745d9fafec9c559dac8cc61813739a28007fcdf'
 
         result = self.client.report_unihash(taskhash, self.METHOD, outhash, unihash)
         self.assertEqual(result['unihash'], unihash, 'Server returned bad unihash')
@@ -186,7 +187,7 @@ class HashEquivalenceCommonTests(object):
         # Report a different task with the same outhash. The returned unihash
         # should match the first task
         taskhash2 = '3bf6f1e89d26205aec90da04854fbdbf73afe6b4'
-        unihash2 = 'af36b199320e611fbb16f1f277d3ee1d619ca58b'
+        unihash2 = 'bf6e81926066f770e960f9f777cd088c62bea9addb7745f3e77deaa81a645747'
         result = self.client.report_unihash(taskhash2, self.METHOD, outhash, unihash2)
         self.assertEqual(result['unihash'], unihash, 'Server returned bad unihash')
 
@@ -196,19 +197,19 @@ class HashEquivalenceCommonTests(object):
         # taskhash
         taskhash = '8aa96fcffb5831b3c2c0cb75f0431e3f8b20554a'
         outhash = 'afe240a439959ce86f5e322f8c208e1fedefea9e813f2140c81af866cc9edf7e'
-        unihash = '218e57509998197d570e2c98512d0105985dffc9'
+        unihash = '5b521d8a12683086cc08bc2c6d94a7a2dcff17eba53b9911e145d51164689380'
         self.client.report_unihash(taskhash, self.METHOD, outhash, unihash)
 
         self.assertClientGetHash(self.client, taskhash, unihash)
 
         outhash2 = '0904a7fe3dc712d9fd8a74a616ddca2a825a8ee97adf0bd3fc86082c7639914d'
-        unihash2 = 'ae9a7d252735f0dafcdb10e2e02561ca3a47314c'
+        unihash2 = 'a37541b54fd22440e292f617eb30ba07455e88fb0b9f0952eca229b6356290e3'
         self.client.report_unihash(taskhash, self.METHOD, outhash2, unihash2)
 
         self.assertClientGetHash(self.client, taskhash, unihash)
 
         outhash3 = '77623a549b5b1a31e3732dfa8fe61d7ce5d44b3370f253c5360e136b852967b4'
-        unihash3 = '9217a7d6398518e5dc002ed58f2cbbbc78696603'
+        unihash3 = '6842f1f2daccd96ddef15c9154d4e41ac8a2300d781ac9a9db7f8afeb8a96808'
         self.client.report_unihash(taskhash, self.METHOD, outhash3, unihash3)
 
         self.assertClientGetHash(self.client, taskhash, unihash)
@@ -268,7 +269,7 @@ class HashEquivalenceCommonTests(object):
         # Simple test that hashes can be created
         taskhash = 'c665584ee6817aa99edfc77a44dd853828279370'
         outhash = '3c979c3db45c569f51ab7626a4651074be3a9d11a84b1db076f5b14f7d39db44'
-        unihash = '90e9bc1d1f094c51824adca7f8ea79a048d68824'
+        unihash = '06f89b8f329ba8124ff73c56d09ef921b42624747c421277bddaf5e23f136e57'
 
         self.assertClientGetHash(self.client, taskhash, None)
 
@@ -361,7 +362,7 @@ class HashEquivalenceCommonTests(object):
         # Basic report
         taskhash = '8aa96fcffb5831b3c2c0cb75f0431e3f8b20554a'
         outhash = 'afe240a439959ce86f5e322f8c208e1fedefea9e813f2140c81af866cc9edf7e'
-        unihash = '218e57509998197d570e2c98512d0105985dffc9'
+        unihash = '5b521d8a12683086cc08bc2c6d94a7a2dcff17eba53b9911e145d51164689380'
         self.client.report_unihash(taskhash, self.METHOD, outhash, unihash)
 
         check_hash(taskhash, unihash, None)
@@ -369,7 +370,7 @@ class HashEquivalenceCommonTests(object):
         # Duplicated taskhash with multiple output hashes and unihashes.
         # All servers should agree with the originally reported hash
         outhash2 = '0904a7fe3dc712d9fd8a74a616ddca2a825a8ee97adf0bd3fc86082c7639914d'
-        unihash2 = 'ae9a7d252735f0dafcdb10e2e02561ca3a47314c'
+        unihash2 = 'a37541b54fd22440e292f617eb30ba07455e88fb0b9f0952eca229b6356290e3'
         self.client.report_unihash(taskhash, self.METHOD, outhash2, unihash2)
 
         check_hash(taskhash, unihash, unihash)
@@ -377,7 +378,7 @@ class HashEquivalenceCommonTests(object):
         # Report an equivalent task. The sideload will originally report
         # no unihash until backfilled
         taskhash3 = "044c2ec8aaf480685a00ff6ff49e6162e6ad34e1"
-        unihash3 = "def64766090d28f627e816454ed46894bb3aab36"
+        unihash3 = "aca636d800aef40e6ddcea4b2262cc4ea0d1180a6783e5b4653a20c7dd73458d"
         self.client.report_unihash(taskhash3, self.METHOD, outhash, unihash3)
 
         check_hash(taskhash3, unihash, None)
@@ -386,7 +387,7 @@ class HashEquivalenceCommonTests(object):
         # propagating to the upstream server
         taskhash4 = "e3da00593d6a7fb435c7e2114976c59c5fd6d561"
         outhash4 = "1cf8713e645f491eb9c959d20b5cae1c47133a292626dda9b10709857cbe688a"
-        unihash4 = "3b5d3d83f07f259e9086fcb422c855286e18a57d"
+        unihash4 = "7aebef07d66a8c0f92d0c4f65ec8b1fbb850a3693c53827b8774b64fa9a8a9fe"
         down_client.report_unihash(taskhash4, self.METHOD, outhash4, unihash4)
         down_client.backfill_wait()
 
@@ -398,18 +399,18 @@ class HashEquivalenceCommonTests(object):
         # match which was previously reported to the upstream server
         taskhash5 = '35788efcb8dfb0a02659d81cf2bfd695fb30faf9'
         outhash5 = '2765d4a5884be49b28601445c2760c5f21e7e5c0ee2b7e3fce98fd7e5970796f'
-        unihash5 = 'f46d3fbb439bd9b921095da657a4de906510d2cd'
+        unihash5 = 'a69ec97f5af2e21e1a1f9cc8896965515d5559425666f734e245a3d40cee33d9'
         result = self.client.report_unihash(taskhash5, self.METHOD, outhash5, unihash5)
 
         taskhash6 = '35788efcb8dfb0a02659d81cf2bfd695fb30fafa'
-        unihash6 = 'f46d3fbb439bd9b921095da657a4de906510d2ce'
+        unihash6 = 'eabc7a98e0c12bbeb8394dbdf055eb81aac60e4a14cca5c1f069d36efc933b23'
         result = down_client.report_unihash(taskhash6, self.METHOD, outhash5, unihash6)
         self.assertEqual(result['unihash'], unihash5, 'Server failed to copy unihash from upstream')
 
         # Tests read through from server with
         taskhash7 = '9d81d76242cc7cfaf7bf74b94b9cd2e29324ed74'
         outhash7 = '8470d56547eea6236d7c81a644ce74670ca0bbda998e13c629ef6bb3f0d60b69'
-        unihash7 = '05d2a63c81e32f0a36542ca677e8ad852365c538'
+        unihash7 = '7521a98a0c645341bc51559b234ef37a097e8f3a01665e0303a317925ab7b4d5'
         self.client.report_unihash(taskhash7, self.METHOD, outhash7, unihash7)
 
         result = down_client.get_taskhash(self.METHOD, taskhash7, True)
@@ -420,7 +421,7 @@ class HashEquivalenceCommonTests(object):
 
         taskhash8 = '86978a4c8c71b9b487330b0152aade10c1ee58aa'
         outhash8 = 'ca8c128e9d9e4a28ef24d0508aa20b5cf880604eacd8f65c0e366f7e0cc5fbcf'
-        unihash8 = 'd8bcf25369d40590ad7d08c84d538982f2023e01'
+        unihash8 = '83386d9385b0bf3ba25693127ddcaaadeaa1c4bf8cb0baecfb5314b9a20072a1'
         self.client.report_unihash(taskhash8, self.METHOD, outhash8, unihash8)
 
         result = down_client.get_outhash(self.METHOD, outhash8, taskhash8)
@@ -431,7 +432,7 @@ class HashEquivalenceCommonTests(object):
 
         taskhash9 = 'ae6339531895ddf5b67e663e6a374ad8ec71d81c'
         outhash9 = 'afc78172c81880ae10a1fec994b5b4ee33d196a001a1b66212a15ebe573e00b5'
-        unihash9 = '6662e699d6e3d894b24408ff9a4031ef9b038ee8'
+        unihash9 = 'cc74784b2c0ad5b378a6b783c74c518d2c46b8b52fba29cb39a8430d742440d7'
         self.client.report_unihash(taskhash9, self.METHOD, outhash9, unihash9)
 
         result = down_client.get_taskhash(self.METHOD, taskhash9, False)
@@ -454,7 +455,7 @@ class HashEquivalenceCommonTests(object):
         # Report a hash via the read-write server
         taskhash = '35788efcb8dfb0a02659d81cf2bfd695fb30faf9'
         outhash = '2765d4a5884be49b28601445c2760c5f21e7e5c0ee2b7e3fce98fd7e5970796f'
-        unihash = 'f46d3fbb439bd9b921095da657a4de906510d2cd'
+        unihash = 'a69ec97f5af2e21e1a1f9cc8896965515d5559425666f734e245a3d40cee33d9'
 
         result = rw_client.report_unihash(taskhash, self.METHOD, outhash, unihash)
         self.assertEqual(result['unihash'], unihash, 'Server returned bad unihash')
@@ -465,7 +466,7 @@ class HashEquivalenceCommonTests(object):
         # Ensure that reporting via the read-only server fails
         taskhash2 = 'c665584ee6817aa99edfc77a44dd853828279370'
         outhash2 = '3c979c3db45c569f51ab7626a4651074be3a9d11a84b1db076f5b14f7d39db44'
-        unihash2 = '90e9bc1d1f094c51824adca7f8ea79a048d68824'
+        unihash2 = '06f89b8f329ba8124ff73c56d09ef921b42624747c421277bddaf5e23f136e57'
 
         result = ro_client.report_unihash(taskhash2, self.METHOD, outhash2, unihash2)
         self.assertEqual(result['unihash'], unihash2)
@@ -554,15 +555,43 @@ class HashEquivalenceCommonTests(object):
     def test_get_unihash_batch(self):
         TEST_INPUT = (
             # taskhash                                   outhash                                                            unihash
-            ('8aa96fcffb5831b3c2c0cb75f0431e3f8b20554a', 'afe240a439959ce86f5e322f8c208e1fedefea9e813f2140c81af866cc9edf7e','218e57509998197d570e2c98512d0105985dffc9'),
+            (
+                '8aa96fcffb5831b3c2c0cb75f0431e3f8b20554a',
+                'afe240a439959ce86f5e322f8c208e1fedefea9e813f2140c81af866cc9edf7e',
+                '5b521d8a12683086cc08bc2c6d94a7a2dcff17eba53b9911e145d51164689380',
+            ),
             # Duplicated taskhash with multiple output hashes and unihashes.
-            ('8aa96fcffb5831b3c2c0cb75f0431e3f8b20554a', '0904a7fe3dc712d9fd8a74a616ddca2a825a8ee97adf0bd3fc86082c7639914d', 'ae9a7d252735f0dafcdb10e2e02561ca3a47314c'),
+            (
+                '8aa96fcffb5831b3c2c0cb75f0431e3f8b20554a',
+                '0904a7fe3dc712d9fd8a74a616ddca2a825a8ee97adf0bd3fc86082c7639914d',
+                'a37541b54fd22440e292f617eb30ba07455e88fb0b9f0952eca229b6356290e3',
+            ),
             # Equivalent hash
-            ("044c2ec8aaf480685a00ff6ff49e6162e6ad34e1", '0904a7fe3dc712d9fd8a74a616ddca2a825a8ee97adf0bd3fc86082c7639914d', "def64766090d28f627e816454ed46894bb3aab36"),
-            ("e3da00593d6a7fb435c7e2114976c59c5fd6d561", "1cf8713e645f491eb9c959d20b5cae1c47133a292626dda9b10709857cbe688a", "3b5d3d83f07f259e9086fcb422c855286e18a57d"),
-            ('35788efcb8dfb0a02659d81cf2bfd695fb30faf9', '2765d4a5884be49b28601445c2760c5f21e7e5c0ee2b7e3fce98fd7e5970796f', 'f46d3fbb439bd9b921095da657a4de906510d2cd'),
-            ('35788efcb8dfb0a02659d81cf2bfd695fb30fafa', '2765d4a5884be49b28601445c2760c5f21e7e5c0ee2b7e3fce98fd7e5970796f', 'f46d3fbb439bd9b921095da657a4de906510d2ce'),
-            ('9d81d76242cc7cfaf7bf74b94b9cd2e29324ed74', '8470d56547eea6236d7c81a644ce74670ca0bbda998e13c629ef6bb3f0d60b69', '05d2a63c81e32f0a36542ca677e8ad852365c538'),
+            (
+                "044c2ec8aaf480685a00ff6ff49e6162e6ad34e1",
+                '0904a7fe3dc712d9fd8a74a616ddca2a825a8ee97adf0bd3fc86082c7639914d',
+                "aca636d800aef40e6ddcea4b2262cc4ea0d1180a6783e5b4653a20c7dd73458d",
+            ),
+            (
+                "e3da00593d6a7fb435c7e2114976c59c5fd6d561",
+                "1cf8713e645f491eb9c959d20b5cae1c47133a292626dda9b10709857cbe688a",
+                "7aebef07d66a8c0f92d0c4f65ec8b1fbb850a3693c53827b8774b64fa9a8a9fe",
+            ),
+            (
+                '35788efcb8dfb0a02659d81cf2bfd695fb30faf9',
+                '2765d4a5884be49b28601445c2760c5f21e7e5c0ee2b7e3fce98fd7e5970796f',
+                'a69ec97f5af2e21e1a1f9cc8896965515d5559425666f734e245a3d40cee33d9',
+            ),
+            (
+                '35788efcb8dfb0a02659d81cf2bfd695fb30fafa',
+                '2765d4a5884be49b28601445c2760c5f21e7e5c0ee2b7e3fce98fd7e5970796f',
+                'eabc7a98e0c12bbeb8394dbdf055eb81aac60e4a14cca5c1f069d36efc933b23',
+            ),
+            (
+                '9d81d76242cc7cfaf7bf74b94b9cd2e29324ed74',
+                '8470d56547eea6236d7c81a644ce74670ca0bbda998e13c629ef6bb3f0d60b69',
+                '7521a98a0c645341bc51559b234ef37a097e8f3a01665e0303a317925ab7b4d5',
+            ),
         )
         EXTRA_QUERIES = (
             "6b6be7a84ab179b4240c4302518dc3f6",
@@ -578,28 +607,56 @@ class HashEquivalenceCommonTests(object):
         )
 
         self.assertListEqual(result, [
-            "218e57509998197d570e2c98512d0105985dffc9",
-            "218e57509998197d570e2c98512d0105985dffc9",
-            "218e57509998197d570e2c98512d0105985dffc9",
-            "3b5d3d83f07f259e9086fcb422c855286e18a57d",
-            "f46d3fbb439bd9b921095da657a4de906510d2cd",
-            "f46d3fbb439bd9b921095da657a4de906510d2cd",
-            "05d2a63c81e32f0a36542ca677e8ad852365c538",
+            "5b521d8a12683086cc08bc2c6d94a7a2dcff17eba53b9911e145d51164689380",
+            "5b521d8a12683086cc08bc2c6d94a7a2dcff17eba53b9911e145d51164689380",
+            "5b521d8a12683086cc08bc2c6d94a7a2dcff17eba53b9911e145d51164689380",
+            "7aebef07d66a8c0f92d0c4f65ec8b1fbb850a3693c53827b8774b64fa9a8a9fe",
+            "a69ec97f5af2e21e1a1f9cc8896965515d5559425666f734e245a3d40cee33d9",
+            "a69ec97f5af2e21e1a1f9cc8896965515d5559425666f734e245a3d40cee33d9",
+            "7521a98a0c645341bc51559b234ef37a097e8f3a01665e0303a317925ab7b4d5",
             None,
         ])
 
     def test_unihash_exists_batch(self):
         TEST_INPUT = (
             # taskhash                                   outhash                                                            unihash
-            ('8aa96fcffb5831b3c2c0cb75f0431e3f8b20554a', 'afe240a439959ce86f5e322f8c208e1fedefea9e813f2140c81af866cc9edf7e','218e57509998197d570e2c98512d0105985dffc9'),
+            (
+                '8aa96fcffb5831b3c2c0cb75f0431e3f8b20554a',
+                'afe240a439959ce86f5e322f8c208e1fedefea9e813f2140c81af866cc9edf7e',
+                '5b521d8a12683086cc08bc2c6d94a7a2dcff17eba53b9911e145d51164689380',
+            ),
             # Duplicated taskhash with multiple output hashes and unihashes.
-            ('8aa96fcffb5831b3c2c0cb75f0431e3f8b20554a', '0904a7fe3dc712d9fd8a74a616ddca2a825a8ee97adf0bd3fc86082c7639914d', 'ae9a7d252735f0dafcdb10e2e02561ca3a47314c'),
+            (
+                '8aa96fcffb5831b3c2c0cb75f0431e3f8b20554a',
+                '0904a7fe3dc712d9fd8a74a616ddca2a825a8ee97adf0bd3fc86082c7639914d',
+                'a37541b54fd22440e292f617eb30ba07455e88fb0b9f0952eca229b6356290e3',
+            ),
             # Equivalent hash
-            ("044c2ec8aaf480685a00ff6ff49e6162e6ad34e1", '0904a7fe3dc712d9fd8a74a616ddca2a825a8ee97adf0bd3fc86082c7639914d', "def64766090d28f627e816454ed46894bb3aab36"),
-            ("e3da00593d6a7fb435c7e2114976c59c5fd6d561", "1cf8713e645f491eb9c959d20b5cae1c47133a292626dda9b10709857cbe688a", "3b5d3d83f07f259e9086fcb422c855286e18a57d"),
-            ('35788efcb8dfb0a02659d81cf2bfd695fb30faf9', '2765d4a5884be49b28601445c2760c5f21e7e5c0ee2b7e3fce98fd7e5970796f', 'f46d3fbb439bd9b921095da657a4de906510d2cd'),
-            ('35788efcb8dfb0a02659d81cf2bfd695fb30fafa', '2765d4a5884be49b28601445c2760c5f21e7e5c0ee2b7e3fce98fd7e5970796f', 'f46d3fbb439bd9b921095da657a4de906510d2ce'),
-            ('9d81d76242cc7cfaf7bf74b94b9cd2e29324ed74', '8470d56547eea6236d7c81a644ce74670ca0bbda998e13c629ef6bb3f0d60b69', '05d2a63c81e32f0a36542ca677e8ad852365c538'),
+            (
+                "044c2ec8aaf480685a00ff6ff49e6162e6ad34e1",
+                '0904a7fe3dc712d9fd8a74a616ddca2a825a8ee97adf0bd3fc86082c7639914d',
+                "aca636d800aef40e6ddcea4b2262cc4ea0d1180a6783e5b4653a20c7dd73458d",
+            ),
+            (
+                "e3da00593d6a7fb435c7e2114976c59c5fd6d561",
+                "1cf8713e645f491eb9c959d20b5cae1c47133a292626dda9b10709857cbe688a",
+                "7aebef07d66a8c0f92d0c4f65ec8b1fbb850a3693c53827b8774b64fa9a8a9fe",
+            ),
+            (
+                '35788efcb8dfb0a02659d81cf2bfd695fb30faf9',
+                '2765d4a5884be49b28601445c2760c5f21e7e5c0ee2b7e3fce98fd7e5970796f',
+                'a69ec97f5af2e21e1a1f9cc8896965515d5559425666f734e245a3d40cee33d9',
+            ),
+            (
+                '35788efcb8dfb0a02659d81cf2bfd695fb30fafa',
+                '2765d4a5884be49b28601445c2760c5f21e7e5c0ee2b7e3fce98fd7e5970796f',
+                'eabc7a98e0c12bbeb8394dbdf055eb81aac60e4a14cca5c1f069d36efc933b23',
+            ),
+            (
+                '9d81d76242cc7cfaf7bf74b94b9cd2e29324ed74',
+                '8470d56547eea6236d7c81a644ce74670ca0bbda998e13c629ef6bb3f0d60b69',
+                '7521a98a0c645341bc51559b234ef37a097e8f3a01665e0303a317925ab7b4d5',
+            ),
         )
         EXTRA_QUERIES = (
             "6b6be7a84ab179b4240c4302518dc3f6",
@@ -939,14 +996,14 @@ class HashEquivalenceCommonTests(object):
     def test_gc(self):
         taskhash = '53b8dce672cb6d0c73170be43f540460bfc347b4'
         outhash = '5a9cb1649625f0bf41fc7791b635cd9c2d7118c7f021ba87dcd03f72b67ce7a8'
-        unihash = 'f37918cc02eb5a520b1aff86faacbc0a38124646'
+        unihash = '46edb5140d2613049332d0bf3745d9fafec9c559dac8cc61813739a28007fcdf'
 
         result = self.client.report_unihash(taskhash, self.METHOD, outhash, unihash)
         self.assertEqual(result['unihash'], unihash, 'Server returned bad unihash')
 
         taskhash2 = '3bf6f1e89d26205aec90da04854fbdbf73afe6b4'
         outhash2 = '77623a549b5b1a31e3732dfa8fe61d7ce5d44b3370f253c5360e136b852967b4'
-        unihash2 = 'af36b199320e611fbb16f1f277d3ee1d619ca58b'
+        unihash2 = 'bf6e81926066f770e960f9f777cd088c62bea9addb7745f3e77deaa81a645747'
 
         result = self.client.report_unihash(taskhash2, self.METHOD, outhash2, unihash2)
         self.assertClientGetHash(self.client, taskhash2, unihash2)
@@ -972,21 +1029,21 @@ class HashEquivalenceCommonTests(object):
     def test_gc_stream(self):
         taskhash = '53b8dce672cb6d0c73170be43f540460bfc347b4'
         outhash = '5a9cb1649625f0bf41fc7791b635cd9c2d7118c7f021ba87dcd03f72b67ce7a8'
-        unihash = 'f37918cc02eb5a520b1aff86faacbc0a38124646'
+        unihash = '46edb5140d2613049332d0bf3745d9fafec9c559dac8cc61813739a28007fcdf'
 
         result = self.client.report_unihash(taskhash, self.METHOD, outhash, unihash)
         self.assertEqual(result['unihash'], unihash, 'Server returned bad unihash')
 
         taskhash2 = '3bf6f1e89d26205aec90da04854fbdbf73afe6b4'
         outhash2 = '77623a549b5b1a31e3732dfa8fe61d7ce5d44b3370f253c5360e136b852967b4'
-        unihash2 = 'af36b199320e611fbb16f1f277d3ee1d619ca58b'
+        unihash2 = 'bf6e81926066f770e960f9f777cd088c62bea9addb7745f3e77deaa81a645747'
 
         result = self.client.report_unihash(taskhash2, self.METHOD, outhash2, unihash2)
         self.assertClientGetHash(self.client, taskhash2, unihash2)
 
         taskhash3 = 'a1117c1f5a7c9ab2f5a39cc6fe5e6152169d09c0'
         outhash3 = '7289c414905303700a1117c1f5a7c9ab2f5a39cc6fe5e6152169d09c04f9a53c'
-        unihash3 = '905303700a1117c1f5a7c9ab2f5a39cc6fe5e615'
+        unihash3 = 'f5966c8588a69948a89131d3aa63b914c5a63a8cb4820cb1c58694135ba77f0b'
 
         result = self.client.report_unihash(taskhash3, self.METHOD, outhash3, unihash3)
         self.assertClientGetHash(self.client, taskhash3, unihash3)
@@ -1014,14 +1071,14 @@ class HashEquivalenceCommonTests(object):
     def test_gc_switch_mark(self):
         taskhash = '53b8dce672cb6d0c73170be43f540460bfc347b4'
         outhash = '5a9cb1649625f0bf41fc7791b635cd9c2d7118c7f021ba87dcd03f72b67ce7a8'
-        unihash = 'f37918cc02eb5a520b1aff86faacbc0a38124646'
+        unihash = '46edb5140d2613049332d0bf3745d9fafec9c559dac8cc61813739a28007fcdf'
 
         result = self.client.report_unihash(taskhash, self.METHOD, outhash, unihash)
         self.assertEqual(result['unihash'], unihash, 'Server returned bad unihash')
 
         taskhash2 = '3bf6f1e89d26205aec90da04854fbdbf73afe6b4'
         outhash2 = '77623a549b5b1a31e3732dfa8fe61d7ce5d44b3370f253c5360e136b852967b4'
-        unihash2 = 'af36b199320e611fbb16f1f277d3ee1d619ca58b'
+        unihash2 = 'bf6e81926066f770e960f9f777cd088c62bea9addb7745f3e77deaa81a645747'
 
         result = self.client.report_unihash(taskhash2, self.METHOD, outhash2, unihash2)
         self.assertClientGetHash(self.client, taskhash2, unihash2)
@@ -1059,14 +1116,14 @@ class HashEquivalenceCommonTests(object):
     def test_gc_switch_sweep_mark(self):
         taskhash = '53b8dce672cb6d0c73170be43f540460bfc347b4'
         outhash = '5a9cb1649625f0bf41fc7791b635cd9c2d7118c7f021ba87dcd03f72b67ce7a8'
-        unihash = 'f37918cc02eb5a520b1aff86faacbc0a38124646'
+        unihash = '46edb5140d2613049332d0bf3745d9fafec9c559dac8cc61813739a28007fcdf'
 
         result = self.client.report_unihash(taskhash, self.METHOD, outhash, unihash)
         self.assertEqual(result['unihash'], unihash, 'Server returned bad unihash')
 
         taskhash2 = '3bf6f1e89d26205aec90da04854fbdbf73afe6b4'
         outhash2 = '77623a549b5b1a31e3732dfa8fe61d7ce5d44b3370f253c5360e136b852967b4'
-        unihash2 = 'af36b199320e611fbb16f1f277d3ee1d619ca58b'
+        unihash2 = 'bf6e81926066f770e960f9f777cd088c62bea9addb7745f3e77deaa81a645747'
 
         result = self.client.report_unihash(taskhash2, self.METHOD, outhash2, unihash2)
         self.assertClientGetHash(self.client, taskhash2, unihash2)
@@ -1089,7 +1146,7 @@ class HashEquivalenceCommonTests(object):
     def test_gc_new_hashes(self):
         taskhash = '53b8dce672cb6d0c73170be43f540460bfc347b4'
         outhash = '5a9cb1649625f0bf41fc7791b635cd9c2d7118c7f021ba87dcd03f72b67ce7a8'
-        unihash = 'f37918cc02eb5a520b1aff86faacbc0a38124646'
+        unihash = '46edb5140d2613049332d0bf3745d9fafec9c559dac8cc61813739a28007fcdf'
 
         result = self.client.report_unihash(taskhash, self.METHOD, outhash, unihash)
         self.assertEqual(result['unihash'], unihash, 'Server returned bad unihash')
@@ -1106,7 +1163,7 @@ class HashEquivalenceCommonTests(object):
 
         taskhash2 = '3bf6f1e89d26205aec90da04854fbdbf73afe6b4'
         outhash2 = '77623a549b5b1a31e3732dfa8fe61d7ce5d44b3370f253c5360e136b852967b4'
-        unihash2 = 'af36b199320e611fbb16f1f277d3ee1d619ca58b'
+        unihash2 = 'bf6e81926066f770e960f9f777cd088c62bea9addb7745f3e77deaa81a645747'
 
         result = self.client.report_unihash(taskhash2, self.METHOD, outhash2, unihash2)
         self.assertClientGetHash(self.client, taskhash2, unihash2)
@@ -1405,14 +1462,14 @@ class TestHashEquivalenceClient(HashEquivalenceTestSetup, unittest.TestCase):
     def test_gc(self):
         taskhash = '53b8dce672cb6d0c73170be43f540460bfc347b4'
         outhash = '5a9cb1649625f0bf41fc7791b635cd9c2d7118c7f021ba87dcd03f72b67ce7a8'
-        unihash = 'f37918cc02eb5a520b1aff86faacbc0a38124646'
+        unihash = '46edb5140d2613049332d0bf3745d9fafec9c559dac8cc61813739a28007fcdf'
 
         result = self.client.report_unihash(taskhash, self.METHOD, outhash, unihash)
         self.assertEqual(result['unihash'], unihash, 'Server returned bad unihash')
 
         taskhash2 = '3bf6f1e89d26205aec90da04854fbdbf73afe6b4'
         outhash2 = '77623a549b5b1a31e3732dfa8fe61d7ce5d44b3370f253c5360e136b852967b4'
-        unihash2 = 'af36b199320e611fbb16f1f277d3ee1d619ca58b'
+        unihash2 = 'bf6e81926066f770e960f9f777cd088c62bea9addb7745f3e77deaa81a645747'
 
         result = self.client.report_unihash(taskhash2, self.METHOD, outhash2, unihash2)
         self.assertClientGetHash(self.client, taskhash2, unihash2)
@@ -1455,7 +1512,7 @@ class TestHashEquivalenceUnixServerLongPath(HashEquivalenceTestSetup, unittest.T
         # Simple test that hashes can be created
         taskhash = '35788efcb8dfb0a02659d81cf2bfd695fb30faf9'
         outhash = '2765d4a5884be49b28601445c2760c5f21e7e5c0ee2b7e3fce98fd7e5970796f'
-        unihash = 'f46d3fbb439bd9b921095da657a4de906510d2cd'
+        unihash = 'a69ec97f5af2e21e1a1f9cc8896965515d5559425666f734e245a3d40cee33d9'
 
         self.assertClientGetHash(self.client, taskhash, None)
 
@@ -1542,4 +1599,3 @@ class TestHashEquivalenceExternalServer(HashEquivalenceTestSetup, HashEquivalenc
 
     def test_auth_get_all_users(self):
         self.skipTest("Cannot test all users with external server")
-
Signature · 1 line
2.34.1
#2 lore raw

PATCH 2/2 hashserv: validate unihash values

In reply to Anders Heimer
---
 lib/bb/siggen.py         |  9 +++++++-
 lib/bb/tests/siggen.py   | 47 ++++++++++++++++++++++++++++++++++++++++
 lib/hashserv/__init__.py |  7 ++++++
 lib/hashserv/server.py   | 27 ++++++++++++++++++-----
 lib/hashserv/tests.py    | 30 +++++++++++++++++++++++++
 5 files changed, 114 insertions(+), 6 deletions(-)
lib/bb/siggen.py +8 -1
index 985fa7e4c..3a203676e 100644
@@ -43,6 +43,10 @@ def check_siggen_version(siggen):
     if siggen.find_siginfo_version < siggen.find_siginfo_minversion:
         bb.fatal("Siggen from metadata (OE-Core?) is too old, please update it (%s vs %s)" % (siggen.find_siginfo_version, siggen.find_siginfo_minversion))
 
+def check_hashserv_unihash(unihash):
+    if not hashserv.is_valid_unihash(unihash):
+        bb.fatal("Hash Equivalence Server returned invalid unihash")
+
 class SetEncoder(json.JSONEncoder):
     def default(self, obj):
         if isinstance(obj, set) or isinstance(obj, frozenset):
@@ -729,6 +733,7 @@ class SignatureGeneratorUniHashMixIn(object):
 
             if unihashes and unihashes[idx]:
                 unihash = unihashes[idx]
+                check_hashserv_unihash(unihash)
                 # A unique hash equal to the taskhash is not very interesting,
                 # so it is reported it at debug level 2. If they differ, that
                 # is much more interesting, so it is reported at debug level 1
@@ -747,7 +752,7 @@ class SignatureGeneratorUniHashMixIn(object):
         import importlib
 
         taskhash = d.getVar('BB_TASKHASH')
-        unihash = d.getVar('BB_UNIHASH')
+        unihash = d.getVar('BB_UNIHASH', expand=False)
         report_taskdata = d.getVar('SSTATE_HASHEQUIV_REPORT_TASKDATA') == '1'
         tempdir = d.getVar('T')
         mcfn = d.getVar('BB_FILENAME')
@@ -809,6 +814,7 @@ class SignatureGeneratorUniHashMixIn(object):
                     data = client.report_unihash(taskhash, method, outhash, unihash, extra_data)
 
                 new_unihash = data['unihash']
+                check_hashserv_unihash(new_unihash)
 
                 if new_unihash != unihash:
                     hashequiv_logger.debug('Task %s unihash changed %s -> %s by server %s' % (taskhash, unihash, new_unihash, self.server))
@@ -848,6 +854,7 @@ class SignatureGeneratorUniHashMixIn(object):
                 return False
 
             finalunihash = data['unihash']
+            check_hashserv_unihash(finalunihash)
 
             if finalunihash == current_unihash:
                 hashequiv_logger.verbose('Task %s unihash %s unchanged by server' % (tid, finalunihash))
lib/bb/tests/siggen.py +47 -0

57 lines not on this page · raw message

lib/hashserv/__init__.py +7 -0
index ac891e017..ba8e0acce 100644
@@ -7,12 +7,19 @@ import asyncio
 from contextlib import closing
 import itertools
 import json
+import re
 from collections import namedtuple
 from urllib.parse import urlparse
 from bb.asyncrpc.client import parse_address, ADDR_TYPE_UNIX, ADDR_TYPE_WS
 
 User = namedtuple("User", ("username", "permissions"))
 
+UNIHASH_REGEX = re.compile(r"^[0-9a-f]{64}$")
+
+
+def is_valid_unihash(value):
+    return isinstance(value, str) and UNIHASH_REGEX.fullmatch(value) is not None
+
 
 def create_server(
     addr,
lib/hashserv/server.py +22 -5
index 58f95c7bc..3ff434785 100644
@@ -13,6 +13,7 @@ import base64
 import json
 import hashlib
 from . import create_async_client
+from . import is_valid_unihash
 import bb.asyncrpc
 
 logger = logging.getLogger("hashserv.server")
@@ -173,6 +174,11 @@ def hash_token(algo, salt, token):
     return ":".join([algo, salt, h.hexdigest()])
 
 
+def validate_unihash(value):
+    if not is_valid_unihash(value):
+        raise bb.asyncrpc.InvokeError("Invalid unihash")
+
+
 def permissions(*permissions, allow_anon=True, allow_self_service=False):
     """
     Function decorator that can be used to decorate an RPC function call and
@@ -345,7 +351,7 @@ class ServerClient(bb.asyncrpc.AsyncServerConnection):
                 d = {k: row[k] for k in row.keys()}
             elif self.upstream_client is not None:
                 d = await self.upstream_client.get_taskhash(method, taskhash)
-                await self.db.insert_unihash(d["method"], d["taskhash"], d["unihash"])
+                await self.insert_unihash(d["method"], d["taskhash"], d["unihash"])
 
         return d
 
@@ -377,9 +383,13 @@ class ServerClient(bb.asyncrpc.AsyncServerConnection):
         if data is None:
             return
 
-        await self.db.insert_unihash(data["method"], data["taskhash"], data["unihash"])
+        await self.insert_unihash(data["method"], data["taskhash"], data["unihash"])
         await self.db.insert_outhash(data)
 
+    async def insert_unihash(self, method, taskhash, unihash):
+        validate_unihash(unihash)
+        return await self.db.insert_unihash(method, taskhash, unihash)
+
     async def _stream_handler(self, handler):
         await self.socket.send_message("ok")
 
@@ -467,6 +477,8 @@ class ServerClient(bb.asyncrpc.AsyncServerConnection):
     # report is made inside the function
     @permissions(READ_PERM)
     async def handle_report(self, data):
+        validate_unihash(data.get("unihash"))
+
         if self.server.read_only or not self.user_has_permissions(REPORT_PERM):
             return await self.report_readonly(data)
 
@@ -509,7 +521,7 @@ class ServerClient(bb.asyncrpc.AsyncServerConnection):
                     if upstream_data is not None:
                         unihash = upstream_data["unihash"]
 
-            await self.db.insert_unihash(data["method"], data["taskhash"], unihash)
+            await self.insert_unihash(data["method"], data["taskhash"], unihash)
 
         unihash_data = await self.get_unihash(data["method"], data["taskhash"])
         if unihash_data is not None:
@@ -525,7 +537,9 @@ class ServerClient(bb.asyncrpc.AsyncServerConnection):
 
     @permissions(READ_PERM, REPORT_PERM)
     async def handle_equivreport(self, data):
-        await self.db.insert_unihash(data["method"], data["taskhash"], data["unihash"])
+        validate_unihash(data.get("unihash"))
+
+        await self.insert_unihash(data["method"], data["taskhash"], data["unihash"])
 
         # Fetch the unihash that will be reported for the taskhash. If the
         # unihash matches, it means this row was inserted (or the mapping
@@ -888,7 +902,10 @@ class Server(bb.asyncrpc.AsyncServer):
                 method, taskhash = item
                 d = await client.get_taskhash(method, taskhash)
                 if d is not None:
-                    await db.insert_unihash(d["method"], d["taskhash"], d["unihash"])
+                    if is_valid_unihash(d.get("unihash")):
+                        await db.insert_unihash(d["method"], d["taskhash"], d["unihash"])
+                    else:
+                        self.logger.warning("Upstream server returned invalid unihash")
                 self.backfill_queue.task_done()
 
     def start(self):
lib/hashserv/tests.py +30 -0
index 83ce0c5ca..c2ed1035a 100644
@@ -292,6 +292,36 @@ class HashEquivalenceCommonTests(object):
         self.assertEqual(result_outhash['outhash'], outhash)
         self.assertEqual(result_outhash['outhash_siginfo'], siginfo)
 
+    def test_report_rejects_invalid_unihash(self):
+        taskhash = '68a9206490b2321bb033fb3eab013a4ec62c41f9'
+        outhash = 'bf5f2efaf1ca351f3b4c3d079363540ab48f7c58db3d23cfbb069cf4ff1ea8f7'
+        invalid_unihashes = (
+            "${@os.system('true')}",
+            'a' * 63,
+            'a' * 65,
+            'A' * 64,
+            None,
+        )
+
+        for unihash in invalid_unihashes:
+            with self.subTest(unihash=unihash):
+                with self.start_client(self.server_address) as client:
+                    with self.assertRaises(InvokeError) as context:
+                        client.report_unihash(taskhash, self.METHOD, outhash, unihash)
+
+                self.assertEqual(str(context.exception), "Invalid unihash")
+
+        self.assertClientGetHash(self.client, taskhash, None)
+
+    def test_equivreport_rejects_invalid_unihash(self):
+        taskhash = 'ae6339531895ddf5b67e663e6a374ad8ec71d81c'
+
+        with self.assertRaises(InvokeError) as context:
+            self.client.report_unihash_equiv(taskhash, self.METHOD, "${@os.system('true')}")
+
+        self.assertEqual(str(context.exception), "Invalid unihash")
+        self.assertClientGetHash(self.start_client(self.server_address), taskhash, None)
+
     def test_stress(self):
         def query_server(failures):
             client = Client(self.server_address)
Signature · 1 line
2.34.1