hashserv: Pipeline Upstream Queries
cover hashserv: Pipeline Upstream Queries
Adds a true asynchronous streaming API to the hash equivalence clients, then uses that streaming API to pipeline requests to an upstream server. This means that when a server wants to defer a request to an upstream, it no longer has to wait for a complete round-trip API call. Instead, it can use the faster streaming API to send the requests, and continue processing other requests while a response inbound.
Joshua Watt (6):
hashserv: client: Add asynchronous streaming API
hashserv: server: Fix formatting
hashserv: server: Add queued streaming API
hashserv: server: Use streaming and queue API for upstream unihash
queries
hashserv: server: Use streaming and queue API for upstream exist
queries
hashserv: tests: Add test for upstream pipelining
bin/bitbake-hashclient | 2 +- lib/hashserv/client.py | 306 ++++++++++++++++++++++++++++++++--------- lib/hashserv/server.py | 191 +++++++++++++++++++------ lib/hashserv/tests.py | 105 +++++++++++++- 4 files changed, 492 insertions(+), 112 deletions(-)
Signature · 1 line
2.54.0
PATCH 1/6 hashserv: client: Add asynchronous streaming API
Implments a true asynchronous streaming API for getting a unihash (get_unihash_stream()) and checking if a unihash exists (unihash_exists_stream()). These APIs allow a client to send a query the to the server, then wait for the reply later. Using this API, it is possible to interleave queries and replies.
In addition, the gc_mark_stream() API is renamed gc_mark_batch() to match the pattern of the other APIs, and an actual gc_mark_stream() API is implemented that allows asynchronous streaming like the others.
Note that the new stream APIs are not accessible in the "synchronous" client API since async constructs must be used for them to make sense.
--- bin/bitbake-hashclient | 2 +- lib/hashserv/client.py | 306 ++++++++++++++++++++++++++++++++--------- lib/hashserv/tests.py | 4 +- 3 files changed, 243 insertions(+), 69 deletions(-)
bin/bitbake-hashclient +1 -1
@@ -240,7 +240,7 @@ def main(): marked_hashes = 0 try: - result = client.gc_mark_stream(args.mark, stdin) + result = client.gc_mark_batch(args.mark, stdin) marked_hashes = result["count"] except ConnectionError: logger.warning(
lib/hashserv/client.py +240 -66
389 lines not on this page · raw message
lib/hashserv/tests.py +2 -2
@@ -1055,7 +1055,7 @@ class HashEquivalenceCommonTests(object): # First hash is still present self.assertClientGetHash(self.client, taskhash, unihash) - def test_gc_stream(self): + def test_gc_batch(self): taskhash = '53b8dce672cb6d0c73170be43f540460bfc347b4' outhash = '5a9cb1649625f0bf41fc7791b635cd9c2d7118c7f021ba87dcd03f72b67ce7a8' unihash = '46edb5140d2613049332d0bf3745d9fafec9c559dac8cc61813739a28007fcdf'
@@ -1078,7 +1078,7 @@ class HashEquivalenceCommonTests(object): self.assertClientGetHash(self.client, taskhash3, unihash3) # Mark the first unihash to be kept - ret = self.client.gc_mark_stream("ABC", (f"unihash {h}" for h in [unihash, unihash2])) + ret = self.client.gc_mark_batch("ABC", (f"unihash {h}" for h in [unihash, unihash2])) self.assertEqual(ret, {"count": 2}) ret = self.client.gc_status()
Signature · 1 line
2.54.0
PATCH 2/6 hashserv: server: Fix formatting
In reply to Joshua WattReformats the file with black
--- lib/hashserv/server.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-)
lib/hashserv/server.py +5 -3
@@ -424,7 +424,7 @@ class ServerClient(bb.asyncrpc.AsyncServerConnection): @permissions(READ_PERM) async def handle_get_stream(self, request): async def handler(l): - (method, taskhash) = l.split() + method, taskhash = l.split() # self.logger.debug('Looking up %s %s' % (method, taskhash)) row = await self.db.get_equivalent(method, taskhash)
@@ -646,7 +646,7 @@ class ServerClient(bb.asyncrpc.AsyncServerConnection): @permissions(DB_ADMIN_PERM) async def handle_gc_status(self, request): - (keep_rows, remove_rows, current_mark) = await self.db.gc_status() + keep_rows, remove_rows, current_mark = await self.db.gc_status() return { "keep": keep_rows, "remove": remove_rows,
@@ -903,7 +903,9 @@ class Server(bb.asyncrpc.AsyncServer): d = await client.get_taskhash(method, taskhash) if d is not None: if is_valid_unihash(d.get("unihash")): - await db.insert_unihash(d["method"], d["taskhash"], d["unihash"]) + await db.insert_unihash( + d["method"], d["taskhash"], d["unihash"] + ) else: self.logger.warning("Upstream server returned invalid unihash") self.backfill_queue.task_done()
Signature · 1 line
2.54.0
PATCH 3/6 hashserv: server: Add queued streaming API
In reply to Joshua WattAdds an API that allows a stream handler to more precisely control when a response is sent to the client. The new API does not directly send the response from the handler to the remote client, but instead the handler is expected to put the result in a provided queue when the response is ready. In particular, this allows a stream handler to defer to an upstream server (utilizing the new client streaming API) in an efficient way that does not require waiting on a roundtrip with the upstream server. Instead, several queries to the upstream can be in-flight at once.
--- lib/hashserv/server.py | 100 ++++++++++++++++++++++++++++++++--------- 1 file changed, 79 insertions(+), 21 deletions(-)
lib/hashserv/server.py +79 -21
@@ -229,6 +229,58 @@ def permissions(*permissions, allow_anon=True, allow_self_service=False): return wrapper +class UpstreamQueue(object): + UPSTREAM_NONCE = object() + + def __init__(self, queue, get_local_result, send_upstream, get_upstream_result): + self.queue = queue + self.pending = [] + self.cond = asyncio.Condition() + self.done = False + self.get_local_result = get_local_result + self.send_upstream = send_upstream + self.get_upstream_result = get_upstream_result + + async def process_results(self): + try: + while True: + async with self.cond: + await self.cond.wait_for(lambda: self.pending or self.done) + if not self.pending: + if self.done: + return + continue + + value, m = self.pending.pop(0) + + if value is self.UPSTREAM_NONCE: + value = await self.get_upstream_result(m) + + await self.queue.put(value) + finally: + await self.queue.put(None) + + async def handler(self, m): + try: + if m is None: + return + + value = await self.get_local_result(m) + if value is None: + await self.send_upstream(m) + value = self.UPSTREAM_NONCE + + async with self.cond: + self.pending.append((value, m)) + self.cond.notify_all() + + finally: + async with self.cond: + self.done = True + self.cond.notify_all() + # await stream.done() + + class ServerClient(bb.asyncrpc.AsyncServerConnection): def __init__(self, socket, server): super().__init__(socket, "OEHASHEQUIV", server.logger)
@@ -390,35 +442,41 @@ class ServerClient(bb.asyncrpc.AsyncServerConnection): validate_unihash(unihash) return await self.db.insert_unihash(method, taskhash, unihash) - async def _stream_handler(self, handler): + async def _stream_queue_handler(self, handler, queue): await self.socket.send_message("ok") - while True: - upstream = None + async def recv(): + try: + while True: + m = await self.socket.recv() + if not m or m == "END": + break - l = await self.socket.recv() - if not l: - break + await handler(m) + finally: + await handler(None) - try: - # This inner loop is very sensitive and must be as fast as - # possible (which is why the request sample is handled manually - # instead of using 'with', and also why logging statements are - # commented out. - self.request_sample = self.server.request_stats.start_sample() - request_measure = self.request_sample.measure() - request_measure.start() - - if l == "END": + async def process(): + while True: + m = await queue.get() + if m is None: break - msg = await handler(l) - await self.socket.send(msg) - finally: - request_measure.end() - self.request_sample.end() + await self.socket.send(m) + await asyncio.gather(recv(), process()) await self.socket.send("ok") + + async def _stream_handler(self, handler): + queue = asyncio.Queue(1000) + + async def h(m): + if m is None: + await queue.put(None) + else: + await queue.put(await handler(m)) + + await self._stream_queue_handler(h, queue) return self.NO_RESPONSE @permissions(READ_PERM)
Signature · 1 line
2.54.0
PATCH 4/6 hashserv: server: Use streaming and queue API for upstream unihash queries
In reply to Joshua WattReworks the "get-unihash" handler to use the new server queue API and client streaming API to efficiently stream requests to the upstream server instead of having to wait for a roundtrip on the requests.
--- lib/hashserv/server.py | 52 ++++++++++++++++++++++++++++++------------ 1 file changed, 38 insertions(+), 14 deletions(-)
lib/hashserv/server.py +38 -14
@@ -481,24 +481,48 @@ class ServerClient(bb.asyncrpc.AsyncServerConnection): @permissions(READ_PERM) async def handle_get_stream(self, request): - async def handler(l): - method, taskhash = l.split() - # self.logger.debug('Looking up %s %s' % (method, taskhash)) - row = await self.db.get_equivalent(method, taskhash) - - if row is not None: - # self.logger.debug('Found equivalent task %s -> %s', (row['taskhash'], row['unihash'])) + async def get_unihash(m): + method, taskhash = m.split() + if (row := await self.db.get_equivalent(method, taskhash)) is not None: return row["unihash"] - if self.upstream_client is not None: - upstream = await self.upstream_client.get_unihash(method, taskhash) - if upstream: - await self.server.backfill_queue.put((method, taskhash)) - return upstream - return "" - return await self._stream_handler(handler) + if not self.upstream_client: + return await self._stream_handler(get_unihash) + + async with self.upstream_client.get_unihash_stream() as stream: + + async def get_local_result(m): + method, taskhash = m.split() + if (row := await self.db.get_equivalent(method, taskhash)) is not None: + return row["unihash"] + return None + + async def send_upstream(m): + method, taskhash = m.split() + await stream.send_query(method, taskhash) + + async def get_upstream_result(m): + unihash = await stream.get_result() + if unihash: + method, taskhash = m.split() + await self.server.backfill_queue.put((method, taskhash)) + return unihash + + queue = asyncio.Queue() + upstream = UpstreamQueue( + queue, + get_local_result, + send_upstream, + get_upstream_result, + ) + + await asyncio.gather( + self._stream_queue_handler(upstream.handler, queue), + upstream.process_results(), + ) + return self.NO_RESPONSE @permissions(READ_PERM) async def handle_exists_stream(self, request):
Signature · 1 line
2.54.0
PATCH 5/6 hashserv: server: Use streaming and queue API for upstream exist queries
In reply to Joshua WattReworks the "unihash-exists" handler to use the new server queue API and client streaming API to efficiently stream requests to the upstream server instead of having to wait for a roundtrip on the requests.
--- lib/hashserv/server.py | 31 ++++++++++++++++++++++++++----- lib/hashserv/tests.py | 5 +++++ 2 files changed, 31 insertions(+), 5 deletions(-)
lib/hashserv/server.py +26 -5
@@ -526,17 +526,38 @@ class ServerClient(bb.asyncrpc.AsyncServerConnection): @permissions(READ_PERM) async def handle_exists_stream(self, request): - async def handler(l): + async def exists_handler(l): if await self.db.unihash_exists(l): return "true" + return "false" + + if not self.upstream_client: + return await self._stream_handler(exists_handler) - if self.upstream_client is not None: - if await self.upstream_client.unihash_exists(l): + async with self.upstream_client.unihash_exists_stream() as stream: + + async def get_local_result(m): + if await self.db.unihash_exists(m): return "true" + return None - return "false" + async def get_upstream_result(m): + exists = await stream.get_result() + return "true" if exists else "false" - return await self._stream_handler(handler) + queue = asyncio.Queue() + upstream = UpstreamQueue( + queue, + get_local_result, + stream.send_query, + get_upstream_result, + ) + + await asyncio.gather( + self._stream_queue_handler(upstream.handler, queue), + upstream.process_results(), + ) + return self.NO_RESPONSE async def report_readonly(self, data): method = data["method"]
lib/hashserv/tests.py +5 -0
@@ -372,20 +372,25 @@ class HashEquivalenceCommonTests(object): nonlocal side_client # check upstream server + self.assertTrue(self.client.unihash_exists(unihash)) self.assertClientGetHash(self.client, taskhash, unihash) # Hash should *not* be present on the side server + if old_sidehash and unihash != old_sidehash: + self.assertFalse(side_client.unihash_exists(unihash)) self.assertClientGetHash(side_client, taskhash, old_sidehash) # Hash should be present on the downstream server, since it # will defer to the upstream server. This will trigger # the backfill in the downstream server + self.assertTrue(down_client.unihash_exists(unihash)) self.assertClientGetHash(down_client, taskhash, unihash) # After waiting for the downstream client to finish backfilling the # task from the upstream server, it should appear in the side server # since the database is populated down_client.backfill_wait() + self.assertTrue(side_client.unihash_exists(unihash)) self.assertClientGetHash(side_client, taskhash, unihash) # Basic report
Signature · 1 line
2.54.0
PATCH 6/6 hashserv: tests: Add test for upstream pipelining
In reply to Joshua WattAdds a test that verifies that pipelining to an upstream server works as expected.
AI-Generated: Uses Cursor, Claude Sonnet 5
--- lib/hashserv/tests.py | 96 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 96 insertions(+)
lib/hashserv/tests.py +96 -0
@@ -1561,6 +1561,102 @@ class TestHashEquivalenceTCPServer(HashEquivalenceTestSetup, HashEquivalenceComm # case it is more reliable to resolve the IP address explicitly. return socket.gethostbyname("localhost") + ":0" + def test_get_stream_upstream_pipelined(self): + # Verify that, with an upstream configured, the get-stream handler + # pipelines its upstream queries instead of doing one blocking + # round-trip per task. A latency proxy injects RTT between this server + # and its upstream; a serial (one round-trip per task) implementation + # could not beat N * RTT, so finishing far faster proves the queries + # are pipelined. + import asyncio + + LATENCY = 0.01 # 10ms each direction => ~20ms round trip + upstream_host, upstream_port = self.server_address.rsplit(":", 1) + upstream_port = int(upstream_port) + + ready = threading.Event() + proxy = {} + + def run_proxy(): + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + + async def handle(creader, cwriter): + ureader, uwriter = await asyncio.open_connection(upstream_host, upstream_port) + + async def pipe(r, w): + try: + while True: + data = await r.read(65536) + if not data: + break + await asyncio.sleep(LATENCY) + w.write(data) + await w.drain() + except Exception: + pass + finally: + try: + w.close() + except Exception: + pass + + await asyncio.gather(pipe(creader, uwriter), pipe(ureader, cwriter)) + + stop_event = asyncio.Event() + proxy["stop_event"] = stop_event + proxy["loop"] = loop + + async def main(): + server = await asyncio.start_server(handle, "127.0.0.1", 0) + proxy["port"] = server.sockets[0].getsockname()[1] + ready.set() + async with server: + await stop_event.wait() + + try: + loop.run_until_complete(main()) + except Exception: + pass + finally: + loop.close() + + proxy_thread = threading.Thread(target=run_proxy, daemon=True) + proxy_thread.start() + self.assertTrue(ready.wait(10), "latency proxy did not start") + self.addCleanup(proxy_thread.join, 10) + self.addCleanup(lambda: proxy["loop"].call_soon_threadsafe(proxy["stop_event"].set)) + proxy_addr = "127.0.0.1:%d" % proxy["port"] + + N = 400 + expected = [] + for i in range(N): + taskhash = hashlib.sha256(("task%d" % i).encode()).hexdigest() + outhash = hashlib.sha256(("out%d" % i).encode()).hexdigest() + unihash = hashlib.sha256(("uni%d" % i).encode()).hexdigest() + self.client.report_unihash(taskhash, self.METHOD, outhash, unihash) + expected.append((self.METHOD, taskhash, unihash)) + + # Downstream server with an EMPTY local DB whose upstream is the slow proxy. + down_server = self.start_server(upstream=proxy_addr) + down_client = self.start_client(down_server.address) + + args = [(m, th) for (m, th, _uh) in expected] + start = time.time() + results = down_client.get_unihash_batch(args) + elapsed = time.time() - start + + self.assertEqual(results, [uh for (_m, _th, uh) in expected]) + + # A serial (one round-trip per task) implementation cannot beat N*RTT. + # Allow a generous margin to avoid flakiness on loaded CI machines. + serial_lower_bound = N * 2 * LATENCY + self.assertLess(elapsed, serial_lower_bound / 4, + "Upstream queries are not being pipelined " + "(%.3fs for %d queries at %.0fms injected RTT)" + % (elapsed, N, 2 * LATENCY * 1000)) + + class TestHashEquivalenceWebsocketServer(HashEquivalenceTestSetup, HashEquivalenceCommonTests, unittest.TestCase): def setUp(self):
Signature · 1 line
2.54.0
Hi Joshua,
Quoted 7 lines · Joshua Watt via lists.openembedded.org
Adds a true asynchronous streaming API to the hash equivalence clients, then uses that streaming API to pipeline requests to an upstream server. This means that when a server wants to defer a request to an upstream, it no longer has to wait for a complete round-trip API call. Instead, it can use the faster streaming API to send the requests, and continue processing other requests while a response inbound.
Quoted 9 lines
Joshua Watt (6):
hashserv: client: Add asynchronous streaming API
hashserv: server: Fix formatting
hashserv: server: Add queued streaming API
hashserv: server: Use streaming and queue API for upstream unihash queries
hashserv: server: Use streaming and queue API for upstream exist queries
hashserv: tests: Add test for upstream pipelining
Thanks, this looks promising but builds didn't like it:
https://autobuilder.yoctoproject.org/valkyrie/#/builders/22/builds/4225
(and more in that a-full).
Cheers,
Richard
v2 cover hashserv: Pipeline Upstream Queries
In reply to Joshua WattAdds a true asynchronous streaming API to the hash equivalence clients, then uses that streaming API to pipeline requests to an upstream server. This means that when a server wants to defer a request to an upstream, it no longer has to wait for a complete round-trip API call. Instead, it can use the faster streaming API to send the requests, and continue processing other requests while a response inbound.
V2: Fixes several bugs related to when the connection between the client and upstream server was disconnected, and added tests for them.
Joshua Watt (10):
asyncrpc: Add Task Group
asyncrpc: serv: Use Task Group
asyncrpc: serv: Cancel all clients on server stop
hashserv: tests: Improve test logging
hashserv: client: Add asynchronous streaming API
hashserv: server: Add queued streaming API
hashserv: server: Use streaming and queue API for upstream unihash
queries
hashserv: server: Use streaming and queue API for upstream exist
queries
hashserv: tests: Add more upstream tests
hashserv: tests: Add test for upstream pipelining
bin/bitbake-hashclient | 2 +- lib/bb/asyncrpc/__init__.py | 1 + lib/bb/asyncrpc/serv.py | 13 +- lib/bb/asyncrpc/taskgroup.py | 52 ++++++ lib/hashserv/client.py | 349 +++++++++++++++++++++++++++-------- lib/hashserv/server.py | 181 ++++++++++++++---- lib/hashserv/tests.py | 261 ++++++++++++++++++++++++-- 7 files changed, 723 insertions(+), 136 deletions(-) create mode 100644 lib/bb/asyncrpc/taskgroup.py
Signature · 1 line
2.54.0
PATCH v2 2/10 asyncrpc: serv: Use Task Group
Uses a task group instead of asyncio.gather(). The task group ensures that all tasks are canceled if an exception occurs, which asyncio.gather() does not.
--- lib/bb/asyncrpc/serv.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
lib/bb/asyncrpc/serv.py +1 -1
@@ -334,7 +334,7 @@ class AsyncServer(object): self.loop.add_signal_handler(signal.SIGQUIT, self.signal_handler) signal.pthread_sigmask(signal.SIG_UNBLOCK, [signal.SIGTERM]) - self.loop.run_until_complete(asyncio.gather(*tasks)) + self.loop.run_until_complete(bb.asyncrpc.TaskGroup.run(*tasks)) self.logger.debug("Server shutting down") finally:
Signature · 1 line
2.54.0
PATCH v2 3/10 asyncrpc: serv: Cancel all clients on server stop
In reply to Joshua WattWhen the server is stopped, all clients should be stopped, otherwise the server will hang waiting for a disconnect.
--- lib/bb/asyncrpc/serv.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-)
lib/bb/asyncrpc/serv.py +9 -2
@@ -56,7 +56,7 @@ class AsyncServerConnection(object): if not client_protocol: return - (client_proto_name, client_proto_version) = client_protocol.split() + client_proto_name, client_proto_version = client_protocol.split() if client_proto_name != self.proto_name: self.logger.debug("Rejecting invalid protocol %s" % (self.proto_name)) return
@@ -123,6 +123,7 @@ class StreamServer(object): self.handler = handler self.logger = logger self.closed = False + self.clients = [] async def handle_stream_client(self, reader, writer): # writer.transport.set_write_buffer_limits(0)
@@ -131,10 +132,16 @@ class StreamServer(object): await socket.close() return - await self.handler(socket) + self.clients.append(socket) + try: + await self.handler(socket) + finally: + self.clients.remove(socket) async def stop(self): self.closed = True + for socket in self.clients: + await socket.close() class TCPStreamServer(StreamServer):
Signature · 1 line
2.54.0
PATCH v2 5/10 hashserv: client: Add asynchronous streaming API
In reply to Joshua WattImplments a true asynchronous streaming API for getting a unihash (get_unihash_stream()) and checking if a unihash exists (unihash_exists_stream()). These APIs allow a client to send a query the to the server, then wait for the reply later. Using this API, it is possible to interleave queries and replies.
In addition, the gc_mark_stream() API is renamed gc_mark_batch() to match the pattern of the other APIs, and an actual gc_mark_stream() API is implemented that allows asynchronous streaming like the others.
Note that the new stream APIs are not accessible in the "synchronous" client API since async constructs must be used for them to make sense.
--- bin/bitbake-hashclient | 2 +- lib/hashserv/client.py | 349 ++++++++++++++++++++++++++++++++--------- lib/hashserv/tests.py | 4 +- 3 files changed, 275 insertions(+), 80 deletions(-)
bin/bitbake-hashclient +1 -1
@@ -240,7 +240,7 @@ def main(): marked_hashes = 0 try: - result = client.gc_mark_stream(args.mark, stdin) + result = client.gc_mark_batch(args.mark, stdin) marked_hashes = result["count"] except ConnectionError: logger.warning(
lib/hashserv/client.py +272 -77
428 lines not on this page · raw message
lib/hashserv/tests.py +2 -2
@@ -1057,7 +1057,7 @@ class HashEquivalenceCommonTests(object): # First hash is still present self.assertClientGetHash(self.client, taskhash, unihash) - def test_gc_stream(self): + def test_gc_batch(self): taskhash = '53b8dce672cb6d0c73170be43f540460bfc347b4' outhash = '5a9cb1649625f0bf41fc7791b635cd9c2d7118c7f021ba87dcd03f72b67ce7a8' unihash = '46edb5140d2613049332d0bf3745d9fafec9c559dac8cc61813739a28007fcdf'
@@ -1080,7 +1080,7 @@ class HashEquivalenceCommonTests(object): self.assertClientGetHash(self.client, taskhash3, unihash3) # Mark the first unihash to be kept - ret = self.client.gc_mark_stream("ABC", (f"unihash {h}" for h in [unihash, unihash2])) + ret = self.client.gc_mark_batch("ABC", (f"unihash {h}" for h in [unihash, unihash2])) self.assertEqual(ret, {"count": 2}) ret = self.client.gc_status()
Signature · 1 line
2.54.0
PATCH v2 6/10 hashserv: server: Add queued streaming API
In reply to Joshua WattAdds an API that allows a stream handler to more precisely control when a response is sent to the client. The new API does not directly send the response from the handler to the remote client, but instead the handler is expected to put the result in a provided queue when the response is ready. In particular, this allows a stream handler to defer to an upstream server (utilizing the new client streaming API) in an efficient way that does not require waiting on a roundtrip with the upstream server. Instead, several queries to the upstream can be in-flight at once.
--- lib/hashserv/server.py | 96 +++++++++++++++++++++++++++++++++--------- 1 file changed, 75 insertions(+), 21 deletions(-)
lib/hashserv/server.py +75 -21
@@ -229,6 +229,54 @@ def permissions(*permissions, allow_anon=True, allow_self_service=False): return wrapper +class UpstreamQueue(object): + UPSTREAM_NONCE = object() + + def __init__(self, queue, get_local_result, send_upstream, get_upstream_result): + self.queue = queue + self.pending = [] + self.cond = asyncio.Condition() + self.done = False + self.get_local_result = get_local_result + self.send_upstream = send_upstream + self.get_upstream_result = get_upstream_result + + async def process_results(self): + try: + while True: + async with self.cond: + await self.cond.wait_for(lambda: self.pending or self.done) + if not self.pending: + if self.done: + return + continue + + value, m = self.pending.pop(0) + + if value is self.UPSTREAM_NONCE: + value = await self.get_upstream_result(m) + + await self.queue.put(value) + finally: + await self.queue.put(None) + + async def handler(self, m): + if m is None: + async with self.cond: + self.done = True + self.cond.notify_all() + return + + value = await self.get_local_result(m) + if value is None: + await self.send_upstream(m) + value = self.UPSTREAM_NONCE + + async with self.cond: + self.pending.append((value, m)) + self.cond.notify_all() + + class ServerClient(bb.asyncrpc.AsyncServerConnection): def __init__(self, socket, server): super().__init__(socket, "OEHASHEQUIV", server.logger)
@@ -390,35 +438,41 @@ class ServerClient(bb.asyncrpc.AsyncServerConnection): validate_unihash(unihash) return await self.db.insert_unihash(method, taskhash, unihash) - async def _stream_handler(self, handler): + async def _stream_queue_handler(self, handler, queue): await self.socket.send_message("ok") - while True: - upstream = None + async def recv(): + try: + while True: + m = await self.socket.recv() + if not m or m == "END": + break - l = await self.socket.recv() - if not l: - break + await handler(m) + finally: + await handler(None) - try: - # This inner loop is very sensitive and must be as fast as - # possible (which is why the request sample is handled manually - # instead of using 'with', and also why logging statements are - # commented out. - self.request_sample = self.server.request_stats.start_sample() - request_measure = self.request_sample.measure() - request_measure.start() - - if l == "END": + async def process(): + while True: + m = await queue.get() + if m is None: break - msg = await handler(l) - await self.socket.send(msg) - finally: - request_measure.end() - self.request_sample.end() + await self.socket.send(m) + await bb.asyncrpc.TaskGroup.run(recv(), process()) await self.socket.send("ok") + + async def _stream_handler(self, handler): + queue = asyncio.Queue(1000) + + async def h(m): + if m is None: + await queue.put(None) + else: + await queue.put(await handler(m)) + + await self._stream_queue_handler(h, queue) return self.NO_RESPONSE @permissions(READ_PERM)
Signature · 1 line
2.54.0
PATCH v2 7/10 hashserv: server: Use streaming and queue API for upstream unihash queries
In reply to Joshua WattReworks the "get-unihash" handler to use the new server queue API and client streaming API to efficiently stream requests to the upstream server instead of having to wait for a roundtrip on the requests.
--- lib/hashserv/server.py | 52 ++++++++++++++++++++++++++++++------------ 1 file changed, 38 insertions(+), 14 deletions(-)
lib/hashserv/server.py +38 -14
@@ -477,24 +477,48 @@ class ServerClient(bb.asyncrpc.AsyncServerConnection): @permissions(READ_PERM) async def handle_get_stream(self, request): - async def handler(l): - method, taskhash = l.split() - # self.logger.debug('Looking up %s %s' % (method, taskhash)) - row = await self.db.get_equivalent(method, taskhash) - - if row is not None: - # self.logger.debug('Found equivalent task %s -> %s', (row['taskhash'], row['unihash'])) + async def get_unihash(m): + method, taskhash = m.split() + if (row := await self.db.get_equivalent(method, taskhash)) is not None: return row["unihash"] - if self.upstream_client is not None: - upstream = await self.upstream_client.get_unihash(method, taskhash) - if upstream: - await self.server.backfill_queue.put((method, taskhash)) - return upstream - return "" - return await self._stream_handler(handler) + if not self.upstream_client: + return await self._stream_handler(get_unihash) + + async with self.upstream_client.get_unihash_stream() as stream: + + async def get_local_result(m): + method, taskhash = m.split() + if (row := await self.db.get_equivalent(method, taskhash)) is not None: + return row["unihash"] + return None + + async def send_upstream(m): + method, taskhash = m.split() + await stream.send_query(method, taskhash) + + async def get_upstream_result(m): + unihash = await stream.get_result() + if unihash: + method, taskhash = m.split() + await self.server.backfill_queue.put((method, taskhash)) + return unihash + + queue = asyncio.Queue() + upstream = UpstreamQueue( + queue, + get_local_result, + send_upstream, + get_upstream_result, + ) + + await bb.asyncrpc.TaskGroup.run( + self._stream_queue_handler(upstream.handler, queue), + upstream.process_results(), + ) + return self.NO_RESPONSE @permissions(READ_PERM) async def handle_exists_stream(self, request):
Signature · 1 line
2.54.0
PATCH v2 8/10 hashserv: server: Use streaming and queue API for upstream exist queries
In reply to Joshua WattReworks the "unihash-exists" handler to use the new server queue API and client streaming API to efficiently stream requests to the upstream server instead of having to wait for a roundtrip on the requests.
--- lib/hashserv/server.py | 31 ++++++++++++++++++++++++++----- lib/hashserv/tests.py | 5 +++++ 2 files changed, 31 insertions(+), 5 deletions(-)
lib/hashserv/server.py +26 -5
@@ -522,17 +522,38 @@ class ServerClient(bb.asyncrpc.AsyncServerConnection): @permissions(READ_PERM) async def handle_exists_stream(self, request): - async def handler(l): + async def exists_handler(l): if await self.db.unihash_exists(l): return "true" + return "false" + + if not self.upstream_client: + return await self._stream_handler(exists_handler) - if self.upstream_client is not None: - if await self.upstream_client.unihash_exists(l): + async with self.upstream_client.unihash_exists_stream() as stream: + + async def get_local_result(m): + if await self.db.unihash_exists(m): return "true" + return None - return "false" + async def get_upstream_result(m): + exists = await stream.get_result() + return "true" if exists else "false" - return await self._stream_handler(handler) + queue = asyncio.Queue() + upstream = UpstreamQueue( + queue, + get_local_result, + stream.send_query, + get_upstream_result, + ) + + await bb.asyncrpc.TaskGroup.run( + self._stream_queue_handler(upstream.handler, queue), + upstream.process_results(), + ) + return self.NO_RESPONSE async def report_readonly(self, data): method = data["method"]
lib/hashserv/tests.py +5 -0
@@ -374,20 +374,25 @@ class HashEquivalenceCommonTests(object): nonlocal side_client # check upstream server + self.assertTrue(self.client.unihash_exists(unihash)) self.assertClientGetHash(self.client, taskhash, unihash) # Hash should *not* be present on the side server + if old_sidehash and unihash != old_sidehash: + self.assertFalse(side_client.unihash_exists(unihash)) self.assertClientGetHash(side_client, taskhash, old_sidehash) # Hash should be present on the downstream server, since it # will defer to the upstream server. This will trigger # the backfill in the downstream server + self.assertTrue(down_client.unihash_exists(unihash)) self.assertClientGetHash(down_client, taskhash, unihash) # After waiting for the downstream client to finish backfilling the # task from the upstream server, it should appear in the side server # since the database is populated down_client.backfill_wait() + self.assertTrue(side_client.unihash_exists(unihash)) self.assertClientGetHash(side_client, taskhash, unihash) # Basic report
Signature · 1 line
2.54.0
PATCH v2 9/10 hashserv: tests: Add more upstream tests
In reply to Joshua WattAdds a test to verify that the batch API works properly with an upstream server and also a test to verify that if the upstream server restarts or is temporarily disconnected that the client recovers properly.
--- lib/hashserv/tests.py | 152 +++++++++++++++++++++++++++++++++++++++--- 1 file changed, 141 insertions(+), 11 deletions(-)
lib/hashserv/tests.py +141 -11
@@ -5,12 +5,13 @@ # SPDX-License-Identifier: GPL-2.0-only # -from . import create_server, create_client +from . import create_server, create_client, create_async_client from .server import DEFAULT_ANON_PERMS, ALL_PERMISSIONS from bb.asyncrpc import InvokeError import hashlib import logging from bb import multiprocessing +import asyncio import os import sys import tempfile
@@ -41,19 +42,15 @@ class HashEquivalenceTestSetup(object): server_index = 0 client_index = 0 - def start_server(self, dbpath=None, upstream=None, read_only=False, prefunc=server_prefunc, anon_perms=DEFAULT_ANON_PERMS, admin_username=None, admin_password=None): + def start_server(self, dbpath=None, upstream=None, read_only=False, prefunc=server_prefunc, anon_perms=DEFAULT_ANON_PERMS, admin_username=None, admin_password=None, addr=None): self.server_index += 1 + if addr is None: + addr = self.get_server_addr(self.server_index) + if dbpath is None: dbpath = self.make_dbpath() - def cleanup_server(server): - if server.process.exitcode is not None: - return - - server.process.terminate() - server.process.join() - - server = create_server(self.get_server_addr(self.server_index), + server = create_server(addr, dbpath, upstream=upstream, read_only=read_only,
@@ -63,7 +60,7 @@ class HashEquivalenceTestSetup(object): server.dbpath = dbpath server.serve_as_process(prefunc=prefunc, args=(self.server_index,)) - self.addCleanup(cleanup_server, server) + self.addCleanup(self.stop_server, server) return server
@@ -83,6 +80,13 @@ class HashEquivalenceTestSetup(object): self.server = self.start_server() return self.server.address + def stop_server(self, server): + if not server or server.process.exitcode is not None: + return + + server.process.terminate() + server.process.join() + def start_auth_server(self): auth_server = self.start_server(self.server.dbpath, anon_perms=[], admin_username="admin", admin_password="password") self.auth_server_address = auth_server.address
@@ -476,6 +480,132 @@ class HashEquivalenceCommonTests(object): self.assertEqual(result['taskhash'], taskhash9, 'Server failed to copy unihash from upstream') self.assertEqual(result['method'], self.METHOD) + def test_upstream_batch(self): + down_server = self.start_server(upstream=self.server.address) + down_client = self.start_client(down_server.address) + + taskhash1 = '8aa96fcffb5831b3c2c0cb75f0431e3f8b20554a' + outhash1 = 'afe240a439959ce86f5e322f8c208e1fedefea9e813f2140c81af866cc9edf7e' + unihash1 = '5b521d8a12683086cc08bc2c6d94a7a2dcff17eba53b9911e145d51164689380' + self.client.report_unihash(taskhash1, self.METHOD, outhash1, unihash1) + + taskhash2 = "e3da00593d6a7fb435c7e2114976c59c5fd6d561" + outhash2 = "1cf8713e645f491eb9c959d20b5cae1c47133a292626dda9b10709857cbe688a" + unihash2 = "7aebef07d66a8c0f92d0c4f65ec8b1fbb850a3693c53827b8774b64fa9a8a9fe" + self.client.report_unihash(taskhash2, self.METHOD, outhash2, unihash2) + + taskhash3 = '35788efcb8dfb0a02659d81cf2bfd695fb30faf9' + outhash3 = '2765d4a5884be49b28601445c2760c5f21e7e5c0ee2b7e3fce98fd7e5970796f' + unihash3 = 'a69ec97f5af2e21e1a1f9cc8896965515d5559425666f734e245a3d40cee33d9' + self.client.report_unihash(taskhash3, self.METHOD, outhash3, unihash3) + + def query_generator(): + yield unihash1 + yield unihash2 + yield unihash3 + yield "cc74784b2c0ad5b378a6b783c74c518d2c46b8b52fba29cb39a8430d742440d7" + + results = down_client.unihash_exists_batch(query_generator()) + self.assertEqual(results, [True, True, True, False]) + + def test_upstream_interrupted(self): + up_server = self.start_server() + down_server = self.start_server(upstream=up_server.address) + + def restart_upstream(): + nonlocal up_server + + self.stop_server(up_server) + up_server = self.start_server(addr=up_server.address, dbpath=up_server.dbpath) + + # Report some hashes + with self.start_client(up_server.address) as up_client: + taskhash1 = '8aa96fcffb5831b3c2c0cb75f0431e3f8b20554a' + outhash1 = 'afe240a439959ce86f5e322f8c208e1fedefea9e813f2140c81af866cc9edf7e' + unihash1 = '5b521d8a12683086cc08bc2c6d94a7a2dcff17eba53b9911e145d51164689380' + up_client.report_unihash(taskhash1, self.METHOD, outhash1, unihash1) + + taskhash2 = "e3da00593d6a7fb435c7e2114976c59c5fd6d561" + outhash2 = "1cf8713e645f491eb9c959d20b5cae1c47133a292626dda9b10709857cbe688a" + unihash2 = "7aebef07d66a8c0f92d0c4f65ec8b1fbb850a3693c53827b8774b64fa9a8a9fe" + up_client.report_unihash(taskhash2, self.METHOD, outhash2, unihash2) + + taskhash3 = '35788efcb8dfb0a02659d81cf2bfd695fb30faf9' + outhash3 = '2765d4a5884be49b28601445c2760c5f21e7e5c0ee2b7e3fce98fd7e5970796f' + unihash3 = 'a69ec97f5af2e21e1a1f9cc8896965515d5559425666f734e245a3d40cee33d9' + up_client.report_unihash(taskhash3, self.METHOD, outhash3, unihash3) + + restart_upstream() + + with self.start_client(up_server.address) as up_client: + # Verify that reported hashes are correct after restaring server + self.assertTrue(up_client.unihash_exists(unihash1)) + self.assertClientGetHash(up_client, taskhash1, unihash1) + + self.assertTrue(up_client.unihash_exists(unihash2)) + self.assertClientGetHash(up_client, taskhash2, unihash2) + + async def check_unihashes(): + async with await create_async_client(down_server.address) as down_client: + async with down_client.unihash_exists_stream() as stream: + await stream.send_query(unihash1) + r = await stream.get_result() + self.assertTrue(r) + + restart_upstream() + + await stream.send_query(unihash2) + r = await stream.get_result() + self.assertTrue(r) + + await stream.send_query(unihash3) + r = await stream.get_result() + self.assertTrue(r) + + asyncio.run(check_unihashes()) + + def test_upstream_lost(self): + up_server = self.start_server() + down_server = self.start_server(upstream=up_server.address) + + def restart_upstream(): + nonlocal up_server + + self.stop_server(up_server) + up_server = self.start_server(addr=up_server.address, dbpath=up_server.dbpath) + + # Report some hashes + with self.start_client(up_server.address) as up_client: + taskhash1 = '8aa96fcffb5831b3c2c0cb75f0431e3f8b20554a' + outhash1 = 'afe240a439959ce86f5e322f8c208e1fedefea9e813f2140c81af866cc9edf7e' + unihash1 = '5b521d8a12683086cc08bc2c6d94a7a2dcff17eba53b9911e145d51164689380' + up_client.report_unihash(taskhash1, self.METHOD, outhash1, unihash1) + + taskhash2 = "e3da00593d6a7fb435c7e2114976c59c5fd6d561" + outhash2 = "1cf8713e645f491eb9c959d20b5cae1c47133a292626dda9b10709857cbe688a" + unihash2 = "7aebef07d66a8c0f92d0c4f65ec8b1fbb850a3693c53827b8774b64fa9a8a9fe" + up_client.report_unihash(taskhash2, self.METHOD, outhash2, unihash2) + + taskhash3 = '35788efcb8dfb0a02659d81cf2bfd695fb30faf9' + outhash3 = '2765d4a5884be49b28601445c2760c5f21e7e5c0ee2b7e3fce98fd7e5970796f' + unihash3 = 'a69ec97f5af2e21e1a1f9cc8896965515d5559425666f734e245a3d40cee33d9' + up_client.report_unihash(taskhash3, self.METHOD, outhash3, unihash3) + + async def check_unihashes(): + async with await create_async_client(down_server.address) as down_client: + with self.assertRaises(ConnectionError): + async with down_client.unihash_exists_stream() as stream: + await stream.send_query(unihash1) + r = await stream.get_result() + self.assertTrue(r) + + self.stop_server(up_server) + + await stream.send_query(unihash2) + r = await stream.get_result() + + asyncio.run(check_unihashes()) + def test_unihash_exsits(self): taskhash, outhash, unihash = self.create_test_hash(self.client) self.assertTrue(self.client.unihash_exists(unihash))
Signature · 1 line
2.54.0
PATCH v2 10/10 hashserv: tests: Add test for upstream pipelining
In reply to Joshua WattAdds a test that verifies that pipelining to an upstream server works as expected.
AI-Generated: Uses Cursor, Claude Sonnet 5
--- lib/hashserv/tests.py | 96 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 96 insertions(+)
lib/hashserv/tests.py +96 -0
@@ -1693,6 +1693,102 @@ class TestHashEquivalenceTCPServer(HashEquivalenceTestSetup, HashEquivalenceComm # case it is more reliable to resolve the IP address explicitly. return socket.gethostbyname("localhost") + ":0" + def test_get_stream_upstream_pipelined(self): + # Verify that, with an upstream configured, the get-stream handler + # pipelines its upstream queries instead of doing one blocking + # round-trip per task. A latency proxy injects RTT between this server + # and its upstream; a serial (one round-trip per task) implementation + # could not beat N * RTT, so finishing far faster proves the queries + # are pipelined. + import asyncio + + LATENCY = 0.01 # 10ms each direction => ~20ms round trip + upstream_host, upstream_port = self.server_address.rsplit(":", 1) + upstream_port = int(upstream_port) + + ready = threading.Event() + proxy = {} + + def run_proxy(): + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + + async def handle(creader, cwriter): + ureader, uwriter = await asyncio.open_connection(upstream_host, upstream_port) + + async def pipe(r, w): + try: + while True: + data = await r.read(65536) + if not data: + break + await asyncio.sleep(LATENCY) + w.write(data) + await w.drain() + except Exception: + pass + finally: + try: + w.close() + except Exception: + pass + + await bb.asyncrpc.TaskGroup.run(pipe(creader, uwriter), pipe(ureader, cwriter)) + + stop_event = asyncio.Event() + proxy["stop_event"] = stop_event + proxy["loop"] = loop + + async def main(): + server = await asyncio.start_server(handle, "127.0.0.1", 0) + proxy["port"] = server.sockets[0].getsockname()[1] + ready.set() + async with server: + await stop_event.wait() + + try: + loop.run_until_complete(main()) + except Exception: + pass + finally: + loop.close() + + proxy_thread = threading.Thread(target=run_proxy, daemon=True) + proxy_thread.start() + self.assertTrue(ready.wait(10), "latency proxy did not start") + self.addCleanup(proxy_thread.join, 10) + self.addCleanup(lambda: proxy["loop"].call_soon_threadsafe(proxy["stop_event"].set)) + proxy_addr = "127.0.0.1:%d" % proxy["port"] + + N = 400 + expected = [] + for i in range(N): + taskhash = hashlib.sha256(("task%d" % i).encode()).hexdigest() + outhash = hashlib.sha256(("out%d" % i).encode()).hexdigest() + unihash = hashlib.sha256(("uni%d" % i).encode()).hexdigest() + self.client.report_unihash(taskhash, self.METHOD, outhash, unihash) + expected.append((self.METHOD, taskhash, unihash)) + + # Downstream server with an EMPTY local DB whose upstream is the slow proxy. + down_server = self.start_server(upstream=proxy_addr) + down_client = self.start_client(down_server.address) + + args = [(m, th) for (m, th, _uh) in expected] + start = time.time() + results = down_client.get_unihash_batch(args) + elapsed = time.time() - start + + self.assertEqual(results, [uh for (_m, _th, uh) in expected]) + + # A serial (one round-trip per task) implementation cannot beat N*RTT. + # Allow a generous margin to avoid flakiness on loaded CI machines. + serial_lower_bound = N * 2 * LATENCY + self.assertLess(elapsed, serial_lower_bound / 4, + "Upstream queries are not being pipelined " + "(%.3fs for %d queries at %.0fms injected RTT)" + % (elapsed, N, 2 * LATENCY * 1000)) + + class TestHashEquivalenceWebsocketServer(HashEquivalenceTestSetup, HashEquivalenceCommonTests, unittest.TestCase): def setUp(self):
Signature · 1 line
2.54.0
PATCH v2 1/10 asyncrpc: Add Task Group
In reply to Joshua WattAdds a Task Group object which can be used to manage multiple asynchronous tasks and correctly cancel them if an exception is raised
--- lib/bb/asyncrpc/__init__.py | 1 + lib/bb/asyncrpc/taskgroup.py | 52 ++++++++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+) create mode 100644 lib/bb/asyncrpc/taskgroup.py
lib/bb/asyncrpc/__init__.py +1 -0
@@ -14,3 +14,4 @@ from .exceptions import ( ConnectionClosedError, InvokeError, ) +from .taskgroup import TaskGroup
lib/bb/asyncrpc/taskgroup.py new file +52 -0
52 lines not on this page · raw message
Signature · 1 line
2.54.0
PATCH v2 4/10 hashserv: tests: Improve test logging
In reply to Joshua WattLog client messages to a file to aid in test debugging
--- lib/hashserv/tests.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-)
lib/hashserv/tests.py +3 -1
@@ -30,7 +30,7 @@ BIN_DIR = THIS_DIR.parent.parent / "bin" def server_prefunc(server, idx): logging.basicConfig(level=logging.DEBUG, filename='bbhashserv-%d.log' % idx, filemode='w', - format='%(levelname)s %(filename)s:%(lineno)d %(message)s') + format='%(levelname)s %(filename)s:%(lineno)d %(message)s', force=True) server.logger.debug("Running server %d" % idx) sys.stdout = open('bbhashserv-stdout-%d.log' % idx, 'w') sys.stderr = sys.stdout
@@ -93,6 +93,8 @@ class HashEquivalenceTestSetup(object): return self.start_client(self.auth_server_address, user["username"], user["token"]) def setUp(self): + logging.basicConfig(level=logging.DEBUG, filename='bbhashtest.log', filemode='w', + format='%(levelname)s %(filename)s:%(lineno)d %(message)s') self.temp_dir = tempfile.TemporaryDirectory(prefix='bb-hashserv') self.addCleanup(self.temp_dir.cleanup)
Signature · 1 line
2.54.0
Hi,
something is not working for me...
In the first run I got it stuck for an hour at:
WARNING: Error contacting Hash Equivalence Server unix:///path/to/my/workdir/builds/yoctobuild/hashserve.sock: Sent 192 messages but only received 49
WARNING: Error contacting Hash Equivalence Server unix:///path/to/my/workdir/builds/yoctobuild/hashserve.sock: Sent 115 messages but only received 0
WARNING: Error contacting Hash Equivalence Server unix:///path/to/my/workdir/builds/yoctobuild/hashserve.sock: Sent 115 messages but only received 0
WARNING: Error contacting Hash Equivalence Server unix:///path/to/my/workdir/builds/yoctobuild/hashserve.sock: Sent 165 messages but only received 0
WARNING: Error contacting Hash Equivalence Server unix:///path/to/my/workdir/builds/yoctobuild/hashserve.sock: Sent 9 messages but only received 4
WARNING: Error contacting Hash Equivalence Server unix:///path/to/my/workdir/builds/yoctobuild/hashserve.sock: Sent 99 messages but only received 0
WARNING: Error contacting Hash Equivalence Server unix:///path/to/my/workdir/builds/yoctobuild/hashserve.sock: Sent 15 messages but only received 0
WARNING: Error contacting Hash Equivalence Server unix:///path/to/my/workdir/builds/yoctobuild/hashserve.sock: Sent 9 messages but only received 0
WARNING: Error contacting Hash Equivalence Server unix:///path/to/my/workdir/builds/yoctobuild/hashserve.sock: Sent 1 messages but only received 0
WARNING: Error contacting Hash Equivalence Server unix:///path/to/my/workdir/builds/yoctobuild/hashserve.sock: Sent 4 messages but only received 0
WARNING: Error contacting Hash Equivalence Server unix:///path/to/my/workdir/builds/yoctobuild/hashserve.sock: Sent 47 messages but only received 0
WARNING: Error contacting Hash Equivalence Server unix:///path/to/my/workdir/builds/yoctobuild/hashserve.sock: Sent 1 messages but only received 0
WARNING: Error contacting Hash Equivalence Server unix:///path/to/my/workdir/builds/yoctobuild/hashserve.sock: Sent 116 messages but only received 0
WARNING: Error contacting Hash Equivalence Server unix:///path/to/my/workdir/builds/yoctobuild/hashserve.sock: Sent 112 messages but only received 0
WARNING: Error contacting Hash Equivalence Server unix:///path/to/my/workdir/builds/yoctobuild/hashserve.sock: Sent 117 messages but only received 0
WARNING: Error contacting Hash Equivalence Server unix:///path/to/my/workdir/builds/yoctobuild/hashserve.sock: Sent 277 messages but only received 0
WARNING: Error contacting Hash Equivalence Server unix:///path/to/my/workdir/builds/yoctobuild/hashserve.sock: Sent 66 messages but only received 0
WARNING: Error contacting Hash Equivalence Server unix:///path/to/my/workdir/builds/yoctobuild/hashserve.sock: Sent 72 messages but only received 0
WARNING: Error contacting Hash Equivalence Server unix:///path/to/my/workdir/builds/yoctobuild/hashserve.sock: Sent 157 messages but only received 0
WARNING: Error contacting Hash Equivalence Server unix:///path/to/my/workdir/builds/yoctobuild/hashserve.sock: Sent 45 messages but only received 0
WARNING: Error contacting Hash Equivalence Server unix:///path/to/my/workdir/builds/yoctobuild/hashserve.sock: Sent 52 messages but only received 0
WARNING: Error contacting Hash Equivalence Server unix:///path/to/my/workdir/builds/yoctobuild/hashserve.sock: Sent 75 messages but only received 0
WARNING: Error contacting Hash Equivalence Server unix:///path/to/my/workdir/builds/yoctobuild/hashserve.sock: Sent 18 messages but only received 0
WARNING: Error contacting Hash Equivalence Server unix:///path/to/my/workdir/builds/yoctobuild/hashserve.sock: Sent 10 messages but only received 0
WARNING: Error contacting Hash Equivalence Server unix:///path/to/my/workdir/builds/yoctobuild/hashserve.sock: Sent 98 messages but only received 0 Initialising tasks: 44% |########################################################## | ETA: 1:03:30
Then in another run, after reporting that 8 messages were sent, but 0 received, hashserve.sock disappeared!
WARNING: Error contacting Hash Equivalence Server unix:///path/to/my/workdir/builds/yoctobuild/hashserve.sock: Sent 8 messages but only received 0
WARNING: Error contacting Hash Equivalence Server unix:///path/to/my/workdir/builds/yoctobuild/hashserve.sock: [Errno 2] No such file or directory
WARNING: Error contacting Hash Equivalence Server unix:///path/to/my/workdir/builds/yoctobuild/hashserve.sock: [Errno 2] No such file or directory
WARNING: Error contacting Hash Equivalence Server unix:///path/to/my/workdir/builds/yoctobuild/hashserve.sock: [Errno 2] No such file or directory Although that might be a different issue after killing previous run and not making sure all Cooker processes are actually dead?
Anyway, after properly making sure there are no lingering processes, I am still getting stuck at:
WARNING: [...] Sent 192 messages but only received 26
WARNING: [...] Sent 115 messages but only received 0
WARNING: [...] Sent 115 messages but only received 0
WARNING: [...] Sent 165 messages but only received 0
WARNING: [...] Sent 9 messages but only received 1
I will be going on vacation now, so unfortunately, I won't be able to test more if you come up with anything new.
Best regards,
Michal
Michal,
Any chance you've had time to look at this more? I've been unable to reproduce this particular issue. It looks like the server is just crashing or stuck somewhere where it's no longer responding, but I can't quite figure out where that might be occurring.
Thanks,
Joshua Watt
Quoted 126 lines · Michal Sieron via lists.openembedded.org
Hi,
something is not working for me...
In the first run I got it stuck for an hour at:
WARNING: Error contacting Hash Equivalence Server unix:///path/to/my/workdir/builds/yoctobuild/hashserve.sock: Sent 192 messages but only received 49
WARNING: Error contacting Hash Equivalence Server unix:///path/to/my/workdir/builds/yoctobuild/hashserve.sock: Sent 115 messages but only received 0
WARNING: Error contacting Hash Equivalence Server unix:///path/to/my/workdir/builds/yoctobuild/hashserve.sock: Sent 115 messages but only received 0
WARNING: Error contacting Hash Equivalence Server unix:///path/to/my/workdir/builds/yoctobuild/hashserve.sock: Sent 165 messages but only received 0
WARNING: Error contacting Hash Equivalence Server unix:///path/to/my/workdir/builds/yoctobuild/hashserve.sock: Sent 9 messages but only received 4
WARNING: Error contacting Hash Equivalence Server unix:///path/to/my/workdir/builds/yoctobuild/hashserve.sock: Sent 99 messages but only received 0
WARNING: Error contacting Hash Equivalence Server unix:///path/to/my/workdir/builds/yoctobuild/hashserve.sock: Sent 15 messages but only received 0
WARNING: Error contacting Hash Equivalence Server unix:///path/to/my/workdir/builds/yoctobuild/hashserve.sock: Sent 9 messages but only received 0
WARNING: Error contacting Hash Equivalence Server unix:///path/to/my/workdir/builds/yoctobuild/hashserve.sock: Sent 1 messages but only received 0
WARNING: Error contacting Hash Equivalence Server unix:///path/to/my/workdir/builds/yoctobuild/hashserve.sock: Sent 4 messages but only received 0
WARNING: Error contacting Hash Equivalence Server unix:///path/to/my/workdir/builds/yoctobuild/hashserve.sock: Sent 47 messages but only received 0
WARNING: Error contacting Hash Equivalence Server unix:///path/to/my/workdir/builds/yoctobuild/hashserve.sock: Sent 1 messages but only received 0
WARNING: Error contacting Hash Equivalence Server unix:///path/to/my/workdir/builds/yoctobuild/hashserve.sock: Sent 116 messages but only received 0
WARNING: Error contacting Hash Equivalence Server unix:///path/to/my/workdir/builds/yoctobuild/hashserve.sock: Sent 112 messages but only received 0
WARNING: Error contacting Hash Equivalence Server unix:///path/to/my/workdir/builds/yoctobuild/hashserve.sock: Sent 117 messages but only received 0
WARNING: Error contacting Hash Equivalence Server unix:///path/to/my/workdir/builds/yoctobuild/hashserve.sock: Sent 277 messages but only received 0
WARNING: Error contacting Hash Equivalence Server unix:///path/to/my/workdir/builds/yoctobuild/hashserve.sock: Sent 66 messages but only received 0
WARNING: Error contacting Hash Equivalence Server unix:///path/to/my/workdir/builds/yoctobuild/hashserve.sock: Sent 72 messages but only received 0
WARNING: Error contacting Hash Equivalence Server unix:///path/to/my/workdir/builds/yoctobuild/hashserve.sock: Sent 157 messages but only received 0
WARNING: Error contacting Hash Equivalence Server unix:///path/to/my/workdir/builds/yoctobuild/hashserve.sock: Sent 45 messages but only received 0
WARNING: Error contacting Hash Equivalence Server unix:///path/to/my/workdir/builds/yoctobuild/hashserve.sock: Sent 52 messages but only received 0
WARNING: Error contacting Hash Equivalence Server unix:///path/to/my/workdir/builds/yoctobuild/hashserve.sock: Sent 75 messages but only received 0
WARNING: Error contacting Hash Equivalence Server unix:///path/to/my/workdir/builds/yoctobuild/hashserve.sock: Sent 18 messages but only received 0
WARNING: Error contacting Hash Equivalence Server unix:///path/to/my/workdir/builds/yoctobuild/hashserve.sock: Sent 10 messages but only received 0
WARNING: Error contacting Hash Equivalence Server unix:///path/to/my/workdir/builds/yoctobuild/hashserve.sock: Sent 98 messages but only received 0
Initialising tasks: 44% |##########################################################| ETA: 1:03:30Then in another run, after reporting that 8 messages were sent, but 0 received, hashserve.sock disappeared!
WARNING: Error contacting Hash Equivalence Server unix:///path/to/my/workdir/builds/yoctobuild/hashserve.sock: Sent 8 messages but only received 0
WARNING: Error contacting Hash Equivalence Server unix:///path/to/my/workdir/builds/yoctobuild/hashserve.sock: [Errno 2] No such file or directory
WARNING: Error contacting Hash Equivalence Server unix:///path/to/my/workdir/builds/yoctobuild/hashserve.sock: [Errno 2] No such file or directory
WARNING: Error contacting Hash Equivalence Server unix:///path/to/my/workdir/builds/yoctobuild/hashserve.sock: [Errno 2] No such file or directory
Although that might be a different issue after killing previous run and not making sure all Cooker processes are actually dead?Anyway, after properly making sure there are no lingering processes, I am still getting stuck at:
WARNING: [...] Sent 192 messages but only received 26
WARNING: [...] Sent 115 messages but only received 0
WARNING: [...] Sent 115 messages but only received 0
WARNING: [...] Sent 165 messages but only received 0
WARNING: [...] Sent 9 messages but only received 1I will be going on vacation now, so unfortunately, I won't be able to test more if you come up with anything new.
Best regards,
Michal-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group. View/Reply Online (#19895):
https://lists.openembedded.org/g/bitbake-devel/message/19895
Mute This Topic: https://lists.openembedded.org/mt/120523853/3616693
Group Owner: [email protected]
Unsubscribe: https://lists.openembedded.org/g/bitbake-devel/unsub [ [email protected]]
-=-=-=-=-=-=-=-=-=-=-=-
Hi Joshua,
Quoted 1 line
Any chance you've had time to look at this more? I've been unable to reproduce this particular issue. It looks like the server is just crashing or stuck somewhere where it's no longer responding, but I can't quite figure out where that might be occurring.
Unfortunately, I did not. But I just tried it again on my setup (new changes pulled since last time), and it reproduced immediately. As I still don't have *that much* time to debug this I went and threw this at Cursor, which did find the issue and generated a fix and a minimal repro test. I attached the patch.
Br,
Michał
From 5bc6f9834c56327db2e853af6d7d188f40011b4f Mon Sep 17 00:00:00 2001
From: Michal Sieron <[email protected]>
Date: Thu, 27 Aug 2026 00:28:24 +0200
Subject: [PATCH] hashserv: server: Fix upstream get-unihash miss truncating
stream
When an upstream hash equivalence server is configured, handle_get_stream() resolves each query through get_upstream_result(). On an upstream miss this returned None, but None is the end-of-stream sentinel consumed by _stream_queue_handler(). Returning it there terminated the response stream early, so the downstream client received fewer replies than it sent, timed out and retried.
Return an empty string ("") on a miss instead, which matches the non-upstream get-stream handler.
Also add test_upstream_get_stream_miss covering an interleaved miss/hit batch, an all-miss batch, and a singular miss through an upstream.
AI-Generated: Uses Cursor
--- lib/hashserv/server.py | 4 +++- lib/hashserv/tests.py | 37 +++++++++++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 1 deletion(-)
lib/hashserv/server.py +3 -1
@@ -504,7 +504,9 @@ class ServerClient(bb.asyncrpc.AsyncServerConnection): if unihash: method, taskhash = m.split() await self.server.backfill_queue.put((method, taskhash)) - return unihash + # On miss replace `None` with `""` to match non-upstream handler + # and avoid stream truncation. + return unihash or "" queue = asyncio.Queue() upstream = UpstreamQueue(
lib/hashserv/tests.py +37 -0
@@ -606,6 +606,43 @@ class HashEquivalenceCommonTests(object): asyncio.run(check_unihashes()) + def test_upstream_get_stream_miss(self): + down_server = self.start_server(upstream=self.server.address) + down_client = self.start_client(down_server.address) + + # Two hashes present upstream (hits) + taskhash1 = '8aa96fcffb5831b3c2c0cb75f0431e3f8b20554a' + outhash1 = 'afe240a439959ce86f5e322f8c208e1fedefea9e813f2140c81af866cc9edf7e' + unihash1 = '5b521d8a12683086cc08bc2c6d94a7a2dcff17eba53b9911e145d51164689380' + self.client.report_unihash(taskhash1, self.METHOD, outhash1, unihash1) + + taskhash2 = 'e3da00593d6a7fb435c7e2114976c59c5fd6d561' + outhash2 = '1cf8713e645f491eb9c959d20b5cae1c47133a292626dda9b10709857cbe688a' + unihash2 = '7aebef07d66a8c0f92d0c4f65ec8b1fbb850a3693c53827b8774b64fa9a8a9fe' + self.client.report_unihash(taskhash2, self.METHOD, outhash2, unihash2) + + # Two taskhashes present nowhere (upstream misses) + miss1 = '0000000000000000000000000000000000000001' + miss2 = '0000000000000000000000000000000000000002' + + # Miss interleaved with hits: a miss must not truncate the stream + results = down_client.get_unihash_batch([ + (self.METHOD, miss1), + (self.METHOD, taskhash1), + (self.METHOD, miss2), + (self.METHOD, taskhash2), + ]) + self.assertEqual(results, [None, unihash1, None, unihash2]) + + # All-miss batch + self.assertEqual( + down_client.get_unihash_batch([(self.METHOD, miss1), (self.METHOD, miss2)]), + [None, None], + ) + + # Singular get-unihash miss + self.assertClientGetHash(down_client, miss1, None) + def test_unihash_exsits(self): taskhash, outhash, unihash = self.create_test_hash(self.client) self.assertTrue(self.client.unihash_exists(unihash))
Ah, the upstream server is the part I was missing. Ya, that fix is probably correct. I'll run it through more testing and post it to the ML.
Thanks.
Quoted 28 lines · Michal Sieron via lists.openembedded.org
Hi Joshua,
Quoted 1 line
Any chance you've had time to look at this more? I've been unable to
reproduce this particular issue. It looks like the server is just crashing or stuck somewhere where it's no longer responding, but I can't quite figure out where that might be occurring.
Unfortunately, I did not. But I just tried it again on my setup (new changes pulled since last time), and it reproduced immediately. As I still don't have *that much* time to debug this I went and threw this at Cursor, which did find the issue and generated a fix and a minimal repro test.
I attached the patch.Br,
Michał-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group. View/Reply Online (#20062):
https://lists.openembedded.org/g/bitbake-devel/message/20062
Mute This Topic: https://lists.openembedded.org/mt/120523853/3616693
Group Owner: [email protected]
Unsubscribe: https://lists.openembedded.org/g/bitbake-devel/unsub [ [email protected]]
-=-=-=-=-=-=-=-=-=-=-=-
v3 cover hashserv: Pipeline Upstream Queries
In reply to Joshua WattAdds a true asynchronous streaming API to the hash equivalence clients, then uses that streaming API to pipeline requests to an upstream server. This means that when a server wants to defer a request to an upstream, it no longer has to wait for a complete round-trip API call. Instead, it can use the faster streaming API to send the requests, and continue processing other requests while a response inbound.
V2: Fixes several bugs related to when the connection between the client and upstream server was disconnected, and added tests for them.
V3: Fixes an unexpected disconnect when there was a unihash miss from the upstream server while streaming
Joshua Watt (10):
asyncrpc: Add Task Group
asyncrpc: serv: Use Task Group
asyncrpc: serv: Cancel all clients on server stop
hashserv: tests: Improve test logging
hashserv: client: Add asynchronous streaming API
hashserv: server: Add queued streaming API
hashserv: server: Use streaming and queue API for upstream unihash
queries
hashserv: server: Use streaming and queue API for upstream exist
queries
hashserv: tests: Add more upstream tests
hashserv: tests: Add test for upstream pipelining
Michal Sieron (1):
hashserv: server: Fix upstream get-unihash miss truncating stream
bin/bitbake-hashclient | 2 +- lib/bb/asyncrpc/__init__.py | 1 + lib/bb/asyncrpc/serv.py | 13 +- lib/bb/asyncrpc/taskgroup.py | 52 ++++++ lib/hashserv/client.py | 349 +++++++++++++++++++++++++++-------- lib/hashserv/server.py | 197 ++++++++++++++++---- lib/hashserv/tests.py | 298 ++++++++++++++++++++++++++++-- 7 files changed, 776 insertions(+), 136 deletions(-) create mode 100644 lib/bb/asyncrpc/taskgroup.py
Signature · 1 line
2.55.0
PATCH v3 2/11 asyncrpc: serv: Use Task Group
Uses a task group instead of asyncio.gather(). The task group ensures that all tasks are canceled if an exception occurs, which asyncio.gather() does not.
--- lib/bb/asyncrpc/serv.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
lib/bb/asyncrpc/serv.py +1 -1
@@ -334,7 +334,7 @@ class AsyncServer(object): self.loop.add_signal_handler(signal.SIGQUIT, self.signal_handler) signal.pthread_sigmask(signal.SIG_UNBLOCK, [signal.SIGTERM]) - self.loop.run_until_complete(asyncio.gather(*tasks)) + self.loop.run_until_complete(bb.asyncrpc.TaskGroup.run(*tasks)) self.logger.debug("Server shutting down") finally:
Signature · 1 line
2.55.0
PATCH v3 5/11 hashserv: client: Add asynchronous streaming API
In reply to Joshua WattImplments a true asynchronous streaming API for getting a unihash (get_unihash_stream()) and checking if a unihash exists (unihash_exists_stream()). These APIs allow a client to send a query the to the server, then wait for the reply later. Using this API, it is possible to interleave queries and replies.
In addition, the gc_mark_stream() API is renamed gc_mark_batch() to match the pattern of the other APIs, and an actual gc_mark_stream() API is implemented that allows asynchronous streaming like the others.
Note that the new stream APIs are not accessible in the "synchronous" client API since async constructs must be used for them to make sense.
--- bin/bitbake-hashclient | 2 +- lib/hashserv/client.py | 349 ++++++++++++++++++++++++++++++++--------- lib/hashserv/tests.py | 4 +- 3 files changed, 275 insertions(+), 80 deletions(-)
bin/bitbake-hashclient +1 -1
@@ -240,7 +240,7 @@ def main(): marked_hashes = 0 try: - result = client.gc_mark_stream(args.mark, stdin) + result = client.gc_mark_batch(args.mark, stdin) marked_hashes = result["count"] except ConnectionError: logger.warning(
lib/hashserv/client.py +272 -77
428 lines not on this page · raw message
lib/hashserv/tests.py +2 -2
@@ -1057,7 +1057,7 @@ class HashEquivalenceCommonTests(object): # First hash is still present self.assertClientGetHash(self.client, taskhash, unihash) - def test_gc_stream(self): + def test_gc_batch(self): taskhash = '53b8dce672cb6d0c73170be43f540460bfc347b4' outhash = '5a9cb1649625f0bf41fc7791b635cd9c2d7118c7f021ba87dcd03f72b67ce7a8' unihash = '46edb5140d2613049332d0bf3745d9fafec9c559dac8cc61813739a28007fcdf'
@@ -1080,7 +1080,7 @@ class HashEquivalenceCommonTests(object): self.assertClientGetHash(self.client, taskhash3, unihash3) # Mark the first unihash to be kept - ret = self.client.gc_mark_stream("ABC", (f"unihash {h}" for h in [unihash, unihash2])) + ret = self.client.gc_mark_batch("ABC", (f"unihash {h}" for h in [unihash, unihash2])) self.assertEqual(ret, {"count": 2}) ret = self.client.gc_status()
Signature · 1 line
2.55.0
PATCH v3 7/11 hashserv: server: Use streaming and queue API for upstream unihash queries
In reply to Joshua WattReworks the "get-unihash" handler to use the new server queue API and client streaming API to efficiently stream requests to the upstream server instead of having to wait for a roundtrip on the requests.
--- lib/hashserv/server.py | 52 ++++++++++++++++++++++++++++++------------ 1 file changed, 38 insertions(+), 14 deletions(-)
lib/hashserv/server.py +38 -14
@@ -477,24 +477,48 @@ class ServerClient(bb.asyncrpc.AsyncServerConnection): @permissions(READ_PERM) async def handle_get_stream(self, request): - async def handler(l): - method, taskhash = l.split() - # self.logger.debug('Looking up %s %s' % (method, taskhash)) - row = await self.db.get_equivalent(method, taskhash) - - if row is not None: - # self.logger.debug('Found equivalent task %s -> %s', (row['taskhash'], row['unihash'])) + async def get_unihash(m): + method, taskhash = m.split() + if (row := await self.db.get_equivalent(method, taskhash)) is not None: return row["unihash"] - if self.upstream_client is not None: - upstream = await self.upstream_client.get_unihash(method, taskhash) - if upstream: - await self.server.backfill_queue.put((method, taskhash)) - return upstream - return "" - return await self._stream_handler(handler) + if not self.upstream_client: + return await self._stream_handler(get_unihash) + + async with self.upstream_client.get_unihash_stream() as stream: + + async def get_local_result(m): + method, taskhash = m.split() + if (row := await self.db.get_equivalent(method, taskhash)) is not None: + return row["unihash"] + return None + + async def send_upstream(m): + method, taskhash = m.split() + await stream.send_query(method, taskhash) + + async def get_upstream_result(m): + unihash = await stream.get_result() + if unihash: + method, taskhash = m.split() + await self.server.backfill_queue.put((method, taskhash)) + return unihash + + queue = asyncio.Queue() + upstream = UpstreamQueue( + queue, + get_local_result, + send_upstream, + get_upstream_result, + ) + + await bb.asyncrpc.TaskGroup.run( + self._stream_queue_handler(upstream.handler, queue), + upstream.process_results(), + ) + return self.NO_RESPONSE @permissions(READ_PERM) async def handle_exists_stream(self, request):
Signature · 1 line
2.55.0
PATCH v3 8/11 hashserv: server: Use streaming and queue API for upstream exist queries
In reply to Joshua WattReworks the "unihash-exists" handler to use the new server queue API and client streaming API to efficiently stream requests to the upstream server instead of having to wait for a roundtrip on the requests.
--- lib/hashserv/server.py | 31 ++++++++++++++++++++++++++----- lib/hashserv/tests.py | 5 +++++ 2 files changed, 31 insertions(+), 5 deletions(-)
lib/hashserv/server.py +26 -5
@@ -522,17 +522,38 @@ class ServerClient(bb.asyncrpc.AsyncServerConnection): @permissions(READ_PERM) async def handle_exists_stream(self, request): - async def handler(l): + async def exists_handler(l): if await self.db.unihash_exists(l): return "true" + return "false" + + if not self.upstream_client: + return await self._stream_handler(exists_handler) - if self.upstream_client is not None: - if await self.upstream_client.unihash_exists(l): + async with self.upstream_client.unihash_exists_stream() as stream: + + async def get_local_result(m): + if await self.db.unihash_exists(m): return "true" + return None - return "false" + async def get_upstream_result(m): + exists = await stream.get_result() + return "true" if exists else "false" - return await self._stream_handler(handler) + queue = asyncio.Queue() + upstream = UpstreamQueue( + queue, + get_local_result, + stream.send_query, + get_upstream_result, + ) + + await bb.asyncrpc.TaskGroup.run( + self._stream_queue_handler(upstream.handler, queue), + upstream.process_results(), + ) + return self.NO_RESPONSE async def report_readonly(self, data): method = data["method"]
lib/hashserv/tests.py +5 -0
@@ -374,20 +374,25 @@ class HashEquivalenceCommonTests(object): nonlocal side_client # check upstream server + self.assertTrue(self.client.unihash_exists(unihash)) self.assertClientGetHash(self.client, taskhash, unihash) # Hash should *not* be present on the side server + if old_sidehash and unihash != old_sidehash: + self.assertFalse(side_client.unihash_exists(unihash)) self.assertClientGetHash(side_client, taskhash, old_sidehash) # Hash should be present on the downstream server, since it # will defer to the upstream server. This will trigger # the backfill in the downstream server + self.assertTrue(down_client.unihash_exists(unihash)) self.assertClientGetHash(down_client, taskhash, unihash) # After waiting for the downstream client to finish backfilling the # task from the upstream server, it should appear in the side server # since the database is populated down_client.backfill_wait() + self.assertTrue(side_client.unihash_exists(unihash)) self.assertClientGetHash(side_client, taskhash, unihash) # Basic report
Signature · 1 line
2.55.0
PATCH v3 9/11 hashserv: tests: Add more upstream tests
In reply to Joshua WattAdds a test to verify that the batch API works properly with an upstream server and also a test to verify that if the upstream server restarts or is temporarily disconnected that the client recovers properly.
--- lib/hashserv/tests.py | 152 +++++++++++++++++++++++++++++++++++++++--- 1 file changed, 141 insertions(+), 11 deletions(-)
lib/hashserv/tests.py +141 -11
@@ -5,12 +5,13 @@ # SPDX-License-Identifier: GPL-2.0-only # -from . import create_server, create_client +from . import create_server, create_client, create_async_client from .server import DEFAULT_ANON_PERMS, ALL_PERMISSIONS from bb.asyncrpc import InvokeError import hashlib import logging from bb import multiprocessing +import asyncio import os import sys import tempfile
@@ -41,19 +42,15 @@ class HashEquivalenceTestSetup(object): server_index = 0 client_index = 0 - def start_server(self, dbpath=None, upstream=None, read_only=False, prefunc=server_prefunc, anon_perms=DEFAULT_ANON_PERMS, admin_username=None, admin_password=None): + def start_server(self, dbpath=None, upstream=None, read_only=False, prefunc=server_prefunc, anon_perms=DEFAULT_ANON_PERMS, admin_username=None, admin_password=None, addr=None): self.server_index += 1 + if addr is None: + addr = self.get_server_addr(self.server_index) + if dbpath is None: dbpath = self.make_dbpath() - def cleanup_server(server): - if server.process.exitcode is not None: - return - - server.process.terminate() - server.process.join() - - server = create_server(self.get_server_addr(self.server_index), + server = create_server(addr, dbpath, upstream=upstream, read_only=read_only,
@@ -63,7 +60,7 @@ class HashEquivalenceTestSetup(object): server.dbpath = dbpath server.serve_as_process(prefunc=prefunc, args=(self.server_index,)) - self.addCleanup(cleanup_server, server) + self.addCleanup(self.stop_server, server) return server
@@ -83,6 +80,13 @@ class HashEquivalenceTestSetup(object): self.server = self.start_server() return self.server.address + def stop_server(self, server): + if not server or server.process.exitcode is not None: + return + + server.process.terminate() + server.process.join() + def start_auth_server(self): auth_server = self.start_server(self.server.dbpath, anon_perms=[], admin_username="admin", admin_password="password") self.auth_server_address = auth_server.address
@@ -476,6 +480,132 @@ class HashEquivalenceCommonTests(object): self.assertEqual(result['taskhash'], taskhash9, 'Server failed to copy unihash from upstream') self.assertEqual(result['method'], self.METHOD) + def test_upstream_batch(self): + down_server = self.start_server(upstream=self.server.address) + down_client = self.start_client(down_server.address) + + taskhash1 = '8aa96fcffb5831b3c2c0cb75f0431e3f8b20554a' + outhash1 = 'afe240a439959ce86f5e322f8c208e1fedefea9e813f2140c81af866cc9edf7e' + unihash1 = '5b521d8a12683086cc08bc2c6d94a7a2dcff17eba53b9911e145d51164689380' + self.client.report_unihash(taskhash1, self.METHOD, outhash1, unihash1) + + taskhash2 = "e3da00593d6a7fb435c7e2114976c59c5fd6d561" + outhash2 = "1cf8713e645f491eb9c959d20b5cae1c47133a292626dda9b10709857cbe688a" + unihash2 = "7aebef07d66a8c0f92d0c4f65ec8b1fbb850a3693c53827b8774b64fa9a8a9fe" + self.client.report_unihash(taskhash2, self.METHOD, outhash2, unihash2) + + taskhash3 = '35788efcb8dfb0a02659d81cf2bfd695fb30faf9' + outhash3 = '2765d4a5884be49b28601445c2760c5f21e7e5c0ee2b7e3fce98fd7e5970796f' + unihash3 = 'a69ec97f5af2e21e1a1f9cc8896965515d5559425666f734e245a3d40cee33d9' + self.client.report_unihash(taskhash3, self.METHOD, outhash3, unihash3) + + def query_generator(): + yield unihash1 + yield unihash2 + yield unihash3 + yield "cc74784b2c0ad5b378a6b783c74c518d2c46b8b52fba29cb39a8430d742440d7" + + results = down_client.unihash_exists_batch(query_generator()) + self.assertEqual(results, [True, True, True, False]) + + def test_upstream_interrupted(self): + up_server = self.start_server() + down_server = self.start_server(upstream=up_server.address) + + def restart_upstream(): + nonlocal up_server + + self.stop_server(up_server) + up_server = self.start_server(addr=up_server.address, dbpath=up_server.dbpath) + + # Report some hashes + with self.start_client(up_server.address) as up_client: + taskhash1 = '8aa96fcffb5831b3c2c0cb75f0431e3f8b20554a' + outhash1 = 'afe240a439959ce86f5e322f8c208e1fedefea9e813f2140c81af866cc9edf7e' + unihash1 = '5b521d8a12683086cc08bc2c6d94a7a2dcff17eba53b9911e145d51164689380' + up_client.report_unihash(taskhash1, self.METHOD, outhash1, unihash1) + + taskhash2 = "e3da00593d6a7fb435c7e2114976c59c5fd6d561" + outhash2 = "1cf8713e645f491eb9c959d20b5cae1c47133a292626dda9b10709857cbe688a" + unihash2 = "7aebef07d66a8c0f92d0c4f65ec8b1fbb850a3693c53827b8774b64fa9a8a9fe" + up_client.report_unihash(taskhash2, self.METHOD, outhash2, unihash2) + + taskhash3 = '35788efcb8dfb0a02659d81cf2bfd695fb30faf9' + outhash3 = '2765d4a5884be49b28601445c2760c5f21e7e5c0ee2b7e3fce98fd7e5970796f' + unihash3 = 'a69ec97f5af2e21e1a1f9cc8896965515d5559425666f734e245a3d40cee33d9' + up_client.report_unihash(taskhash3, self.METHOD, outhash3, unihash3) + + restart_upstream() + + with self.start_client(up_server.address) as up_client: + # Verify that reported hashes are correct after restaring server + self.assertTrue(up_client.unihash_exists(unihash1)) + self.assertClientGetHash(up_client, taskhash1, unihash1) + + self.assertTrue(up_client.unihash_exists(unihash2)) + self.assertClientGetHash(up_client, taskhash2, unihash2) + + async def check_unihashes(): + async with await create_async_client(down_server.address) as down_client: + async with down_client.unihash_exists_stream() as stream: + await stream.send_query(unihash1) + r = await stream.get_result() + self.assertTrue(r) + + restart_upstream() + + await stream.send_query(unihash2) + r = await stream.get_result() + self.assertTrue(r) + + await stream.send_query(unihash3) + r = await stream.get_result() + self.assertTrue(r) + + asyncio.run(check_unihashes()) + + def test_upstream_lost(self): + up_server = self.start_server() + down_server = self.start_server(upstream=up_server.address) + + def restart_upstream(): + nonlocal up_server + + self.stop_server(up_server) + up_server = self.start_server(addr=up_server.address, dbpath=up_server.dbpath) + + # Report some hashes + with self.start_client(up_server.address) as up_client: + taskhash1 = '8aa96fcffb5831b3c2c0cb75f0431e3f8b20554a' + outhash1 = 'afe240a439959ce86f5e322f8c208e1fedefea9e813f2140c81af866cc9edf7e' + unihash1 = '5b521d8a12683086cc08bc2c6d94a7a2dcff17eba53b9911e145d51164689380' + up_client.report_unihash(taskhash1, self.METHOD, outhash1, unihash1) + + taskhash2 = "e3da00593d6a7fb435c7e2114976c59c5fd6d561" + outhash2 = "1cf8713e645f491eb9c959d20b5cae1c47133a292626dda9b10709857cbe688a" + unihash2 = "7aebef07d66a8c0f92d0c4f65ec8b1fbb850a3693c53827b8774b64fa9a8a9fe" + up_client.report_unihash(taskhash2, self.METHOD, outhash2, unihash2) + + taskhash3 = '35788efcb8dfb0a02659d81cf2bfd695fb30faf9' + outhash3 = '2765d4a5884be49b28601445c2760c5f21e7e5c0ee2b7e3fce98fd7e5970796f' + unihash3 = 'a69ec97f5af2e21e1a1f9cc8896965515d5559425666f734e245a3d40cee33d9' + up_client.report_unihash(taskhash3, self.METHOD, outhash3, unihash3) + + async def check_unihashes(): + async with await create_async_client(down_server.address) as down_client: + with self.assertRaises(ConnectionError): + async with down_client.unihash_exists_stream() as stream: + await stream.send_query(unihash1) + r = await stream.get_result() + self.assertTrue(r) + + self.stop_server(up_server) + + await stream.send_query(unihash2) + r = await stream.get_result() + + asyncio.run(check_unihashes()) + def test_unihash_exsits(self): taskhash, outhash, unihash = self.create_test_hash(self.client) self.assertTrue(self.client.unihash_exists(unihash))
Signature · 1 line
2.55.0
PATCH v3 3/11 asyncrpc: serv: Cancel all clients on server stop
In reply to Joshua WattWhen the server is stopped, all clients should be stopped, otherwise the server will hang waiting for a disconnect.
--- lib/bb/asyncrpc/serv.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-)
lib/bb/asyncrpc/serv.py +9 -2
@@ -56,7 +56,7 @@ class AsyncServerConnection(object): if not client_protocol: return - (client_proto_name, client_proto_version) = client_protocol.split() + client_proto_name, client_proto_version = client_protocol.split() if client_proto_name != self.proto_name: self.logger.debug("Rejecting invalid protocol %s" % (self.proto_name)) return
@@ -123,6 +123,7 @@ class StreamServer(object): self.handler = handler self.logger = logger self.closed = False + self.clients = [] async def handle_stream_client(self, reader, writer): # writer.transport.set_write_buffer_limits(0)
@@ -131,10 +132,16 @@ class StreamServer(object): await socket.close() return - await self.handler(socket) + self.clients.append(socket) + try: + await self.handler(socket) + finally: + self.clients.remove(socket) async def stop(self): self.closed = True + for socket in self.clients: + await socket.close() class TCPStreamServer(StreamServer):
Signature · 1 line
2.55.0
PATCH v3 4/11 hashserv: tests: Improve test logging
In reply to Joshua WattLog client messages to a file to aid in test debugging
--- lib/hashserv/tests.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-)
lib/hashserv/tests.py +3 -1
@@ -30,7 +30,7 @@ BIN_DIR = THIS_DIR.parent.parent / "bin" def server_prefunc(server, idx): logging.basicConfig(level=logging.DEBUG, filename='bbhashserv-%d.log' % idx, filemode='w', - format='%(levelname)s %(filename)s:%(lineno)d %(message)s') + format='%(levelname)s %(filename)s:%(lineno)d %(message)s', force=True) server.logger.debug("Running server %d" % idx) sys.stdout = open('bbhashserv-stdout-%d.log' % idx, 'w') sys.stderr = sys.stdout
@@ -93,6 +93,8 @@ class HashEquivalenceTestSetup(object): return self.start_client(self.auth_server_address, user["username"], user["token"]) def setUp(self): + logging.basicConfig(level=logging.DEBUG, filename='bbhashtest.log', filemode='w', + format='%(levelname)s %(filename)s:%(lineno)d %(message)s') self.temp_dir = tempfile.TemporaryDirectory(prefix='bb-hashserv') self.addCleanup(self.temp_dir.cleanup)
Signature · 1 line
2.55.0
PATCH v3 6/11 hashserv: server: Add queued streaming API
In reply to Joshua WattAdds an API that allows a stream handler to more precisely control when a response is sent to the client. The new API does not directly send the response from the handler to the remote client, but instead the handler is expected to put the result in a provided queue when the response is ready. In particular, this allows a stream handler to defer to an upstream server (utilizing the new client streaming API) in an efficient way that does not require waiting on a roundtrip with the upstream server. Instead, several queries to the upstream can be in-flight at once.
--- lib/hashserv/server.py | 96 +++++++++++++++++++++++++++++++++--------- 1 file changed, 75 insertions(+), 21 deletions(-)
lib/hashserv/server.py +75 -21
@@ -229,6 +229,54 @@ def permissions(*permissions, allow_anon=True, allow_self_service=False): return wrapper +class UpstreamQueue(object): + UPSTREAM_NONCE = object() + + def __init__(self, queue, get_local_result, send_upstream, get_upstream_result): + self.queue = queue + self.pending = [] + self.cond = asyncio.Condition() + self.done = False + self.get_local_result = get_local_result + self.send_upstream = send_upstream + self.get_upstream_result = get_upstream_result + + async def process_results(self): + try: + while True: + async with self.cond: + await self.cond.wait_for(lambda: self.pending or self.done) + if not self.pending: + if self.done: + return + continue + + value, m = self.pending.pop(0) + + if value is self.UPSTREAM_NONCE: + value = await self.get_upstream_result(m) + + await self.queue.put(value) + finally: + await self.queue.put(None) + + async def handler(self, m): + if m is None: + async with self.cond: + self.done = True + self.cond.notify_all() + return + + value = await self.get_local_result(m) + if value is None: + await self.send_upstream(m) + value = self.UPSTREAM_NONCE + + async with self.cond: + self.pending.append((value, m)) + self.cond.notify_all() + + class ServerClient(bb.asyncrpc.AsyncServerConnection): def __init__(self, socket, server): super().__init__(socket, "OEHASHEQUIV", server.logger)
@@ -390,35 +438,41 @@ class ServerClient(bb.asyncrpc.AsyncServerConnection): validate_unihash(unihash) return await self.db.insert_unihash(method, taskhash, unihash) - async def _stream_handler(self, handler): + async def _stream_queue_handler(self, handler, queue): await self.socket.send_message("ok") - while True: - upstream = None + async def recv(): + try: + while True: + m = await self.socket.recv() + if not m or m == "END": + break - l = await self.socket.recv() - if not l: - break + await handler(m) + finally: + await handler(None) - try: - # This inner loop is very sensitive and must be as fast as - # possible (which is why the request sample is handled manually - # instead of using 'with', and also why logging statements are - # commented out. - self.request_sample = self.server.request_stats.start_sample() - request_measure = self.request_sample.measure() - request_measure.start() - - if l == "END": + async def process(): + while True: + m = await queue.get() + if m is None: break - msg = await handler(l) - await self.socket.send(msg) - finally: - request_measure.end() - self.request_sample.end() + await self.socket.send(m) + await bb.asyncrpc.TaskGroup.run(recv(), process()) await self.socket.send("ok") + + async def _stream_handler(self, handler): + queue = asyncio.Queue(1000) + + async def h(m): + if m is None: + await queue.put(None) + else: + await queue.put(await handler(m)) + + await self._stream_queue_handler(h, queue) return self.NO_RESPONSE @permissions(READ_PERM)
Signature · 1 line
2.55.0
PATCH v3 1/11 asyncrpc: Add Task Group
In reply to Joshua WattAdds a Task Group object which can be used to manage multiple asynchronous tasks and correctly cancel them if an exception is raised
--- lib/bb/asyncrpc/__init__.py | 1 + lib/bb/asyncrpc/taskgroup.py | 52 ++++++++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+) create mode 100644 lib/bb/asyncrpc/taskgroup.py
lib/bb/asyncrpc/__init__.py +1 -0
@@ -14,3 +14,4 @@ from .exceptions import ( ConnectionClosedError, InvokeError, ) +from .taskgroup import TaskGroup
lib/bb/asyncrpc/taskgroup.py new file +52 -0
52 lines not on this page · raw message
Signature · 1 line
2.55.0
PATCH v3 10/11 hashserv: tests: Add test for upstream pipelining
In reply to Joshua WattAdds a test that verifies that pipelining to an upstream server works as expected.
AI-Generated: Uses Cursor, Claude Sonnet 5
--- lib/hashserv/tests.py | 96 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 96 insertions(+)
lib/hashserv/tests.py +96 -0
@@ -1693,6 +1693,102 @@ class TestHashEquivalenceTCPServer(HashEquivalenceTestSetup, HashEquivalenceComm # case it is more reliable to resolve the IP address explicitly. return socket.gethostbyname("localhost") + ":0" + def test_get_stream_upstream_pipelined(self): + # Verify that, with an upstream configured, the get-stream handler + # pipelines its upstream queries instead of doing one blocking + # round-trip per task. A latency proxy injects RTT between this server + # and its upstream; a serial (one round-trip per task) implementation + # could not beat N * RTT, so finishing far faster proves the queries + # are pipelined. + import asyncio + + LATENCY = 0.01 # 10ms each direction => ~20ms round trip + upstream_host, upstream_port = self.server_address.rsplit(":", 1) + upstream_port = int(upstream_port) + + ready = threading.Event() + proxy = {} + + def run_proxy(): + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + + async def handle(creader, cwriter): + ureader, uwriter = await asyncio.open_connection(upstream_host, upstream_port) + + async def pipe(r, w): + try: + while True: + data = await r.read(65536) + if not data: + break + await asyncio.sleep(LATENCY) + w.write(data) + await w.drain() + except Exception: + pass + finally: + try: + w.close() + except Exception: + pass + + await bb.asyncrpc.TaskGroup.run(pipe(creader, uwriter), pipe(ureader, cwriter)) + + stop_event = asyncio.Event() + proxy["stop_event"] = stop_event + proxy["loop"] = loop + + async def main(): + server = await asyncio.start_server(handle, "127.0.0.1", 0) + proxy["port"] = server.sockets[0].getsockname()[1] + ready.set() + async with server: + await stop_event.wait() + + try: + loop.run_until_complete(main()) + except Exception: + pass + finally: + loop.close() + + proxy_thread = threading.Thread(target=run_proxy, daemon=True) + proxy_thread.start() + self.assertTrue(ready.wait(10), "latency proxy did not start") + self.addCleanup(proxy_thread.join, 10) + self.addCleanup(lambda: proxy["loop"].call_soon_threadsafe(proxy["stop_event"].set)) + proxy_addr = "127.0.0.1:%d" % proxy["port"] + + N = 400 + expected = [] + for i in range(N): + taskhash = hashlib.sha256(("task%d" % i).encode()).hexdigest() + outhash = hashlib.sha256(("out%d" % i).encode()).hexdigest() + unihash = hashlib.sha256(("uni%d" % i).encode()).hexdigest() + self.client.report_unihash(taskhash, self.METHOD, outhash, unihash) + expected.append((self.METHOD, taskhash, unihash)) + + # Downstream server with an EMPTY local DB whose upstream is the slow proxy. + down_server = self.start_server(upstream=proxy_addr) + down_client = self.start_client(down_server.address) + + args = [(m, th) for (m, th, _uh) in expected] + start = time.time() + results = down_client.get_unihash_batch(args) + elapsed = time.time() - start + + self.assertEqual(results, [uh for (_m, _th, uh) in expected]) + + # A serial (one round-trip per task) implementation cannot beat N*RTT. + # Allow a generous margin to avoid flakiness on loaded CI machines. + serial_lower_bound = N * 2 * LATENCY + self.assertLess(elapsed, serial_lower_bound / 4, + "Upstream queries are not being pipelined " + "(%.3fs for %d queries at %.0fms injected RTT)" + % (elapsed, N, 2 * LATENCY * 1000)) + + class TestHashEquivalenceWebsocketServer(HashEquivalenceTestSetup, HashEquivalenceCommonTests, unittest.TestCase): def setUp(self):
Signature · 1 line
2.55.0
PATCH v3 11/11 hashserv: server: Fix upstream get-unihash miss truncating stream
In reply to Joshua WattFrom: Michal Sieron <[email protected]>
When an upstream hash equivalence server is configured, handle_get_stream() resolves each query through get_upstream_result(). On an upstream miss this returned None, but None is the end-of-stream sentinel consumed by _stream_queue_handler(). Returning it there terminated the response stream early, so the downstream client received fewer replies than it sent, timed out and retried.
Return an empty string ("") on a miss instead, which matches the non-upstream get-stream handler.
AI-Generated: Uses Cursor
--- lib/hashserv/server.py | 20 ++++++++++++++++++-- lib/hashserv/tests.py | 37 +++++++++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 2 deletions(-)
lib/hashserv/server.py +18 -2
@@ -232,7 +232,15 @@ def permissions(*permissions, allow_anon=True, allow_self_service=False): class UpstreamQueue(object): UPSTREAM_NONCE = object() - def __init__(self, queue, get_local_result, send_upstream, get_upstream_result): + def __init__( + self, + logger, + queue, + get_local_result, + send_upstream, + get_upstream_result, + ): + self.logger = logger self.queue = queue self.pending = [] self.cond = asyncio.Condition()
@@ -256,6 +264,12 @@ class UpstreamQueue(object): if value is self.UPSTREAM_NONCE: value = await self.get_upstream_result(m) + if value is None: + self.logger.error( + "None is not allowed as a stream value. Terminating stream" + ) + return + await self.queue.put(value) finally: await self.queue.put(None)
@@ -504,10 +518,11 @@ class ServerClient(bb.asyncrpc.AsyncServerConnection): if unihash: method, taskhash = m.split() await self.server.backfill_queue.put((method, taskhash)) - return unihash + return unihash or "" queue = asyncio.Queue() upstream = UpstreamQueue( + self.logger, queue, get_local_result, send_upstream,
@@ -543,6 +558,7 @@ class ServerClient(bb.asyncrpc.AsyncServerConnection): queue = asyncio.Queue() upstream = UpstreamQueue( + self.logger, queue, get_local_result, stream.send_query,
lib/hashserv/tests.py +37 -0
@@ -606,6 +606,43 @@ class HashEquivalenceCommonTests(object): asyncio.run(check_unihashes()) + def test_upstream_get_stream_miss(self): + down_server = self.start_server(upstream=self.server.address) + down_client = self.start_client(down_server.address) + + # Two hashes present upstream (hits) + taskhash1 = '8aa96fcffb5831b3c2c0cb75f0431e3f8b20554a' + outhash1 = 'afe240a439959ce86f5e322f8c208e1fedefea9e813f2140c81af866cc9edf7e' + unihash1 = '5b521d8a12683086cc08bc2c6d94a7a2dcff17eba53b9911e145d51164689380' + self.client.report_unihash(taskhash1, self.METHOD, outhash1, unihash1) + + taskhash2 = 'e3da00593d6a7fb435c7e2114976c59c5fd6d561' + outhash2 = '1cf8713e645f491eb9c959d20b5cae1c47133a292626dda9b10709857cbe688a' + unihash2 = '7aebef07d66a8c0f92d0c4f65ec8b1fbb850a3693c53827b8774b64fa9a8a9fe' + self.client.report_unihash(taskhash2, self.METHOD, outhash2, unihash2) + + # Two taskhashes present nowhere (upstream misses) + miss1 = '0000000000000000000000000000000000000001' + miss2 = '0000000000000000000000000000000000000002' + + # Miss interleaved with hits: a miss must not truncate the stream + results = down_client.get_unihash_batch([ + (self.METHOD, miss1), + (self.METHOD, taskhash1), + (self.METHOD, miss2), + (self.METHOD, taskhash2), + ]) + self.assertEqual(results, [None, unihash1, None, unihash2]) + + # All-miss batch + self.assertEqual( + down_client.get_unihash_batch([(self.METHOD, miss1), (self.METHOD, miss2)]), + [None, None], + ) + + # Singular get-unihash miss + self.assertClientGetHash(down_client, miss1, None) + def test_unihash_exsits(self): taskhash, outhash, unihash = self.create_test_hash(self.client) self.assertTrue(self.client.unihash_exists(unihash))
Signature · 1 line
2.55.0