hashserv: Chunkify large messages
PATCH 1/1 hashserv: Chunkify large messages
The hash equivalence client and server can occasionally send messages that are too large for the server to fit in the receive buffer (64 KB). To prevent this, support is added to the protocol to "chunkify" the stream and break it up into manageable pieces that the server can each side can back together.
Ideally, this would be negotiated by the client and server, but it's currently hard coded to 32 KB to prevent the round-trip delay.
--- bitbake/lib/hashserv/__init__.py | 22 +++++++ bitbake/lib/hashserv/client.py | 43 ++++++++++--- bitbake/lib/hashserv/server.py | 105 +++++++++++++++++++++---------- bitbake/lib/hashserv/tests.py | 27 +++++++- 4 files changed, 154 insertions(+), 43 deletions(-)
bitbake/lib/hashserv/__init__.py +22 -0
@@ -6,12 +6,20 @@ from contextlib import closing import re import sqlite3 +import itertools +import json UNIX_PREFIX = "unix://" ADDR_TYPE_UNIX = 0 ADDR_TYPE_TCP = 1 +# The Python async server defaults to a 64K receive buffer, so we hardcode our +# maximum chunk size. It would be better if the client and server reported to +# each other what the maximum chunk sizes were, but that will slow down the +# connection setup with a round trip delay so I'd rather not do that unless it +# is necessary +DEFAULT_MAX_CHUNK = 32 * 1024 def setup_database(database, sync=True): db = sqlite3.connect(database)
@@ -66,6 +74,20 @@ def parse_address(addr): return (ADDR_TYPE_TCP, (host, int(port))) +def chunkify(msg, max_chunk): + if len(msg) < max_chunk - 1: + yield ''.join((msg, "\n")) + else: + yield ''.join((json.dumps({ + 'chunk-stream': None + }), "\n")) + + args = [iter(msg)] * (max_chunk - 1) + for m in map(''.join, itertools.zip_longest(*args, fillvalue='')): + yield ''.join(itertools.chain(m, "\n")) + yield "\n" + + def create_server(addr, dbname, *, sync=True): from . import server db = setup_database(dbname, sync=sync)
bitbake/lib/hashserv/client.py +35 -8
78 lines not on this page · raw message
bitbake/lib/hashserv/server.py +72 -33
177 lines not on this page · raw message
bitbake/lib/hashserv/tests.py +25 -2
@@ -5,7 +5,7 @@ # SPDX-License-Identifier: GPL-2.0-only # -from . import create_server, create_client +from . import create_server, create_client, DEFAULT_MAX_CHUNK import hashlib import logging import multiprocessing
@@ -99,6 +99,30 @@ class TestHashEquivalenceServer(object): result = self.client.get_unihash(self.METHOD, taskhash) self.assertEqual(result, unihash) + def test_huge_message(self): + # Simple test that hashes can be created + taskhash = 'c665584ee6817aa99edfc77a44dd853828279370' + outhash = '3c979c3db45c569f51ab7626a4651074be3a9d11a84b1db076f5b14f7d39db44' + unihash = '90e9bc1d1f094c51824adca7f8ea79a048d68824' + + result = self.client.get_unihash(self.METHOD, taskhash) + self.assertIsNone(result, msg='Found unexpected task, %r' % result) + + siginfo = "0" * (self.client.max_chunk * 4) + + result = self.client.report_unihash(taskhash, self.METHOD, outhash, unihash, { + 'outhash_siginfo': siginfo + }) + self.assertEqual(result['unihash'], unihash, 'Server returned bad unihash') + + result = self.client.get_taskhash(self.METHOD, taskhash, True) + print(result) + self.assertEqual(result['taskhash'], taskhash) + self.assertEqual(result['unihash'], unihash) + self.assertEqual(result['method'], self.METHOD) + self.assertEqual(result['outhash'], outhash) + self.assertEqual(result['outhash_siginfo'], siginfo) + def test_stress(self): def query_server(failures): client = Client(self.server.address)
@@ -136,7 +160,6 @@ class TestHashEquivalenceUnixServer(TestHashEquivalenceServer, unittest.TestCase def get_server_addr(self): return "unix://" + os.path.join(self.temp_dir.name, 'sock') - class TestHashEquivalenceTCPServer(TestHashEquivalenceServer, unittest.TestCase): def get_server_addr(self): return "localhost:0"
Signature · 1 line
2.27.0
PATCH v2 1/1 hashserv: Chunkify large messages
The hash equivalence client and server can occasionally send messages that are too large for the server to fit in the receive buffer (64 KB). To prevent this, support is added to the protocol to "chunkify" the stream and break it up into manageable pieces that the server can each side can back together.
Ideally, this would be negotiated by the client and server, but it's currently hard coded to 32 KB to prevent the round-trip delay.
---
V2:
Removed some debugging information I left in accidently
bitbake/lib/hashserv/__init__.py | 22 +++++++ bitbake/lib/hashserv/client.py | 43 ++++++++++--- bitbake/lib/hashserv/server.py | 105 +++++++++++++++++++++---------- bitbake/lib/hashserv/tests.py | 23 +++++++ 4 files changed, 152 insertions(+), 41 deletions(-)
bitbake/lib/hashserv/__init__.py +22 -0
@@ -6,12 +6,20 @@ from contextlib import closing import re import sqlite3 +import itertools +import json UNIX_PREFIX = "unix://" ADDR_TYPE_UNIX = 0 ADDR_TYPE_TCP = 1 +# The Python async server defaults to a 64K receive buffer, so we hardcode our +# maximum chunk size. It would be better if the client and server reported to +# each other what the maximum chunk sizes were, but that will slow down the +# connection setup with a round trip delay so I'd rather not do that unless it +# is necessary +DEFAULT_MAX_CHUNK = 32 * 1024 def setup_database(database, sync=True): db = sqlite3.connect(database)
@@ -66,6 +74,20 @@ def parse_address(addr): return (ADDR_TYPE_TCP, (host, int(port))) +def chunkify(msg, max_chunk): + if len(msg) < max_chunk - 1: + yield ''.join((msg, "\n")) + else: + yield ''.join((json.dumps({ + 'chunk-stream': None + }), "\n")) + + args = [iter(msg)] * (max_chunk - 1) + for m in map(''.join, itertools.zip_longest(*args, fillvalue='')): + yield ''.join(itertools.chain(m, "\n")) + yield "\n" + + def create_server(addr, dbname, *, sync=True): from . import server db = setup_database(dbname, sync=sync)
bitbake/lib/hashserv/client.py +35 -8
78 lines not on this page · raw message
bitbake/lib/hashserv/server.py +72 -33
177 lines not on this page · raw message
bitbake/lib/hashserv/tests.py +23 -0
@@ -99,6 +99,29 @@ class TestHashEquivalenceServer(object): result = self.client.get_unihash(self.METHOD, taskhash) self.assertEqual(result, unihash) + def test_huge_message(self): + # Simple test that hashes can be created + taskhash = 'c665584ee6817aa99edfc77a44dd853828279370' + outhash = '3c979c3db45c569f51ab7626a4651074be3a9d11a84b1db076f5b14f7d39db44' + unihash = '90e9bc1d1f094c51824adca7f8ea79a048d68824' + + result = self.client.get_unihash(self.METHOD, taskhash) + self.assertIsNone(result, msg='Found unexpected task, %r' % result) + + siginfo = "0" * (self.client.max_chunk * 4) + + result = self.client.report_unihash(taskhash, self.METHOD, outhash, unihash, { + 'outhash_siginfo': siginfo + }) + self.assertEqual(result['unihash'], unihash, 'Server returned bad unihash') + + result = self.client.get_taskhash(self.METHOD, taskhash, True) + self.assertEqual(result['taskhash'], taskhash) + self.assertEqual(result['unihash'], unihash) + self.assertEqual(result['method'], self.METHOD) + self.assertEqual(result['outhash'], outhash) + self.assertEqual(result['outhash_siginfo'], siginfo) + def test_stress(self): def query_server(failures): client = Client(self.server.address)
Signature · 1 line
2.27.0
Quoted 18 lines · Joshua Watt
The hash equivalence client and server can occasionally send messages that are too large for the server to fit in the receive buffer (64 KB).
To prevent this, support is added to the protocol to "chunkify" the stream and break it up into manageable pieces that the server can each
side can back together.Ideally, this would be negotiated by the client and server, but it's currently hard coded to 32 KB to prevent the round-trip delay.
Signed-off-by Joshua Watt <[email protected]>---
V2:
Removed some debugging information I left in accidently
Quoted 6 lines
bitbake/lib/hashserv/__init__.py | 22 +++++++ bitbake/lib/hashserv/client.py | 43 ++++++++++--- bitbake/lib/hashserv/server.py | 105 +++++++++++++++++++++------ ---- bitbake/lib/hashserv/tests.py | 23 +++++++ 4 files changed, 152 insertions(+), 41 deletions(-)
I naively ran this on the autobuilder, forgetting the server is central and would need updating manually.
How backwards/forward compatible is this change?
With new client and old server we see tons of warnings, e.g.:
https://autobuilder.yoctoproject.org/typhoon/#/builders/61/builds/2068/steps/8/logs/warnings
I know I can upgrade the server but then will it work with old clients (e.g. our LTS)? Do we need to warn users instead of warnings like this?
Cheers,
Richard
Quoted 30 lines · Richard Purdie
Quoted 18 lines · Joshua Watt
The hash equivalence client and server can occasionally send messages that are too large for the server to fit in the receive buffer (64 KB).
To prevent this, support is added to the protocol to "chunkify" the stream and break it up into manageable pieces that the server can each
side can back together.Ideally, this would be negotiated by the client and server, but it's currently hard coded to 32 KB to prevent the round-trip delay.
Signed-off-by Joshua Watt <[email protected]>---
V2:
Removed some debugging information I left in accidentlyQuoted 6 lines
bitbake/lib/hashserv/__init__.py | 22 +++++++ bitbake/lib/hashserv/client.py | 43 ++++++++++--- bitbake/lib/hashserv/server.py | 105 +++++++++++++++++++++------ ---- bitbake/lib/hashserv/tests.py | 23 +++++++ 4 files changed, 152 insertions(+), 41 deletions(-)I naively ran this on the autobuilder, forgetting the server is central and would need updating manually.
How backwards/forward compatible is this change?
Quoted 7 lines
With new client and old server we see tons of warnings, e.g.:
https://autobuilder.yoctoproject.org/typhoon/#/builders/61/builds/2068/steps/8/logs/warnings
I know I can upgrade the server but then will it work with old clients (e.g. our LTS)? Do we need to warn users instead of warnings like this?
The onus for compatibility is always on the server, so the client says what version it is, and the server decides if it can support that version or not. If not it closes the connection, as you're seeing here. The change is backward compatible from the servers perspective (that is, a new server can support old clients), but not the other way around (e.g. an old server has to reject new clients because it doesn't understand the chunking protocol).
Quoted 5 lines
Cheers,
Richard