Add support for readonly upstream hash equivalence server
cover Add support for readonly upstream hash equivalence server
Adds support for a hash equivalence server to specify an "upstream" server where it will look for hash if they are not found in the local database. This upstream is "read-only" from the downstream servers perspective (e.g. it will only read hashes from the upstream server, not report equivalencies back to it), but this is not currently enforced by the upstream server.
The intended use case is to have a local hash equivalent server that uses an upstream server, such as the one used by some CI jobs generating a sstate cache.
There is some additional optimization that could be done; currently the downstream server will ask the upstream server for a hash on any local miss, which incurs a lot of latency because it must make a roundtrip query. Ideally, there could be multiple requests interleaved to reduce the overall latency, but bitbake currently only requests one hash at a time so this isn't possible.
Joshua Watt (2):
bitbake: hashserve: Add async client
bitbake: hashserve: Add support for readonly upstream
bitbake/lib/hashserv/__init__.py | 52 ++++--- bitbake/lib/hashserv/client.py | 243 +++++++++++++++++-------------- bitbake/lib/hashserv/server.py | 149 +++++++++++++++---- bitbake/lib/hashserv/tests.py | 147 +++++++++++++++---- 4 files changed, 404 insertions(+), 187 deletions(-)
Signature · 1 line
2.29.2
PATCH 1/2 bitbake: hashserve: Add async client
Adds support for create a client that operates using Python asynchronous I/O.
--- bitbake/lib/hashserv/__init__.py | 13 ++ bitbake/lib/hashserv/client.py | 238 +++++++++++++++++-------------- 2 files changed, 143 insertions(+), 108 deletions(-)
bitbake/lib/hashserv/__init__.py +13 -0
@@ -3,6 +3,7 @@ # SPDX-License-Identifier: GPL-2.0-only # +import asyncio from contextlib import closing import re import sqlite3
@@ -113,3 +114,15 @@ def create_client(addr): c.connect_tcp(*a) return c + +async def create_async_client(addr): + from . import client + c = client.AsyncClient() + + (typ, a) = parse_address(addr) + if typ == ADDR_TYPE_UNIX: + await c.connect_unix(*a) + else: + await c.connect_tcp(*a) + + return c
bitbake/lib/hashserv/client.py +130 -108
319 lines not on this page · raw message
Signature · 1 line
2.29.2
PATCH 2/2 bitbake: hashserve: Add support for readonly upstream
In reply to Joshua WattAdds support for an upstream server to be specified. The upstream server will be queried for equivalent hashes whenever a miss is found in the local server. If the server returns a match, it is merged into the local database. In order to keep the get stream queries as fast as possible since they are the critical path when bitbake is preparing the run queue, missing tasks provided by the server are not immediately pulled from the upstream server, but instead are put into a queue to be backfilled by a worker task later.
--- bitbake/lib/hashserv/__init__.py | 39 ++++---- bitbake/lib/hashserv/client.py | 19 ++-- bitbake/lib/hashserv/server.py | 149 ++++++++++++++++++++++++------- bitbake/lib/hashserv/tests.py | 147 +++++++++++++++++++++++------- 4 files changed, 268 insertions(+), 86 deletions(-)
bitbake/lib/hashserv/__init__.py +22 -17
@@ -22,6 +22,24 @@ ADDR_TYPE_TCP = 1 # is necessary DEFAULT_MAX_CHUNK = 32 * 1024 +TABLE_DEFINITION = ( + ("method", "TEXT NOT NULL"), + ("outhash", "TEXT NOT NULL"), + ("taskhash", "TEXT NOT NULL"), + ("unihash", "TEXT NOT NULL"), + ("created", "DATETIME"), + + # Optional fields + ("owner", "TEXT"), + ("PN", "TEXT"), + ("PV", "TEXT"), + ("PR", "TEXT"), + ("task", "TEXT"), + ("outhash_siginfo", "TEXT"), +) + +TABLE_COLUMNS = tuple(name for name, _ in TABLE_DEFINITION) + def setup_database(database, sync=True): db = sqlite3.connect(database) db.row_factory = sqlite3.Row
@@ -30,23 +48,10 @@ def setup_database(database, sync=True): cursor.execute(''' CREATE TABLE IF NOT EXISTS tasks_v2 ( id INTEGER PRIMARY KEY AUTOINCREMENT, - method TEXT NOT NULL, - outhash TEXT NOT NULL, - taskhash TEXT NOT NULL, - unihash TEXT NOT NULL, - created DATETIME, - - -- Optional fields - owner TEXT, - PN TEXT, - PV TEXT, - PR TEXT, - task TEXT, - outhash_siginfo TEXT, - + %s UNIQUE(method, outhash, taskhash) ) - ''') + ''' % " ".join("%s %s," % (name, typ) for name, typ in TABLE_DEFINITION)) cursor.execute('PRAGMA journal_mode = WAL') cursor.execute('PRAGMA synchronous = %s' % ('NORMAL' if sync else 'OFF'))
@@ -89,10 +94,10 @@ def chunkify(msg, max_chunk): yield "\n" -def create_server(addr, dbname, *, sync=True): +def create_server(addr, dbname, *, sync=True, upstream=None): from . import server db = setup_database(dbname, sync=sync) - s = server.Server(db) + s = server.Server(db, upstream=upstream) (typ, a) = parse_address(addr) if typ == ADDR_TYPE_UNIX:
bitbake/lib/hashserv/client.py +12 -7
@@ -178,18 +178,16 @@ class AsyncClient(object): await self._set_mode(self.MODE_NORMAL) return await self.send_message({"reset-stats": None}) + async def backfill_wait(self): + await self._set_mode(self.MODE_NORMAL) + return (await self.send_message({"backfill-wait": None}))["tasks"] + class Client(object): def __init__(self): self.client = AsyncClient() self.loop = asyncio.new_event_loop() - def get_wrapper(self, downcall): - def wrapper(*args, **kwargs): - return self.loop.run_until_complete(downcall(*args, **kwargs)) - - return wrapper - for call in ( "connect_tcp", "connect_unix",
@@ -200,9 +198,16 @@ class Client(object): "get_taskhash", "get_stats", "reset_stats", + "backfill_wait", ): downcall = getattr(self.client, call) - setattr(self, call, get_wrapper(self, downcall)) + setattr(self, call, self._get_downcall_wrapper(downcall)) + + def _get_downcall_wrapper(self, downcall): + def wrapper(*args, **kwargs): + return self.loop.run_until_complete(downcall(*args, **kwargs)) + + return wrapper @property def max_chunk(self):
bitbake/lib/hashserv/server.py +119 -30
264 lines not on this page · raw message
bitbake/lib/hashserv/tests.py +115 -32
202 lines not on this page · raw message
Signature · 1 line
2.29.2
Re: [bitbake-devel][PATCH 1/2] bitbake: hashserve: Add async client
In reply to Joshua WattHi Joshua,
I think this is causing issues when the path exceeds 108 chars:
step1b: WARNING: Error contacting Hash Equivalence Server unix:///home/pokybuild/yocto-autobuilder-new/yocto-worker/intel-core2- 32-altcfg-linux-yocto/build/build/hashserve.sock: AF_UNIX path too long
Thanks,
Anuj
Quoted 389 lines · Joshua Watt
Adds support for create a client that operates using Python asynchronous
I/O.Signed-off-by Joshua Watt <[email protected]>---
bitbake/lib/hashserv/__init__.py | 13 ++ bitbake/lib/hashserv/client.py | 238 +++++++++++++++++------------ --2 files changed, 143 insertions(+), 108 deletions(-)bitbake/lib/hashserv/__init__.py +0
-0@@ -3,6 +3,7 @@ # SPDX-License-Identifier: GPL-2.0-only #
+import asyncio
from contextlib import closing
import re
import sqlite3bitbake/lib/hashserv/__init__.py +0
-0@@ -113,3 +114,15 @@ def create_client(addr): c.connect_tcp(*a)
return cbitbake/lib/hashserv/__init__.py +12
-0+ +async def create_async_client(addr): + from . import client + c = client.AsyncClient() + + (typ, a) = parse_address(addr) + if typ == ADDR_TYPE_UNIX: + await c.connect_unix(*a) + else: + await c.connect_tcp(*a) + + return cbitbake/lib/hashserv/client.py +0
-0@@ -3,189 +3,211 @@ # SPDX-License-Identifier: GPL-2.0-only #
+import asyncio
import json
import logging
import socket
import osbitbake/lib/hashserv/client.py +1
-1-from . import chunkify, DEFAULT_MAX_CHUNK +from . import chunkify, DEFAULT_MAX_CHUNK, create_async_client
bitbake/lib/hashserv/client.py +1
-1-logger = logging.getLogger('hashserv.client') +logger = logging.getLogger("hashserv.client")
class HashConnectionError(Exception):
passbitbake/lib/hashserv/client.py +1
-1-class Client(object): +class AsyncClient(object):
MODE_NORMAL= 0
MODE_GET_STREAM= 1
def__init__(self):
- self._socket = None
self.reader = None
self.writer = None
self.mode = self.MODE_NORMAL
self.max_chunk =DEFAULT_MAX_CHUNK
- defconnect_tcp(self, address, port):
- defconnect_sock():
- s =socket.create_connection((address, port))-
-s.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, 1)-s.setsockopt(socket.SOL_TCP, socket.TCP_QUICKACK, 1)-s.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)- return s
+ async defconnect_tcp(self, address, port):
+ async defconnect_sock():
+ return awaitasyncio.open_connection(address, port)
self._connect_sock =connect_sock
- defconnect_unix(self, path):
- defconnect_sock():
- s =socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)- #AF_UNIXhas path length issues so chdir here to workaround
- cwd =os.getcwd()
- try:
- os.chdir(os.path.dirname(path))
- s.connect(os.path.basename(path))
- finally:
-os.chdir(cwd)
- return s
+ async defconnect_unix(self, path):
+ async defconnect_sock():
+ return awaitasyncio.open_unix_connection(path)
self._connect_sock =connect_sock
- def connect(self):
- if self._socket is None:
- self._socket =self._connect_sock()
-
- self.reader =self._socket.makefile('r', encoding='utf- 8')
- self.writer =self._socket.makefile('w', encoding='utf- 8')
+ async def_connect(self):
+ if self.reader is None or self.writer is None:
+ (self.reader, self.writer) = awaitself._connect_sock()
-self.writer.write('OEHASHEQUIV 1.1\n\n')
-self.writer.flush()
+self.writer.write("OEHASHEQUIV 1.1\n\n".encode("utf-8"))+ awaitself.writer.drain()
- # Restore mode if the socket is being re-createdcur_mode= self.mode
self.mode = self.MODE_NORMAL
-self._set_mode(cur_mode)
+ awaitself._set_mode(cur_mode)
- return self._socket
+ async def close(self):
+ self.reader = None
- def close(self):
- if self._socket is not None:
-self._socket.close()
- self._socket = None
- self.reader = None
+ if self.writer is not None:
+self.writer.close()
self.writer = None
- def_send_wrapper(self, proc):
+ async def_send_wrapper(self, proc):
count = 0
while True:
try:
-self.connect()
- returnproc()
- except (OSError, HashConnectionError, json.JSONDecodeError, UnicodeDecodeError) as e:
-logger.warning('Error talking to server: %s' % e)+ awaitself._connect()
+ return awaitproc()
+ except (
+ OSError,
+ HashConnectionError,
+ json.JSONDecodeError,
+ UnicodeDecodeError,
+ ) as e:
+logger.warning("Error talking to server: %s" % e)if count >= 3:
if notisinstance(e, HashConnectionError):
raise HashConnectionError(str(e)) raise e
-self.close()
+ awaitself.close()
count += 1
- defsend_message(self, msg):
- defget_line():
- line =self.reader.readline()
+ async defsend_message(self, msg):
+ async defget_line():
+ line = awaitself.reader.readline()
if not line:
- raise HashConnectionError('Connection closed') + raiseHashConnectionError("Connection closed")+
+ line =line.decode("utf-8")
- if notline.endswith('\n'):
- raise HashConnectionError('Bad message %r' % message)
+ if notline.endswith("\n"):
+ raiseHashConnectionError("Bad message %r" % message)
return line
- defproc():
+ async defproc():
for c inchunkify(json.dumps(msg), self.max_chunk):
-self.writer.write(c)
-self.writer.flush()
+ self.writer.write(c.encode("utf-8"))
+ awaitself.writer.drain()
- l =get_line()
+ l = awaitget_line()
m =json.loads(l)
- if 'chunk-stream' in m:
+ if "chunk-stream" in m:
lines = []
while True:
- l = get_line().rstrip('\n')
+ l = (await get_line()).rstrip("\n") if not l:
break
lines.append(l)
- m = json.loads(''.join(lines))
+ m = json.loads("".join(lines))
return m
- returnself._send_wrapper(proc)
+ return awaitself._send_wrapper(proc)
- defsend_stream(self, msg):
- defproc():
-self.writer.write("%s\n" % msg)
-self.writer.flush()
- l =self.reader.readline()
+ async defsend_stream(self, msg):
+ async defproc():
+self.writer.write(("%s\n" % msg).encode("utf-8"))+ awaitself.writer.drain()
+ l = awaitself.reader.readline()
if not l:
- raise HashConnectionError('Connection closed') - returnl.rstrip()
+ raiseHashConnectionError("Connection closed")+ return l.decode("utf-8").rstrip()
- returnself._send_wrapper(proc)
+ return awaitself._send_wrapper(proc)
- def_set_mode(self, new_mode):
+ async def_set_mode(self, new_mode):
ifnew_mode== self.MODE_NORMAL and self.mode == self.MODE_GET_STREAM:
- r =self.send_stream('END')
- if r != 'ok':
- raise HashConnectionError('Bad response from server %r' % r)
+ r = awaitself.send_stream("END")
+ if r != "ok":
+ raiseHashConnectionError("Bad response from server %r" % r)
elifnew_mode== self.MODE_GET_STREAM and self.mode == self.MODE_NORMAL:
- r =self.send_message({'get-stream': None})
- if r != 'ok':
- raise HashConnectionError('Bad response from server %r' % r)
+ r = awaitself.send_message({"get-stream": None})+ if r != "ok":
+ raiseHashConnectionError("Bad response from server %r" % r)
elifnew_mode!= self.mode:
- raiseException('Undefined mode transition %r -> %r' % (self.mode, new_mode))
+ raise Exception(
+ "Undefined mode transition %r -> %r" % (self.mode,new_mode)
+ )
self.mode =new_mode
- defget_unihash(self, method, taskhash):
-self._set_mode(self.MODE_GET_STREAM)
- r =self.send_stream('%s %s' % (method, taskhash))+ async defget_unihash(self, method, taskhash):
+ awaitself._set_mode(self.MODE_GET_STREAM)
+ r = awaitself.send_stream("%s %s" % (method, taskhash))if not r:
return None
return r
- defreport_unihash(self, taskhash, method, outhash, unihash, extra={}):
-self._set_mode(self.MODE_NORMAL)
+ async defreport_unihash(self, taskhash, method, outhash, unihash, extra={}):
+ awaitself._set_mode(self.MODE_NORMAL)
m =extra.copy()
- m['taskhash'] = taskhash
- m['method'] = method
- m['outhash'] = outhash
- m['unihash'] = unihash
- returnself.send_message({'report': m})
-
- defreport_unihash_equiv(self, taskhash, method, unihash, extra={}):
-self._set_mode(self.MODE_NORMAL)
+ m["taskhash"] = taskhash
+ m["method"] = method
+ m["outhash"] = outhash
+ m["unihash"] = unihash
+ return awaitself.send_message({"report": m})
+
+ async defreport_unihash_equiv(self, taskhash, method, unihash, extra={}):
+ awaitself._set_mode(self.MODE_NORMAL)
m =extra.copy()
- m['taskhash'] = taskhash
- m['method'] = method
- m['unihash'] = unihash
- returnself.send_message({'report-equiv': m})bitbake/lib/hashserv/client.py +6
-16- - def get_taskhash(self, method, taskhash, all_properties=False): - self._set_mode(self.MODE_NORMAL) - return self.send_message({'get': { - 'taskhash': taskhash, - 'method': method, - 'all': all_properties - }}) - - def get_stats(self): - self._set_mode(self.MODE_NORMAL) - return self.send_message({'get-stats': None}) - - def reset_stats(self): - self._set_mode(self.MODE_NORMAL) - return self.send_message({'reset-stats': None}) + m["taskhash"] = taskhash + m["method"] = method + m["unihash"] = unihash + return await self.send_message({"report-equiv": m}) + + async def get_taskhash(self, method, taskhash,all_properties=False):
+ awaitself._set_mode(self.MODE_NORMAL)
+ return await self.send_message(
+ {"get": {"taskhash": taskhash, "method": method, "all":all_properties}}
+ )bitbake/lib/hashserv/client.py +18
-0+ + async def get_stats(self): + await self._set_mode(self.MODE_NORMAL) + return await self.send_message({"get-stats": None}) + + async def reset_stats(self): + await self._set_mode(self.MODE_NORMAL) + return await self.send_message({"reset-stats": None}) + + +class Client(object): + def __init__(self): + self.client = AsyncClient() + self.loop = asyncio.new_event_loop() + + def get_wrapper(self, downcall): + def wrapper(*args, **kwargs): + return self.loop.run_until_complete(downcall(*args,**kwargs))
bitbake/lib/hashserv/client.py +24
-0+ + return wrapper + + for call in ( + "connect_tcp", + "connect_unix", + "close", + "get_unihash", + "report_unihash", + "report_unihash_equiv", + "get_taskhash", + "get_stats", + "reset_stats", + ): + downcall = getattr(self.client, call) + setattr(self, call, get_wrapper(self, downcall)) + + @property + def max_chunk(self): + return self.client.max_chunk + + @max_chunk.setter + def max_chunk(self, value): + self.client.max_chunk = value
Re: [bitbake-devel][PATCH 1/2] bitbake: hashserve: Add async client
Quoted 8 lines · Anuj Mittal
Hi Joshua,
I think this is causing issues when the path exceeds 108 chars:
step1b: WARNING: Error contacting Hash Equivalence Server unix:///home/pokybuild/yocto-autobuilder-new/yocto-worker/intel-core2- 32-altcfg-linux-yocto/build/build/hashserve.sock:
AF_UNIXpath too long
Odd. I would have expected that limit to be a problem with the old (non-async) client as well
Quoted 399 lines
Thanks,
Anuj
Quoted 389 lines · Joshua Watt
Adds support for create a client that operates using Python asynchronous
I/O.Signed-off-by Joshua Watt <[email protected]>--- bitbake/lib/hashserv/__init__.py | 13 ++ bitbake/lib/hashserv/client.py | 238 +++++++++++++++++------------ -- 2 files changed, 143 insertions(+), 108 deletions(-)bitbake/lib/hashserv/__init__.py +13
-0@@ -3,6 +3,7 @@ # SPDX-License-Identifier: GPL-2.0-only # +import asyncio from contextlib import closing import re import sqlite3@@ -113,3 +114,15 @@ def create_client(addr): c.connect_tcp(*a) return c + +async def create_async_client(addr): + from . import client + c = client.AsyncClient() + + (typ, a) = parse_address(addr) + if typ == ADDR_TYPE_UNIX: + await c.connect_unix(*a) + else: + await c.connect_tcp(*a) + + return cbitbake/lib/hashserv/client.py +7
-16@@ -3,189 +3,211 @@ # SPDX-License-Identifier: GPL-2.0-only # +import asyncio import json import logging import socket import os -from . import chunkify, DEFAULT_MAX_CHUNK +from . import chunkify, DEFAULT_MAX_CHUNK, create_async_client -logger = logging.getLogger('hashserv.client') +logger = logging.getLogger("hashserv.client") class HashConnectionError(Exception): pass -class Client(object): +class AsyncClient(object): MODE_NORMAL = 0 MODE_GET_STREAM = 1 def __init__(self): - self._socket = None self.reader = None self.writer = None self.mode = self.MODE_NORMAL self.max_chunk = DEFAULT_MAX_CHUNK - def connect_tcp(self, address, port): - def connect_sock(): - s = socket.create_connection((address, port)) - - s.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, 1) - s.setsockopt(socket.SOL_TCP, socket.TCP_QUICKACK, 1) - s.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1) - return s + async def connect_tcp(self, address, port): + async def connect_sock(): + return await asyncio.open_connection(address, port) self._connect_sock = connect_sock - def connect_unix(self, path): - def connect_sock(): - s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) - # AF_UNIX has path length issues so chdir here toworkaround
bitbake/lib/hashserv/client.py +3
-12- cwd = os.getcwd() - try: - os.chdir(os.path.dirname(path)) - s.connect(os.path.basename(path)) - finally: - os.chdir(cwd) - return s + async def connect_unix(self, path): + async def connect_sock(): + return await asyncio.open_unix_connection(path) self._connect_sock = connect_sock - def connect(self): - if self._socket is None: - self._socket = self._connect_sock() - - self.reader = self._socket.makefile('r', encoding='utf-8')
- self.writer = self._socket.makefile('w', encoding='utf-
8')bitbake/lib/hashserv/client.py +11
-14+ async def _connect(self): + if self.reader is None or self.writer is None: + (self.reader, self.writer) = await self._connect_sock() - self.writer.write('OEHASHEQUIV 1.1\n\n') - self.writer.flush() + self.writer.write("OEHASHEQUIV 1.1\n\n".encode("utf-8")) + await self.writer.drain() - # Restore mode if the socket is being re-created cur_mode = self.mode self.mode = self.MODE_NORMAL - self._set_mode(cur_mode) + await self._set_mode(cur_mode) - return self._socket + async def close(self): + self.reader = None - def close(self): - if self._socket is not None: - self._socket.close() - self._socket = None - self.reader = None + if self.writer is not None: + self.writer.close() self.writer = None - def _send_wrapper(self, proc): + async def _send_wrapper(self, proc): count = 0 while True: try: - self.connect() - return proc() - except (OSError, HashConnectionError,json.JSONDecodeError, UnicodeDecodeError) as e:
bitbake/lib/hashserv/client.py +16
-8- logger.warning('Error talking to server: %s' % e) + await self._connect() + return await proc() + except ( + OSError, + HashConnectionError, + json.JSONDecodeError, + UnicodeDecodeError, + ) as e: + logger.warning("Error talking to server: %s" % e) if count >= 3: if not isinstance(e, HashConnectionError): raise HashConnectionError(str(e)) raise e - self.close() + await self.close() count += 1 - def send_message(self, msg): - def get_line(): - line = self.reader.readline() + async def send_message(self, msg): + async def get_line(): + line = await self.reader.readline() if not line: - raise HashConnectionError('Connection closed') + raise HashConnectionError("Connection closed") + + line = line.decode("utf-8") - if not line.endswith('\n'): - raise HashConnectionError('Bad message %r' %message)
bitbake/lib/hashserv/client.py +2
-0+ if not line.endswith("\n"): + raise HashConnectionError("Bad message %r" %message)
return line
bitbake/lib/hashserv/client.py +17
-17- def proc(): + async def proc(): for c in chunkify(json.dumps(msg), self.max_chunk): - self.writer.write(c) - self.writer.flush() + self.writer.write(c.encode("utf-8")) + await self.writer.drain() - l = get_line() + l = await get_line() m = json.loads(l) - if 'chunk-stream' in m: + if "chunk-stream" in m: lines = [] while True: - l = get_line().rstrip('\n') + l = (await get_line()).rstrip("\n") if not l: break lines.append(l) - m = json.loads(''.join(lines)) + m = json.loads("".join(lines)) return m - return self._send_wrapper(proc) + return await self._send_wrapper(proc) - def send_stream(self, msg): - def proc(): - self.writer.write("%s\n" % msg) - self.writer.flush() - l = self.reader.readline() + async def send_stream(self, msg): + async def proc(): + self.writer.write(("%s\n" % msg).encode("utf-8")) + await self.writer.drain() + l = await self.reader.readline() if not l: - raise HashConnectionError('Connection closed') - return l.rstrip() + raise HashConnectionError("Connection closed") + return l.decode("utf-8").rstrip() - return self._send_wrapper(proc) + return await self._send_wrapper(proc) - def _set_mode(self, new_mode): + async def _set_mode(self, new_mode): if new_mode == self.MODE_NORMAL and self.mode ==self.MODE_GET_STREAM:
bitbake/lib/hashserv/client.py +0
-3- r = self.send_stream('END') - if r != 'ok': - raise HashConnectionError('Bad response from server%r' % r)
bitbake/lib/hashserv/client.py +3
-0+ r = await self.send_stream("END") + if r != "ok": + raise HashConnectionError("Bad response from server%r" % r)
elif new_mode == self.MODE_GET_STREAM and self.mode ==self.MODE_NORMAL:
bitbake/lib/hashserv/client.py +0
-3- r = self.send_message({'get-stream': None}) - if r != 'ok': - raise HashConnectionError('Bad response from server%r' % r)
bitbake/lib/hashserv/client.py +3
-0+ r = await self.send_message({"get-stream": None}) + if r != "ok": + raise HashConnectionError("Bad response from server%r" % r)
elif new_mode != self.mode:- raise Exception('Undefined mode transition %r -> %r' %
(self.mode,new_mode))bitbake/lib/hashserv/client.py +2
-0+ raise Exception( + "Undefined mode transition %r -> %r" % (self.mode,
new_mode)bitbake/lib/hashserv/client.py +4
-4+ ) self.mode = new_mode - def get_unihash(self, method, taskhash): - self._set_mode(self.MODE_GET_STREAM) - r = self.send_stream('%s %s' % (method, taskhash)) + async def get_unihash(self, method, taskhash): + await self._set_mode(self.MODE_GET_STREAM) + r = await self.send_stream("%s %s" % (method, taskhash)) if not r: return None return r - def report_unihash(self, taskhash, method, outhash, unihash,extra={}):
bitbake/lib/hashserv/client.py +1
-1- self._set_mode(self.MODE_NORMAL) + async def report_unihash(self, taskhash, method, outhash,unihash, extra={}):
bitbake/lib/hashserv/client.py +1
-7+ await self._set_mode(self.MODE_NORMAL) m = extra.copy() - m['taskhash'] = taskhash - m['method'] = method - m['outhash'] = outhash - m['unihash'] = unihash - return self.send_message({'report': m}) - - def report_unihash_equiv(self, taskhash, method, unihash,extra={}):
bitbake/lib/hashserv/client.py +7
-1- self._set_mode(self.MODE_NORMAL) + m["taskhash"] = taskhash + m["method"] = method + m["outhash"] = outhash + m["unihash"] = unihash + return await self.send_message({"report": m}) + + async def report_unihash_equiv(self, taskhash, method, unihash,extra={}):
bitbake/lib/hashserv/client.py +7
-20+ await self._set_mode(self.MODE_NORMAL) m = extra.copy() - m['taskhash'] = taskhash - m['method'] = method - m['unihash'] = unihash - return self.send_message({'report-equiv': m}) - - def get_taskhash(self, method, taskhash, all_properties=False): - self._set_mode(self.MODE_NORMAL) - return self.send_message({'get': { - 'taskhash': taskhash, - 'method': method, - 'all': all_properties - }}) - - def get_stats(self): - self._set_mode(self.MODE_NORMAL) - return self.send_message({'get-stats': None}) - - def reset_stats(self): - self._set_mode(self.MODE_NORMAL) - return self.send_message({'reset-stats': None}) + m["taskhash"] = taskhash + m["method"] = method + m["unihash"] = unihash + return await self.send_message({"report-equiv": m}) + + async def get_taskhash(self, method, taskhash,all_properties=False):
bitbake/lib/hashserv/client.py +3
-0+ await self._set_mode(self.MODE_NORMAL) + return await self.send_message( + {"get": {"taskhash": taskhash, "method": method, "all":
all_properties}}bitbake/lib/hashserv/client.py +19
-0+ ) + + async def get_stats(self): + await self._set_mode(self.MODE_NORMAL) + return await self.send_message({"get-stats": None}) + + async def reset_stats(self): + await self._set_mode(self.MODE_NORMAL) + return await self.send_message({"reset-stats": None}) + + +class Client(object): + def __init__(self): + self.client = AsyncClient() + self.loop = asyncio.new_event_loop() + + def get_wrapper(self, downcall): + def wrapper(*args, **kwargs): + return self.loop.run_until_complete(downcall(*args,**kwargs))
bitbake/lib/hashserv/client.py +24
-0+ + return wrapper + + for call in ( + "connect_tcp", + "connect_unix", + "close", + "get_unihash", + "report_unihash", + "report_unihash_equiv", + "get_taskhash", + "get_stats", + "reset_stats", + ): + downcall = getattr(self.client, call) + setattr(self, call, get_wrapper(self, downcall)) + + @property + def max_chunk(self): + return self.client.max_chunk + + @max_chunk.setter + def max_chunk(self, value): + self.client.max_chunk = value
Re: [bitbake-devel][PATCH 1/2] bitbake: hashserve: Add async client
Quoted 8 lines · Joshua Watt
Quoted 4 lines · Anuj Mittal
Hi Joshua,
I think this is causing issues when the path exceeds 108 chars:
Quoted 9 lines
Quoted 5 lines
step1b: WARNING: Error contacting Hash Equivalence Server unix:///home/pokybuild/yocto-autobuilder-new/yocto-worker/intel- core2- 32-altcfg-linux-yocto/build/build/hashserve.sock:
AF_UNIXpath too longOdd. I would have expected that limit to be a problem with the old (non-async) client as well
I see a comment in the old code that says:
Quoted 2 lines
#
AF_UNIXhas path length issues so chdir here to
workaround
So perhaps the old client was explicitly handling this case?
Thanks,
Anuj