Re-implement prserv on top of asyncrpc
v6 cover Re-implement prserv on top of asyncrpc
These changes replace the old XML-based RPC system in prserv with the new asyncrpc implementation originally used by hashserv, and add a read-only mode to match the hash equivalency server's support.
Note that an additional change to the PR export support code in oe-core is being sent to its mailing list. That change is also required for any serious testing, as it resolves another hang issue in the PR export selftest.
Changes from v5:
* Added new patch from Joshua Watt to shift asyncio loop creation into the new server processes. This fixes one of the hangs seen on the autobuilder.
* New patch to always create and set the asyncio loop as default in both the asyncrpc client and server initialization. In my testing this resolves a couple of different hang issues seen in the selftests.
* Task cleanup on shutdown and exception handler patches from Paul dropped from stack, as testing showed the hang issues were not helped by the changes.
* Patch to rebase PR server on asyncrpc code tweaked to split the new client code into a separate source file as is done in hashserv.
Changes from v4:
* Patch 2 updated to conditionalize all_tasks/current_task usage from asyncio vs asyncio.Task based on Python version. This fixes running against Python 3.9 where they were finally removed from asyncio.Task.
Changes from v3:
* Scott Murray taking over upstreaming effort from Paul Barker.
* Dropped patches which are currently applied to master-next, this series should be applied on top of the current master-next branch.
* Patches 2-4 updated by Scott Murray to rebase on top of 3983643 ("bitbake: asyncrpc: Catch early SIGTERM").
* Read-only PR server support patch added to stack to get it into the review process.
Joshua Watt (1):
bitbake: asyncrpc: Defer all asyncio to child process
Paul Barker (2):
prserv: Replace XML RPC with modern asyncrpc implementation
prserv: Add read-only mode
Scott Murray (1):
bitbake: asyncrpc: always create new asyncio loops
bin/bitbake-prserv | 4 +- lib/bb/asyncrpc/client.py | 10 ++ lib/bb/asyncrpc/serv.py | 134 +++++++++++++------- lib/hashserv/server.py | 4 +- lib/prserv/client.py | 48 ++++++++ lib/prserv/db.py | 65 +++++++--- lib/prserv/serv.py | 248 ++++++++++++++++++-------------------- 7 files changed, 319 insertions(+), 194 deletions(-) create mode 100644 lib/prserv/client.py
Signature · 1 line
2.31.1
PATCH v6 1/4 bitbake: asyncrpc: Defer all asyncio to child process
From: Joshua Watt <[email protected]>
Reworks the async I/O API so that the async loop is only created in the child process. This requires deferring the creation of the server until the child process and a queue to transfer the bound address back to the parent process
--- lib/bb/asyncrpc/serv.py | 118 ++++++++++++++++++++++++---------------- lib/hashserv/server.py | 4 +- 2 files changed, 74 insertions(+), 48 deletions(-)
lib/bb/asyncrpc/serv.py +72 -46
@@ -131,53 +131,58 @@ class AsyncServerConnection(object): class AsyncServer(object): - def __init__(self, logger, loop=None): - if loop is None: - self.loop = asyncio.new_event_loop() - self.close_loop = True - else: - self.loop = loop - self.close_loop = False - + def __init__(self, logger): self._cleanup_socket = None self.logger = logger + self.start = None + self.address = None + + @property + def loop(self): + return asyncio.get_event_loop() def start_tcp_server(self, host, port): - self.server = self.loop.run_until_complete( - asyncio.start_server(self.handle_client, host, port, loop=self.loop) - ) - - for s in self.server.sockets: - self.logger.debug('Listening on %r' % (s.getsockname(),)) - # Newer python does this automatically. Do it manually here for - # maximum compatibility - s.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, 1) - s.setsockopt(socket.SOL_TCP, socket.TCP_QUICKACK, 1) - - name = self.server.sockets[0].getsockname() - if self.server.sockets[0].family == socket.AF_INET6: - self.address = "[%s]:%d" % (name[0], name[1]) - else: - self.address = "%s:%d" % (name[0], name[1]) + def start_tcp(): + self.server = self.loop.run_until_complete( + asyncio.start_server(self.handle_client, host, port) + ) + + for s in self.server.sockets: + self.logger.debug('Listening on %r' % (s.getsockname(),)) + # Newer python does this automatically. Do it manually here for + # maximum compatibility + s.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, 1) + s.setsockopt(socket.SOL_TCP, socket.TCP_QUICKACK, 1) + + name = self.server.sockets[0].getsockname() + if self.server.sockets[0].family == socket.AF_INET6: + self.address = "[%s]:%d" % (name[0], name[1]) + else: + self.address = "%s:%d" % (name[0], name[1]) + + self.start = start_tcp def start_unix_server(self, path): def cleanup(): os.unlink(path) - cwd = os.getcwd() - try: - # Work around path length limits in AF_UNIX - os.chdir(os.path.dirname(path)) - self.server = self.loop.run_until_complete( - asyncio.start_unix_server(self.handle_client, os.path.basename(path), loop=self.loop) - ) - finally: - os.chdir(cwd) + def start_unix(): + cwd = os.getcwd() + try: + # Work around path length limits in AF_UNIX + os.chdir(os.path.dirname(path)) + self.server = self.loop.run_until_complete( + asyncio.start_unix_server(self.handle_client, os.path.basename(path)) + ) + finally: + os.chdir(cwd) - self.logger.debug('Listening on %r' % path) + self.logger.debug('Listening on %r' % path) - self._cleanup_socket = cleanup - self.address = "unix://%s" % os.path.abspath(path) + self._cleanup_socket = cleanup + self.address = "unix://%s" % os.path.abspath(path) + + self.start = start_unix @abc.abstractmethod def accept_client(self, reader, writer):
@@ -205,8 +210,7 @@ class AsyncServer(object): self.logger.debug("Got exit signal") self.loop.stop() - def serve_forever(self): - asyncio.set_event_loop(self.loop) + def _serve_forever(self): try: self.loop.add_signal_handler(signal.SIGTERM, self.signal_handler) signal.pthread_sigmask(signal.SIG_UNBLOCK, [signal.SIGTERM])
@@ -217,28 +221,50 @@ class AsyncServer(object): self.loop.run_until_complete(self.server.wait_closed()) self.logger.debug('Server shutting down') finally: - if self.close_loop: - if sys.version_info >= (3, 6): - self.loop.run_until_complete(self.loop.shutdown_asyncgens()) - self.loop.close() - if self._cleanup_socket is not None: self._cleanup_socket() + def serve_forever(self): + """ + Serve requests in the current process + """ + self.start() + self._serve_forever() + def serve_as_process(self, *, prefunc=None, args=()): - def run(): + """ + Serve requests in a child process + """ + def run(queue): + try: + self.start() + finally: + queue.put(self.address) + queue.close() + if prefunc is not None: prefunc(self, *args) - self.serve_forever() + + self._serve_forever() + + if sys.version_info >= (3, 6): + self.loop.run_until_complete(self.loop.shutdown_asyncgens()) + self.loop.close() + + queue = multiprocessing.Queue() # Temporarily block SIGTERM. The server process will inherit this # block which will ensure it doesn't receive the SIGTERM until the # handler is ready for it mask = signal.pthread_sigmask(signal.SIG_BLOCK, [signal.SIGTERM]) try: - self.process = multiprocessing.Process(target=run) + self.process = multiprocessing.Process(target=run, args=(queue,)) self.process.start() + self.address = queue.get() + queue.close() + queue.join_thread() + return self.process finally: signal.pthread_sigmask(signal.SIG_SETMASK, mask)
lib/hashserv/server.py +2 -2
@@ -410,11 +410,11 @@ class ServerClient(bb.asyncrpc.AsyncServerConnection): class Server(bb.asyncrpc.AsyncServer): - def __init__(self, db, loop=None, upstream=None, read_only=False): + def __init__(self, db, upstream=None, read_only=False): if upstream and read_only: raise bb.asyncrpc.ServerError("Read-only hashserv cannot pull from an upstream server") - super().__init__(logger, loop) + super().__init__(logger) self.request_stats = Stats() self.db = db
Signature · 1 line
2.31.1
PATCH v6 2/4 bitbake: asyncrpc: always create new asyncio loops
In reply to Scott Murrayasyncio in older Python 3.x (seen with 3.7) can seemingly hang if new_event_loop is called repeatedly in the same process. The reuse of processes in the Bitbake thread pool during parsing seems to be able to trigger this with the PR server export selftest. It appears that calling set_event_loop with the new loop avoids the issue, so that is now done in the asyncrpc Client initializer (with an explanatory comment). This should be revisited when the day arrives that Python 3.9 becomes the minimum required for BitBake.
Additionally, it was discovered that using get_event_loop in the asyncrpc server initialization can trigger hangs in the hashserv unittests when the second test is run. To avoid this, switch to calling new_event_loop + set_event_loop in the initialization code there as well.
--- lib/bb/asyncrpc/client.py | 10 ++++++++++ lib/bb/asyncrpc/serv.py | 24 ++++++++++++++++++++---- 2 files changed, 30 insertions(+), 4 deletions(-)
lib/bb/asyncrpc/client.py +10 -0
@@ -119,6 +119,16 @@ class Client(object): self.client = self._get_async_client() self.loop = asyncio.new_event_loop() + # Override any pre-existing loop. + # Without this, the PR server export selftest triggers a hang + # when running with Python 3.7. The drawback is that there is + # potential for issues if the PR and hash equiv (or some new) + # clients need to both be instantiated in the same process. + # This should be revisited if/when Python 3.9 becomes the + # minimum required version for BitBake, as it seems not + # required (but harmless) with it. + asyncio.set_event_loop(self.loop) + self._add_methods('connect_tcp', 'close', 'ping') @abc.abstractmethod
lib/bb/asyncrpc/serv.py +20 -4
@@ -136,10 +136,7 @@ class AsyncServer(object): self.logger = logger self.start = None self.address = None - - @property - def loop(self): - return asyncio.get_event_loop() + self.loop = None def start_tcp_server(self, host, port): def start_tcp():
@@ -228,6 +225,12 @@ class AsyncServer(object): """ Serve requests in the current process """ + # Create loop and override any loop that may have existed in + # a parent process. It is possible that the usecases of + # serve_forever might be constrained enough to allow using + # get_event_loop here, but better safe than sorry for now. + self.loop = asyncio.new_event_loop() + asyncio.set_event_loop(self.loop) self.start() self._serve_forever()
@@ -236,6 +239,19 @@ class AsyncServer(object): Serve requests in a child process """ def run(queue): + # Create loop and override any loop that may have existed + # in a parent process. Without doing this and instead + # using get_event_loop, at the very minimum the hashserv + # unit tests will hang when running the second test. + # This happens since get_event_loop in the spawned server + # process for the second testcase ends up with the loop + # from the hashserv client created in the unit test process + # when running the first testcase. The problem is somewhat + # more general, though, as any potential use of asyncio in + # Cooker could create a loop that needs to replaced in this + # new process. + self.loop = asyncio.new_event_loop() + asyncio.set_event_loop(self.loop) try: self.start() finally:
Signature · 1 line
2.31.1
Re: [PATCH v6 2/4] bitbake: asyncrpc: always create new asyncio loops
Quoted 9 lines · Scott Murray
asyncio in older Python 3.x (seen with 3.7) can seemingly hang if
new_event_loopis called repeatedly in the same process. The reuse of processes in the Bitbake thread pool during parsing seems to be able to trigger this with the PR server export selftest. It appears that callingset_event_loopwith the new loop avoids the issue, so that is now done in the asyncrpc Client initializer (with an explanatory comment). This should be revisited when the day arrives that Python 3.9 becomes the minimum required for BitBake.
Quoted 7 lines
Additionally, it was discovered that using
get_event_loopin the asyncrpc server initialization can trigger hangs in the hashserv unittests when the second test is run. To avoid this, switch to callingnew_event_loop+set_event_loopin the initialization code there as well.Signed-off-by Scott Murray <[email protected]>
Looks good to me
Quoted 75 lines
--- lib/bb/asyncrpc/client.py | 10 ++++++++++ lib/bb/asyncrpc/serv.py | 24 ++++++++++++++++++++---- 2 files changed, 30 insertions(+), 4 deletions(-)lib/bb/asyncrpc/client.py +10
-0@@ -119,6 +119,16 @@ class Client(object): self.client = self._get_async_client() self.loop = asyncio.new_event_loop() + # Override any pre-existing loop. + # Without this, the PR server export selftest triggers a hang + # when running with Python 3.7. The drawback is that there is + # potential for issues if the PR and hash equiv (or some new) + # clients need to both be instantiated in the same process. + # This should be revisited if/when Python 3.9 becomes the + # minimum required version for BitBake, as it seems not + # required (but harmless) with it. + asyncio.set_event_loop(self.loop) + self._add_methods('connect_tcp', 'close', 'ping') @abc.abstractmethodlib/bb/asyncrpc/serv.py +20
-4@@ -136,10 +136,7 @@ class AsyncServer(object): self.logger = logger self.start = None self.address = None - - @property - def loop(self): - return asyncio.get_event_loop() + self.loop = None def start_tcp_server(self, host, port): def start_tcp():@@ -228,6 +225,12 @@ class AsyncServer(object): """ Serve requests in the current process """ + # Create loop and override any loop that may have existed in + # a parent process. It is possible that the usecases of + # serve_forever might be constrained enough to allow using + # get_event_loop here, but better safe than sorry for now. + self.loop = asyncio.new_event_loop() + asyncio.set_event_loop(self.loop) self.start() self._serve_forever()@@ -236,6 +239,19 @@ class AsyncServer(object): Serve requests in a child process """ def run(queue): + # Create loop and override any loop that may have existed + # in a parent process. Without doing this and instead + # using get_event_loop, at the very minimum the hashserv + # unit tests will hang when running the second test. + # This happens since get_event_loop in the spawned server + # process for the second testcase ends up with the loop + # from the hashserv client created in the unit test process + # when running the first testcase. The problem is somewhat + # more general, though, as any potential use of asyncio in + # Cooker could create a loop that needs to replaced in this + # new process. + self.loop = asyncio.new_event_loop() + asyncio.set_event_loop(self.loop) try: self.start() finally:
PATCH v6 3/4 prserv: Replace XML RPC with modern asyncrpc implementation
In reply to Scott MurrayFrom: Paul Barker <[email protected]>
Update the prserv client and server classes to use the modern json and asyncio based RPC system implemented by the asyncrpc module.
--- lib/prserv/client.py | 41 ++++++++ lib/prserv/serv.py | 234 ++++++++++++++++++++----------------------- 2 files changed, 147 insertions(+), 128 deletions(-) create mode 100644 lib/prserv/client.py
lib/prserv/client.py new file +41 -0
@@ -0,0 +1,41 @@ +# +# SPDX-License-Identifier: GPL-2.0-only +# + +import logging +import bb.asyncrpc + +logger = logging.getLogger("BitBake.PRserv") + +class PRAsyncClient(bb.asyncrpc.AsyncClient): + def __init__(self): + super().__init__('PRSERVICE', '1.0', logger) + + async def getPR(self, version, pkgarch, checksum): + response = await self.send_message( + {'get-pr': {'version': version, 'pkgarch': pkgarch, 'checksum': checksum}} + ) + if response: + return response['value'] + + async def importone(self, version, pkgarch, checksum, value): + response = await self.send_message( + {'import-one': {'version': version, 'pkgarch': pkgarch, 'checksum': checksum, 'value': value}} + ) + if response: + return response['value'] + + async def export(self, version, pkgarch, checksum, colinfo): + response = await self.send_message( + {'export': {'version': version, 'pkgarch': pkgarch, 'checksum': checksum, 'colinfo': colinfo}} + ) + if response: + return (response['metainfo'], response['datainfo']) + +class PRClient(bb.asyncrpc.Client): + def __init__(self): + super().__init__() + self._add_methods('getPR', 'importone', 'export') + + def _get_async_client(self): + return PRAsyncClient()
lib/prserv/serv.py +106 -128
313 lines not on this page · raw message
Signature · 1 line
2.31.1
Re: [bitbake-devel] [PATCH v6 3/4] prserv: Replace XML RPC with modern asyncrpc implementation
Quoted 27 lines · Scott Murray
From: Paul Barker <[email protected]>
Update the prserv client and server classes to use the modern json and asyncio based RPC system implemented by the asyncrpc module.
Signed-off-by Paul Barker <[email protected]>[updated for asyncrpc changes, client split to separate file]Signed-off-by Scott Murray <[email protected]>....
- def export(self,version=None, pkgarch=None, checksum=None,
colinfo=True):lib/prserv/serv.py +6
-3- return self.connection.export(version, pkgarch, checksum, colinfo) +class PRServSingleton(object): + def __init__(self, dbfile, logfile, host, port): + self.dbfile = dbfile + self.logfile = logfile + self.host = host + self.port = port - def importone(self, version, pkgarch, checksum, value): - return self.connection.importone(version, pkgarch, checksum,value)
lib/prserv/serv.py line 125
Quoted 9 lines
lib/prserv/serv.py +6
-2+ def start(self): + self.prserv = PRServer(self.dbfile) + self.prserv.start_tcp_server(self.host, self.port) + self.process = self.prserv.serve_as_process() - def getinfo(self): - return self.host, self.port + if not self.port: + self.port = int(self.prserv.address.rsplit(':', 1)[1])
Hi,
I've tried this when running inside docker container and it failed with a bit ugly exception:
bitbake@599696cd20aa:~/nodistro/honister$ bitbake -k pkgconfig-native
Traceback (most recent call last):
File "/OE/nodistro/honister/bitbake/bin/bitbake", line 35, in <module>
sys.exit(bitbake_main(BitBakeConfigParameters(sys.argv),
File "/OE/nodistro/honister/bitbake/lib/bb/main.py", line 385, in
bitbake_main
return ui_module.main(server_connection.connection,
server_connection.events,
File "/OE/nodistro/honister/bitbake/lib/bb/ui/knotty.py", line 397, in
main
params.updateToServer(server, os.environ.copy())
File "/OE/nodistro/honister/bitbake/lib/bb/cookerdata.py", line 75, in
updateToServer
raise Exception("Unable to update the server configuration with local
parameters: %s" % error)
Exception: Unable to update the server configuration with local parameters:
Traceback (most recent call last):
File "/OE/nodistro/honister/bitbake/lib/bb/command.py", line 90, in
runCommand
result = command_method(self, commandline)
File "/OE/nodistro/honister/bitbake/lib/bb/command.py", line 286, in
updateConfig
command.cooker.updateConfigOpts(options, environment, cmdline)
File "/OE/nodistro/honister/bitbake/lib/bb/cooker.py", line 491, in
updateConfigOpts
self.reset()
File "/OE/nodistro/honister/bitbake/lib/bb/cooker.py", line 1717, in reset
self.handlePRServ()
File "/OE/nodistro/honister/bitbake/lib/bb/cooker.py", line 383, in
handlePRServ
self.prhost = prserv.serv.auto_start(self.data)
File "/OE/nodistro/honister/bitbake/lib/prserv/serv.py", line 318, in
auto_start
singleton.start()
File "/OE/nodistro/honister/bitbake/lib/prserv/serv.py", line 133, in
start
self.port = int(self.prserv.address.rsplit(':', 1)[1])
AttributeError: 'NoneType' object has no attribute 'rsplit'after removing the rsplit it shows better error message:
bitbake@599696cd20aa:~/nodistro/honister$ bitbake -k pkgconfig-native
WARNING: Error talking to server: Multiple exceptions: [Errno 111] Connect
call failed ('127.0.0.1', 0), [Errno 99] Cannot assign requested address
WARNING: Error talking to server: Multiple exceptions: [Errno 111] Connect
call failed ('127.0.0.1', 0), [Errno 99] Cannot assign requested address
WARNING: Error talking to server: Multiple exceptions: [Errno 111] Connect
call failed ('127.0.0.1', 0), [Errno 99] Cannot assign requested address
WARNING: Error talking to server: Multiple exceptions: [Errno 111] Connect
call failed ('127.0.0.1', 0), [Errno 99] Cannot assign requested address
ERROR: PRservice localhost:0 not available
ERROR: Unable to start PR Server, exittingIt would be nice to improve the error handling a bit.
Regards,
Re: [bitbake-devel] [PATCH v6 3/4] prserv: Replace XML RPC with modern asyncrpc implementation
Quoted 95 lines · Martin Jansa
Quoted 24 lines · Scott Murray
From: Paul Barker <[email protected]>
Update the prserv client and server classes to use the modern json and asyncio based RPC system implemented by the asyncrpc module.
Signed-off-by Paul Barker <[email protected]>[updated for asyncrpc changes, client split to separate file]Signed-off-by Scott Murray <[email protected]>....
- def export(self,version=None, pkgarch=None, checksum=None,
colinfo=True):lib/prserv/serv.py +6
-3- return self.connection.export(version, pkgarch, checksum, colinfo) +class PRServSingleton(object): + def __init__(self, dbfile, logfile, host, port): + self.dbfile = dbfile + self.logfile = logfile + self.host = host + self.port = port - def importone(self, version, pkgarch, checksum, value): - return self.connection.importone(version, pkgarch, checksum,value)
lib/prserv/serv.py line 125
Quoted 9 lines
lib/prserv/serv.py +6
-2+ def start(self): + self.prserv = PRServer(self.dbfile) + self.prserv.start_tcp_server(self.host, self.port) + self.process = self.prserv.serve_as_process() - def getinfo(self): - return self.host, self.port + if not self.port: + self.port = int(self.prserv.address.rsplit(':', 1)[1])Hi,
I've tried this when running inside docker container and it failed with a bit ugly exception:
bitbake@599696cd20aa:~/nodistro/honister$ bitbake -k pkgconfig-native Traceback (most recent call last): File "/OE/nodistro/honister/bitbake/bin/bitbake", line 35, in <module> sys.exit(bitbake_main(BitBakeConfigParameters(sys.argv), File "/OE/nodistro/honister/bitbake/lib/bb/main.py", line 385, in bitbake_main return ui_module.main(server_connection.connection, server_connection.events, File "/OE/nodistro/honister/bitbake/lib/bb/ui/knotty.py", line 397, in main params.updateToServer(server, os.environ.copy()) File "/OE/nodistro/honister/bitbake/lib/bb/cookerdata.py", line 75, in updateToServer raise Exception("Unable to update the server configuration with local parameters: %s" % error) Exception: Unable to update the server configuration with local parameters: Traceback (most recent call last): File "/OE/nodistro/honister/bitbake/lib/bb/command.py", line 90, in runCommand result = command_method(self, commandline) File "/OE/nodistro/honister/bitbake/lib/bb/command.py", line 286, in updateConfig command.cooker.updateConfigOpts(options, environment, cmdline) File "/OE/nodistro/honister/bitbake/lib/bb/cooker.py", line 491, in updateConfigOpts self.reset() File "/OE/nodistro/honister/bitbake/lib/bb/cooker.py", line 1717, in reset self.handlePRServ() File "/OE/nodistro/honister/bitbake/lib/bb/cooker.py", line 383, in handlePRServ self.prhost = prserv.serv.auto_start(self.data) File "/OE/nodistro/honister/bitbake/lib/prserv/serv.py", line 318, in auto_start singleton.start() File "/OE/nodistro/honister/bitbake/lib/prserv/serv.py", line 133, in start self.port = int(self.prserv.address.rsplit(':', 1)[1]) AttributeError: 'NoneType' object has no attribute 'rsplit'after removing the rsplit it shows better error message:
bitbake@599696cd20aa:~/nodistro/honister$ bitbake -k pkgconfig-native WARNING: Error talking to server: Multiple exceptions: [Errno 111] Connect call failed ('127.0.0.1', 0), [Errno 99] Cannot assign requested address WARNING: Error talking to server: Multiple exceptions: [Errno 111] Connect call failed ('127.0.0.1', 0), [Errno 99] Cannot assign requested address WARNING: Error talking to server: Multiple exceptions: [Errno 111] Connect call failed ('127.0.0.1', 0), [Errno 99] Cannot assign requested address WARNING: Error talking to server: Multiple exceptions: [Errno 111] Connect call failed ('127.0.0.1', 0), [Errno 99] Cannot assign requested address ERROR: PRservice localhost:0 not available ERROR: Unable to start PR Server, exitting
Quoted 1 line
It would be nice to improve the error handling a bit.
Could you elaborate a bit on your configuration? Is it just trying to build plain oe-core / poky in Docker? I don't really see right off from the code in bitbake's lib/bb/asyncrpc/serv.py how the port from the asyncio server instance could end up unset.
Thanks,
Scott
PATCH v6 4/4 prserv: Add read-only mode
In reply to Scott MurrayFrom: Paul Barker <[email protected]>
[YOCTO #13659]
--- bin/bitbake-prserv | 4 ++- lib/prserv/client.py | 9 +++++- lib/prserv/db.py | 65 ++++++++++++++++++++++++++++++++++---------- lib/prserv/serv.py | 40 ++++++++++++++++----------- 4 files changed, 86 insertions(+), 32 deletions(-)
bin/bitbake-prserv +3 -1
@@ -36,12 +36,14 @@ def main(): dest="host", type="string", default=PRHOST_DEFAULT) parser.add_option("--port", help="port number(default: 8585)", action="store", dest="port", type="int", default=PRPORT_DEFAULT) + parser.add_option("-r", "--read-only", help="open database in read-only mode", + action="store_true") options, args = parser.parse_args(sys.argv) prserv.init_logger(os.path.abspath(options.logfile),options.loglevel) if options.start: - ret=prserv.serv.start_daemon(options.dbfile, options.host, options.port,os.path.abspath(options.logfile)) + ret=prserv.serv.start_daemon(options.dbfile, options.host, options.port,os.path.abspath(options.logfile), options.read_only) elif options.stop: ret=prserv.serv.stop_daemon(options.host, options.port) else:
lib/prserv/client.py +8 -1
@@ -32,10 +32,17 @@ class PRAsyncClient(bb.asyncrpc.AsyncClient): if response: return (response['metainfo'], response['datainfo']) + async def is_readonly(self): + response = await self.send_message( + {'is-readonly': {}} + ) + if response: + return response['readonly'] + class PRClient(bb.asyncrpc.Client): def __init__(self): super().__init__() - self._add_methods('getPR', 'importone', 'export') + self._add_methods('getPR', 'importone', 'export', 'is_readonly') def _get_async_client(self): return PRAsyncClient()
lib/prserv/db.py +51 -14
130 lines not on this page · raw message
lib/prserv/serv.py +24 -16
@@ -18,14 +18,16 @@ PIDPREFIX = "/tmp/PRServer_%s_%s.pid" singleton = None class PRServerClient(bb.asyncrpc.AsyncServerConnection): - def __init__(self, reader, writer, table): + def __init__(self, reader, writer, table, read_only): super().__init__(reader, writer, 'PRSERVICE', logger) self.handlers.update({ 'get-pr': self.handle_get_pr, 'import-one': self.handle_import_one, 'export': self.handle_export, + 'is-readonly': self.handle_is_readonly, }) self.table = table + self.read_only = read_only def validate_proto_version(self): return (self.proto_version == (1, 0))
@@ -56,16 +58,17 @@ class PRServerClient(bb.asyncrpc.AsyncServerConnection): self.write_message(response) async def handle_import_one(self, request): - version = request['version'] - pkgarch = request['pkgarch'] - checksum = request['checksum'] - value = request['value'] + response = None + if not self.read_only: + version = request['version'] + pkgarch = request['pkgarch'] + checksum = request['checksum'] + value = request['value'] + + value = self.table.importone(version, pkgarch, checksum, value) + if value is not None: + response = {'value': value} - value = self.table.importone(version, pkgarch, checksum, value) - if value is not None: - response = {'value': value} - else: - response = None self.write_message(response) async def handle_export(self, request):
@@ -83,20 +86,25 @@ class PRServerClient(bb.asyncrpc.AsyncServerConnection): response = {'metainfo': metainfo, 'datainfo': datainfo} self.write_message(response) + async def handle_is_readonly(self, request): + response = {'readonly': self.read_only} + self.write_message(response) + class PRServer(bb.asyncrpc.AsyncServer): - def __init__(self, dbfile): + def __init__(self, dbfile, read_only=False): super().__init__(logger) self.dbfile = dbfile self.table = None + self.read_only = read_only def accept_client(self, reader, writer): - return PRServerClient(reader, writer, self.table) + return PRServerClient(reader, writer, self.table, self.read_only) def _serve_forever(self): - self.db = prserv.db.PRData(self.dbfile) + self.db = prserv.db.PRData(self.dbfile, read_only=self.read_only) self.table = self.db["PRMAIN"] - logger.debug("Started PRServer with DBfile: %s, Address: %s, PID: %s" % + logger.info("Started PRServer with DBfile: %s, Address: %s, PID: %s" % (self.dbfile, self.address, str(os.getpid()))) super()._serve_forever()
@@ -194,7 +202,7 @@ def run_as_daemon(func, pidfile, logfile): os.remove(pidfile) os._exit(0) -def start_daemon(dbfile, host, port, logfile): +def start_daemon(dbfile, host, port, logfile, read_only=False): ip = socket.gethostbyname(host) pidfile = PIDPREFIX % (ip, port) try:
@@ -210,7 +218,7 @@ def start_daemon(dbfile, host, port, logfile): dbfile = os.path.abspath(dbfile) def daemon_main(): - server = PRServer(dbfile) + server = PRServer(dbfile, read_only=read_only) server.start_tcp_server(host, port) server.serve_forever()
Signature · 1 line
2.31.1