bitbake: cooker: check if upstream hash equivalence server is available
PATCH 1/1 bitbake: cooker: check if upstream hash equivalence server is available
When the user specify an upstream hash equivalence server in BB_HASHSERVE_UPSTREAM abort if we can't connect otherwise the user will never be notified about the problem.
--- bitbake/lib/bb/cooker.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-)
bitbake/lib/bb/cooker.py +11 -1
@@ -388,12 +388,22 @@ class BBCooker: # Create a new hash server bound to a unix domain socket if not self.hashserv: dbfile = (self.data.getVar("PERSISTENT_DIR") or self.data.getVar("CACHE")) + "/hashserv.db" + upstream = self.data.getVar("BB_HASHSERVE_UPSTREAM") or None + if upstream: + import socket + try: + sock = socket.create_connection(upstream.split(":"), 5) + sock.close() + except socket.error as e: + bb.fatal("BB_HASHSERVE_UPSTREAM is not valid, unable to connect hash equivalence server at '%s': %s" + % (upstream, repr(e))) + self.hashservaddr = "unix://%s/hashserve.sock" % self.data.getVar("TOPDIR") self.hashserv = hashserv.create_server( self.hashservaddr, dbfile, sync=False, - upstream=self.data.getVar("BB_HASHSERVE_UPSTREAM") or None, + upstream=upstream, ) self.hashserv.serve_as_process() self.data.setVar("BB_HASHSERVE", self.hashservaddr)
Signature · 1 line
2.33.1
Quoted 30 lines · Jose Quaresma
When the user specify an upstream hash equivalence server in
BB_HASHSERVE_UPSTREAMabort if we can't connect otherwise the user will never be notified about the problem.Signed-off-by Jose Quaresma <[email protected]>--- bitbake/lib/bb/cooker.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-)bitbake/lib/bb/cooker.py +0
-0@@ -388,12 +388,22 @@ class BBCooker: # Create a new hash server bound to a unix domain socket if not self.hashserv: dbfile = (self.data.getVar("PERSISTENT_DIR") or
self.data.getVar("CACHE")) + "/hashserv.db"
+ upstream =self.data.getVar("BB_HASHSERVE_UPSTREAM")or
Nonebitbake/lib/bb/cooker.py +4
-0+ if upstream: + import socket + try: + sock =
socket.create_connection(upstream.split(":"), 5)bitbake/lib/bb/cooker.py +3
-0+ sock.close() + except socket.error as e: + bb.fatal("BB_HASHSERVE_UPSTREAM is not valid,unable to connect hash equivalence server at '%s': %s"
bitbake/lib/bb/cooker.py class BBCooker: · line 400
Quoted 2 lines
bitbake/lib/bb/cooker.py +2
-0+ % (upstream, repr(e))) +
Perhaps this should just be a warning? The upstream missing should not be fatal for a given server, if it is we probably need to fix that
self.hashservaddr = "unix://%s/hashserve.sock" %
Quoted 26 lines
self.data.getVar("TOPDIR")self.hashserv = hashserv.create_server( self.hashservaddr, dbfile, sync=False, - upstream=self.data.getVar("BB_HASHSERVE_UPSTREAM") or None, + upstream=upstream, ) self.hashserv.serve_as_process() self.data.setVar("BB_HASHSERVE", self.hashservaddr)--
2.33.1-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group. View/Reply Online (#157010):
https://lists.openembedded.org/g/openembedded-core/message/157010
Mute This Topic: https://lists.openembedded.org/mt/86362510/3616693
Group Owner: [email protected]
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [ [email protected]]
-=-=-=-=-=-=-=-=-=-=-=-
Joshua Watt <[email protected]> escreveu no dia sábado, 16/10/2021 à(s) 20:15:
Quoted 32 lines
Quoted 27 lines · Jose Quaresma
When the user specify an upstream hash equivalence server in
BB_HASHSERVE_UPSTREAMabort if we can't connect otherwise the user will never be notified about the problem.Signed-off-by Jose Quaresma <[email protected]>--- bitbake/lib/bb/cooker.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-)bitbake/lib/bb/cooker.py +0
-0@@ -388,12 +388,22 @@ class BBCooker: # Create a new hash server bound to a unix domain socket if not self.hashserv: dbfile = (self.data.getVar("PERSISTENT_DIR") or
self.data.getVar("CACHE")) + "/hashserv.db"
+ upstream =self.data.getVar("BB_HASHSERVE_UPSTREAM")or
Nonebitbake/lib/bb/cooker.py +4
-0+ if upstream: + import socket + try: + sock =
socket.create_connection(upstream.split(":"), 5)bitbake/lib/bb/cooker.py +2
-0+ sock.close() + except socket.error as e:
Quoted 9 lines
bitbake/lib/bb/cooker.py class BBCooker: · line 400
Quoted 5 lines
+ bb.fatal("BB_HASHSERVE_UPSTREAM is not valid,
unable to connect hash equivalence server at '%s': %s"bitbake/lib/bb/cooker.py +2
-0+ % (upstream, repr(e))) +Perhaps this should just be a warning? The upstream missing should not be fatal for a given server, if it is we probably need to fix that
In my opinion this needs to be fatal or fixed with a different approach.
If we launch bitbake in a clean build without build/cache with an invalid server it will deadlock printing the follow message after several minutes:
WARNING: Error contacting Hash Equivalence Server unix:///build/hashserve.sock: Connection closed
The build/bitbake-cookerdaemon.log have all the connections errors as well as exceptions.
This cooker log is a bit ugly imho and I will try to improved it in a follow up patch.
Quoted 27 lines
self.hashservaddr = "unix://%s/hashserve.sock" %
Quoted 26 lines
self.data.getVar("TOPDIR")self.hashserv = hashserv.create_server( self.hashservaddr, dbfile, sync=False, - upstream=self.data.getVar("BB_HASHSERVE_UPSTREAM") or None, + upstream=upstream, ) self.hashserv.serve_as_process() self.data.setVar("BB_HASHSERVE", self.hashservaddr)--
2.33.1-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group. View/Reply Online (#157010):
https://lists.openembedded.org/g/openembedded-core/message/157010
Mute This Topic: https://lists.openembedded.org/mt/86362510/3616693
Group Owner: [email protected]
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [ [email protected]]
-=-=-=-=-=-=-=-=-=-=-=-
Signature · 3 lines
Best regards, José Quaresma
A message that is not in the archive (sent off-list or not yet received): [email protected]
Jose Quaresma via lists.openembedded.org <quaresma.jose= [email protected]> escreveu no dia domingo, 17/10/2021 à(s) 22:14:
Quoted 47 lines
Joshua Watt <[email protected]> escreveu no dia sábado, 16/10/2021 à(s) 20:15:
Quoted 41 lines
Quoted 30 lines · Jose Quaresma
When the user specify an upstream hash equivalence server in
BB_HASHSERVE_UPSTREAMabort if we can't connect otherwise the user will never be notified about the problem.Signed-off-by Jose Quaresma <[email protected]>--- bitbake/lib/bb/cooker.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-)bitbake/lib/bb/cooker.py +0
-0@@ -388,12 +388,22 @@ class BBCooker: # Create a new hash server bound to a unix domain socket if not self.hashserv: dbfile = (self.data.getVar("PERSISTENT_DIR") or
self.data.getVar("CACHE")) + "/hashserv.db"
+ upstream =self.data.getVar("BB_HASHSERVE_UPSTREAM")or
Nonebitbake/lib/bb/cooker.py +4
-0+ if upstream: + import socket + try: + sock =
socket.create_connection(upstream.split(":"), 5)bitbake/lib/bb/cooker.py +3
-0+ sock.close() + except socket.error as e: + bb.fatal("BB_HASHSERVE_UPSTREAM is not valid,unable to connect hash equivalence server at '%s': %s"
bitbake/lib/bb/cooker.py class BBCooker: · line 400
Quoted 2 lines
bitbake/lib/bb/cooker.py +2
-0+ % (upstream, repr(e))) +Perhaps this should just be a warning? The upstream missing should not be fatal for a given server, if it is we probably need to fix that
I have changed it to be a warning in V2
Quoted 49 lines
In my opinion this needs to be fatal or fixed with a different approach.
If we launch bitbake in a clean build without build/cache with an invalid server it will deadlock printing the follow message after several minutes:
WARNING: Error contacting Hash Equivalence Server unix:///build/hashserve.sock: Connection closedThe build/bitbake-cookerdaemon.log have all the connections errors as well as exceptions.
This cooker log is a bit ugly imho and I will try to improved it in a follow up patch.Quoted 19 lines
self.hashservaddr = "unix://%s/hashserve.sock" %
Quoted 18 lines
self.data.getVar("TOPDIR")self.hashserv = hashserv.create_server( self.hashservaddr, dbfile, sync=False, - upstream=self.data.getVar("BB_HASHSERVE_UPSTREAM") or None, + upstream=upstream, ) self.hashserv.serve_as_process() self.data.setVar("BB_HASHSERVE", self.hashservaddr)--
2.33.1--
Best regards,José Quaresma
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group. View/Reply Online (#157046):
https://lists.openembedded.org/g/openembedded-core/message/157046
Mute This Topic: https://lists.openembedded.org/mt/86362510/5052612
Group Owner: [email protected]
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [ [email protected]]
-=-=-=-=-=-=-=-=-=-=-=-
Signature · 3 lines
Best regards, José Quaresma
A message that is not in the archive (sent off-list or not yet received): [email protected]
To reproduce we can set:
# the real port is 8687 and with the 18687 we can simulate a firewall block BB_HASHSERVE_UPSTREAM = "typhoon.yocto.io:18687"
With the V2 patch and we get on the log:
WARNING: BB_HASHSERVE_UPSTREAM is not valid, unable to connect hash equivalence server at 'typhoon.yocto.io:18687': timeout('timed out')
and bitbake deadlock in Initialising tasks:
Initialising tasks: 44% |####################################################
| ETA:
0:00:00
on bitbake-cookerdaemon.log we have:
3 19:32:26.434306 Running command ['buildTargets', ['core-image-minimal'], 'build']
63 19:32:26.434386 Command Completed
Error talking to server: [Errno 110] Connect call failed ('35.233.185.178', 18687)
Error talking to server: [Errno 110] Connect call failed ('35.233.185.178', 18687)
Error talking to server: [Errno 110] Connect call failed ('35.233.185.178', 18687)
Error talking to server: [Errno 110] Connect call failed ('35.233.185.178', 18687)
Error from client: [Errno 110] Connect call failed ('35.233.185.178', 18687)
Traceback (most recent call last):
File "/work/poky/bitbake/lib/bb/asyncrpc/client.py", line 57, in_send_wrapper
await self.connect()
File "/work/poky/bitbake/lib/bb/asyncrpc/client.py", line 43, in connect
(self.reader, self.writer) = await self._connect_sock()
File "/work/poky/bitbake/lib/bb/asyncrpc/client.py", line 26, inconnect_sock
return await asyncio.open_connection(address, port)
File "/usr/lib/python3.7/asyncio/streams.py", line 77, in open_connection
lambda: protocol, host, port, **kwds)
File "/usr/lib/python3.7/asyncio/base_events.py", line 959, increate_connection
raise exceptions[0]
File "/usr/lib/python3.7/asyncio/base_events.py", line 946, increate_connection
await self.sock_connect(sock, address)
File "/usr/lib/python3.7/asyncio/selector_events.py", line 464, insock_connect
return await fut
File "/usr/lib/python3.7/asyncio/selector_events.py", line 494, in_sock_connect_cb
raise OSError(err, f'Connect call failed {address}')
TimeoutError: [Errno 110] Connect call failed ('35.233.185.178', 18687)
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/work/poky/bitbake/lib/bb/asyncrpc/serv.py", line 192, inhandle_client
await client.process_requests()
File "/work/poky/bitbake/lib/hashserv/server.py", line 200, inprocess_requests
await super().process_requests()
File "/work/poky/bitbake/lib/bb/asyncrpc/serv.py", line 72, inprocess_requests
await self.dispatch_message(d)
File "/work/poky/bitbake/lib/hashserv/server.py", line 210, indispatch_message
await self.handlers[k](msg[k])
File "/work/poky/bitbake/lib/hashserv/server.py", line 355, inhandle_get_stream
upstream = await self.upstream_client.get_unihash(method, taskhash)
File "/work/poky/bitbake/lib/hashserv/client.py", line 57, in get_unihash
await self._set_mode(self.MODE_GET_STREAM)
File "/work/poky/bitbake/lib/hashserv/client.py", line 46, in _set_mode
r = await self.send_message({"get-stream": None})
File "/work/poky/bitbake/lib/bb/asyncrpc/client.py", line 110, insend_message
return await self._send_wrapper(proc)
File "/work/poky/bitbake/lib/bb/asyncrpc/client.py", line 68, in_send_wrapper
raise ConnectionError(str(e))
ConnectionError: [Errno 110] Connect call failed ('35.233.185.178', 18687) Error talking to server: Connection closed
Traceback (most recent call last):
File "/work/poky/bitbake/lib/bb/asyncrpc/client.py", line 57, in_send_wrapper
await self.connect()
File "/work/poky/bitbake/lib/bb/asyncrpc/client.py", line 43, in connect
(self.reader, self.writer) = await self._connect_sock()
File "/work/poky/bitbake/lib/bb/asyncrpc/client.py", line 26, inconnect_sock
return await asyncio.open_connection(address, port)
File "/usr/lib/python3.7/asyncio/streams.py", line 77, in open_connection
lambda: protocol, host, port, **kwds)
File "/usr/lib/python3.7/asyncio/base_events.py", line 959, increate_connection
raise exceptions[0]
File "/usr/lib/python3.7/asyncio/base_events.py", line 946, increate_connection
await self.sock_connect(sock, address)
File "/usr/lib/python3.7/asyncio/selector_events.py", line 464, insock_connect
return await fut
File "/usr/lib/python3.7/asyncio/selector_events.py", line 494, in_sock_connect_cb
raise OSError(err, f'Connect call failed {address}')
TimeoutError: [Errno 110] Connect call failed ('35.233.185.178', 18687)
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/work/poky/bitbake/lib/bb/asyncrpc/serv.py", line 192, inhandle_client
await client.process_requests()
File "/work/poky/bitbake/lib/hashserv/server.py", line 200, inprocess_requests
await super().process_requests()
File "/work/poky/bitbake/lib/bb/asyncrpc/serv.py", line 72, inprocess_requests
await self.dispatch_message(d)
File "/work/poky/bitbake/lib/hashserv/server.py", line 210, indispatch_message
await self.handlers[k](msg[k])
File "/work/poky/bitbake/lib/hashserv/server.py", line 355, inhandle_get_stream
upstream = await self.upstream_client.get_unihash(method, taskhash)
File "/work/poky/bitbake/lib/hashserv/client.py", line 57, in get_unihash
await self._set_mode(self.MODE_GET_STREAM)
File "/work/poky/bitbake/lib/hashserv/client.py", line 46, in _set_mode
r = await self.send_message({"get-stream": None})
File "/work/poky/bitbake/lib/bb/asyncrpc/client.py", line 110, insend_message
return await self._send_wrapper(proc)
File "/work/poky/bitbake/lib/bb/asyncrpc/client.py", line 68, in_send_wrapper
raise ConnectionError(str(e))
ConnectionError: [Errno 110] Connect call failed ('35.233.185.178', 18687) Error talking to server: [Errno 110] Connect call failed ('35.233.185.178', 18687)
Jose Quaresma via lists.openembedded.org <quaresma.jose= [email protected]> escreveu no dia quinta, 4/11/2021 à(s) 19:31:
Quoted 114 lines
Jose Quaresma via lists.openembedded.org <quaresma.jose= [email protected]> escreveu no dia domingo, 17/10/2021 à(s) 22:14:
Quoted 47 lines
Joshua Watt <[email protected]> escreveu no dia sábado, 16/10/2021 à(s) 20:15:
Quoted 41 lines
Quoted 30 lines · Jose Quaresma
When the user specify an upstream hash equivalence server in
BB_HASHSERVE_UPSTREAMabort if we can't connect otherwise the user will never be notified about the problem.Signed-off-by Jose Quaresma <[email protected]>--- bitbake/lib/bb/cooker.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-)bitbake/lib/bb/cooker.py +0
-0@@ -388,12 +388,22 @@ class BBCooker: # Create a new hash server bound to a unix domain socket if not self.hashserv: dbfile = (self.data.getVar("PERSISTENT_DIR") or
self.data.getVar("CACHE")) + "/hashserv.db"
+ upstream =self.data.getVar("BB_HASHSERVE_UPSTREAM")
or Nonebitbake/lib/bb/cooker.py +4
-0+ if upstream: + import socket + try: + sock =
socket.create_connection(upstream.split(":"), 5)bitbake/lib/bb/cooker.py +3
-0+ sock.close() + except socket.error as e: + bb.fatal("BB_HASHSERVE_UPSTREAM is not valid,unable to connect hash equivalence server at '%s': %s"
bitbake/lib/bb/cooker.py class BBCooker: · line 400
Quoted 2 lines
bitbake/lib/bb/cooker.py +2
-0+ % (upstream, repr(e))) +Perhaps this should just be a warning? The upstream missing should not be fatal for a given server, if it is we probably need to fix that
I have changed it to be a warning in V2
Quoted 33 lines
In my opinion this needs to be fatal or fixed with a different approach.
If we launch bitbake in a clean build without build/cache with an invalid server it will deadlock printing the follow message after several minutes:
WARNING: Error contacting Hash Equivalence Server unix:///build/hashserve.sock: Connection closedThe build/bitbake-cookerdaemon.log have all the connections errors as well as exceptions.
This cooker log is a bit ugly imho and I will try to improved it in a follow up patch.Quoted 19 lines
self.hashservaddr = "unix://%s/hashserve.sock" %
Quoted 18 lines
self.data.getVar("TOPDIR")self.hashserv = hashserv.create_server( self.hashservaddr, dbfile, sync=False, - upstream=self.data.getVar("BB_HASHSERVE_UPSTREAM") or None, + upstream=upstream, ) self.hashserv.serve_as_process() self.data.setVar("BB_HASHSERVE", self.hashservaddr)--
2.33.1Quoted 8 lines
--
Best regards,José Quaresma
--
Best regards,José Quaresma
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group. View/Reply Online (#157860):
https://lists.openembedded.org/g/openembedded-core/message/157860
Mute This Topic: https://lists.openembedded.org/mt/86362510/5052612
Group Owner: [email protected]
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [ [email protected]]
-=-=-=-=-=-=-=-=-=-=-=-
Signature · 3 lines
Best regards, José Quaresma