runqueue/siggen: Optimise hash equiv queries
PATCH 1/2 runqueue/siggen: Optimise hash equiv queries
We only have hash equivalence for setscene tasks so only query the server for those, reducing the number of connections needed.
Signed-off-by Richard Purdie <[email protected]>
--- lib/bb/runqueue.py | 1 + lib/bb/siggen.py | 8 ++++++++ 2 files changed, 9 insertions(+)
lib/bb/runqueue.py +1 -0
@@ -1173,6 +1173,7 @@ class RunQueueData: self.prepare_task_hash(tid) bb.parse.siggen.writeout_file_checksum_cache() + bb.parse.siggen.set_setscene_tasks(self.runq_setscene_tids) #self.dump_data() return len(self.runtaskentries)
lib/bb/siggen.py +8 -0
@@ -106,6 +106,7 @@ class SignatureGeneratorBasic(SignatureGenerator): self.taints = {} self.gendeps = {} self.lookupcache = {} + self.setscenetasks = {} self.basewhitelist = set((data.getVar("BB_HASHBASE_WHITELIST") or "").split()) self.taskwhitelist = None self.init_rundepcheck(data)
@@ -151,6 +152,9 @@ class SignatureGeneratorBasic(SignatureGenerator): return taskdeps + def set_setscene_tasks(self, setscene_tasks): + self.setscenetasks = setscene_tasks + def finalise(self, fn, d, variant): mc = d.getVar("__BBMULTICONFIG", False) or ""
@@ -394,6 +398,10 @@ class SignatureGeneratorUniHashMixIn(object): taskhash = self.taskhash[tid] + # If its not a setscene task we can return + if self.setscenetasks and tid not in self.setscenetasks: + return taskhash + key = self.__get_task_unihash_key(tid) # TODO: This cache can grow unbounded. It probably only needs to keep
Signature · 1 line
2.20.1
PATCH 2/2 hashserv: Drop handler thread
With performance measurements we've proven the handler thread is slower than not bothering with threading so drop it.
There was also a bug in this code where the database was being repeatedly being reopened/closed.
Signed-off-by Richard Purdie <[email protected]>
--- lib/hashserv/__init__.py | 62 ++++------------------------------------ 1 file changed, 5 insertions(+), 57 deletions(-)
lib/hashserv/__init__.py +5 -57
@@ -24,17 +24,8 @@ class HashEquivalenceServer(BaseHTTPRequestHandler): def log_message(self, f, *args): logger.debug(f, *args) - def opendb(self): - self.db = sqlite3.connect(self.dbname) - self.db.row_factory = sqlite3.Row - self.db.execute("PRAGMA synchronous = OFF;") - self.db.execute("PRAGMA journal_mode = MEMORY;") - def do_GET(self): try: - if not self.db: - self.opendb() - p = urllib.parse.urlparse(self.path) if p.path != self.prefix + '/v1/equivalent':
@@ -67,9 +58,6 @@ class HashEquivalenceServer(BaseHTTPRequestHandler): def do_POST(self): try: - if not self.db: - self.opendb() - p = urllib.parse.urlparse(self.path) if p.path != self.prefix + '/v1/equivalent':
@@ -141,62 +129,22 @@ class HashEquivalenceServer(BaseHTTPRequestHandler): self.send_error(400, explain=traceback.format_exc()) return -class ThreadedHTTPServer(HTTPServer): - quit = False - - def serve_forever(self): - self.requestqueue = queue.Queue() - self.handlerthread = threading.Thread(target=self.process_request_thread) - self.handlerthread.daemon = False - - self.handlerthread.start() - - signal.signal(signal.SIGTERM, self.sigterm_exception) - super().serve_forever() - os._exit(0) - - def sigterm_exception(self, signum, stackframe): - self.server_close() - os._exit(0) - +class HashHTTPServer(HTTPServer): def server_bind(self): HTTPServer.server_bind(self) self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, struct.pack('ii', 1, 0)) - def process_request_thread(self): - while not self.quit: - try: - (request, client_address) = self.requestqueue.get(True) - except queue.Empty: - continue - if request is None: - continue - try: - self.finish_request(request, client_address) - except Exception: - self.handle_error(request, client_address) - finally: - self.shutdown_request(request) - os._exit(0) - - def process_request(self, request, client_address): - self.requestqueue.put((request, client_address)) - - def server_close(self): - super().server_close() - self.quit = True - self.requestqueue.put((None, None)) - self.handlerthread.join() - def create_server(addr, dbname, prefix=''): class Handler(HashEquivalenceServer): pass db = sqlite3.connect(dbname) db.row_factory = sqlite3.Row + db.execute("PRAGMA synchronous = OFF;") + db.execute("PRAGMA journal_mode = MEMORY;") Handler.prefix = prefix - Handler.db = None + Handler.db = db Handler.dbname = dbname with contextlib.closing(db.cursor()) as cursor:
@@ -223,7 +171,7 @@ def create_server(addr, dbname, prefix=''): cursor.execute('CREATE INDEX IF NOT EXISTS taskhash_lookup ON tasks_v2 (method, taskhash)') cursor.execute('CREATE INDEX IF NOT EXISTS outhash_lookup ON tasks_v2 (method, outhash)') - ret = ThreadedHTTPServer(addr, Handler) + ret = HashHTTPServer(addr, Handler) logger.info('Starting server on %s\n', ret.server_port)
Signature · 1 line
2.20.1