Skip to contentSkip to conversation

runqueue/siggen: Optimise hash equiv queries

PATCH New Richard Purdie 1 participant 2 messages Sep 10, 2019

lore
Share & tools
lore
#0 lore raw

PATCH 1/2 runqueue/siggen: Optimise hash equiv queries

We only have hash equivalence for setscene tasks so only query the server for those, reducing the number of connections needed.

---
 lib/bb/runqueue.py | 1 +
 lib/bb/siggen.py   | 8 ++++++++
 2 files changed, 9 insertions(+)
lib/bb/runqueue.py +1 -0
index addb2bb82f..45bfec8c37 100644
@@ -1173,6 +1173,7 @@ class RunQueueData:
                     self.prepare_task_hash(tid)
 
         bb.parse.siggen.writeout_file_checksum_cache()
+        bb.parse.siggen.set_setscene_tasks(self.runq_setscene_tids)
 
         #self.dump_data()
         return len(self.runtaskentries)
lib/bb/siggen.py +8 -0
index b503559305..8d3f2e8784 100644
@@ -106,6 +106,7 @@ class SignatureGeneratorBasic(SignatureGenerator):
         self.taints = {}
         self.gendeps = {}
         self.lookupcache = {}
+        self.setscenetasks = {}
         self.basewhitelist = set((data.getVar("BB_HASHBASE_WHITELIST") or "").split())
         self.taskwhitelist = None
         self.init_rundepcheck(data)
@@ -151,6 +152,9 @@ class SignatureGeneratorBasic(SignatureGenerator):
 
         return taskdeps
 
+    def set_setscene_tasks(self, setscene_tasks):
+        self.setscenetasks = setscene_tasks
+
     def finalise(self, fn, d, variant):
 
         mc = d.getVar("__BBMULTICONFIG", False) or ""
@@ -394,6 +398,10 @@ class SignatureGeneratorUniHashMixIn(object):
 
         taskhash = self.taskhash[tid]
 
+        # If its not a setscene task we can return
+        if self.setscenetasks and tid not in self.setscenetasks:
+            return taskhash
+
         key = self.__get_task_unihash_key(tid)
 
         # TODO: This cache can grow unbounded. It probably only needs to keep
Signature · 1 line
2.20.1
#1 lore raw

PATCH 2/2 hashserv: Drop handler thread

With performance measurements we've proven the handler thread is slower than not bothering with threading so drop it.

There was also a bug in this code where the database was being repeatedly being reopened/closed.

---
 lib/hashserv/__init__.py | 62 ++++------------------------------------
 1 file changed, 5 insertions(+), 57 deletions(-)
lib/hashserv/__init__.py +5 -57
index eb03c32213..9d5722ab88 100644
@@ -24,17 +24,8 @@ class HashEquivalenceServer(BaseHTTPRequestHandler):
     def log_message(self, f, *args):
         logger.debug(f, *args)
 
-    def opendb(self):
-        self.db = sqlite3.connect(self.dbname)
-        self.db.row_factory = sqlite3.Row
-        self.db.execute("PRAGMA synchronous = OFF;")
-        self.db.execute("PRAGMA journal_mode = MEMORY;")
-
     def do_GET(self):
         try:
-            if not self.db:
-                self.opendb()
-
             p = urllib.parse.urlparse(self.path)
 
             if p.path != self.prefix + '/v1/equivalent':
@@ -67,9 +58,6 @@ class HashEquivalenceServer(BaseHTTPRequestHandler):
 
     def do_POST(self):
         try:
-            if not self.db:
-                self.opendb()
-
             p = urllib.parse.urlparse(self.path)
 
             if p.path != self.prefix + '/v1/equivalent':
@@ -141,62 +129,22 @@ class HashEquivalenceServer(BaseHTTPRequestHandler):
             self.send_error(400, explain=traceback.format_exc())
             return
 
-class ThreadedHTTPServer(HTTPServer):
-    quit = False
-
-    def serve_forever(self):
-        self.requestqueue = queue.Queue()
-        self.handlerthread = threading.Thread(target=self.process_request_thread)
-        self.handlerthread.daemon = False
-
-        self.handlerthread.start()
-
-        signal.signal(signal.SIGTERM, self.sigterm_exception)
-        super().serve_forever()
-        os._exit(0)
-
-    def sigterm_exception(self, signum, stackframe):
-        self.server_close()
-        os._exit(0)
-
+class HashHTTPServer(HTTPServer):
     def server_bind(self):
         HTTPServer.server_bind(self)
         self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, struct.pack('ii', 1, 0))
 
-    def process_request_thread(self):
-        while not self.quit:
-            try:
-                (request, client_address) = self.requestqueue.get(True)
-            except queue.Empty:
-                continue
-            if request is None:
-                continue
-            try:
-                self.finish_request(request, client_address)
-            except Exception:
-                self.handle_error(request, client_address)
-            finally:
-                self.shutdown_request(request)
-        os._exit(0)
-
-    def process_request(self, request, client_address):
-        self.requestqueue.put((request, client_address))
-
-    def server_close(self):
-        super().server_close()
-        self.quit = True
-        self.requestqueue.put((None, None))
-        self.handlerthread.join()
-
 def create_server(addr, dbname, prefix=''):
     class Handler(HashEquivalenceServer):
         pass
 
     db = sqlite3.connect(dbname)
     db.row_factory = sqlite3.Row
+    db.execute("PRAGMA synchronous = OFF;")
+    db.execute("PRAGMA journal_mode = MEMORY;")
 
     Handler.prefix = prefix
-    Handler.db = None
+    Handler.db = db
     Handler.dbname = dbname
 
     with contextlib.closing(db.cursor()) as cursor:
@@ -223,7 +171,7 @@ def create_server(addr, dbname, prefix=''):
         cursor.execute('CREATE INDEX IF NOT EXISTS taskhash_lookup ON tasks_v2 (method, taskhash)')
         cursor.execute('CREATE INDEX IF NOT EXISTS outhash_lookup ON tasks_v2 (method, outhash)')
 
-    ret = ThreadedHTTPServer(addr, Handler)
+    ret = HashHTTPServer(addr, Handler)
 
     logger.info('Starting server on %s\n', ret.server_port)
 
Signature · 1 line
2.20.1