cooker: Keep track of watched files using a set instead of a list
PATCH 1/2 cooker: Keep track of watched files using a set instead of a list
When there are many watched files, keeping track of them using lists is suboptimal. Using sets improves the performance considerably.
Signed-off-by Peter Kjellerstedt <[email protected]>
--- bitbake/lib/bb/cooker.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-)
bitbake/lib/bb/cooker.py +8 -8
@@ -169,8 +169,8 @@ class BBCooker: bb.debug(1, "BBCooker pyinotify1 %s" % time.time()) sys.stdout.flush() - self.configwatcher.bbseen = [] - self.configwatcher.bbwatchedfiles = [] + self.configwatcher.bbseen = set() + self.configwatcher.bbwatchedfiles = set() self.confignotifier = pyinotify.Notifier(self.configwatcher, self.config_notifications) bb.debug(1, "BBCooker pyinotify2 %s" % time.time()) sys.stdout.flush()
@@ -180,8 +180,8 @@ class BBCooker: self.watcher = pyinotify.WatchManager() bb.debug(1, "BBCooker pyinotify3 %s" % time.time()) sys.stdout.flush() - self.watcher.bbseen = [] - self.watcher.bbwatchedfiles = [] + self.watcher.bbseen = set() + self.watcher.bbwatchedfiles = set() self.notifier = pyinotify.Notifier(self.watcher, self.notifications) bb.debug(1, "BBCooker pyinotify complete %s" % time.time())
@@ -278,14 +278,14 @@ class BBCooker: if not watcher: watcher = self.watcher for i in deps: - watcher.bbwatchedfiles.append(i[0]) + watcher.bbwatchedfiles.add(i[0]) if dirs: f = i[0] else: f = os.path.dirname(i[0]) if f in watcher.bbseen: continue - watcher.bbseen.append(f) + watcher.bbseen.add(f) watchtarget = None while True: # We try and add watches for files that don't exist but if they did, would influence
@@ -294,7 +294,7 @@ class BBCooker: try: watcher.add_watch(f, self.watchmask, quiet=False) if watchtarget: - watcher.bbwatchedfiles.append(watchtarget) + watcher.bbwatchedfiles.add(watchtarget) break except pyinotify.WatchManagerError as e: if 'ENOENT' in str(e):
@@ -302,7 +302,7 @@ class BBCooker: f = os.path.dirname(f) if f in watcher.bbseen: break - watcher.bbseen.append(f) + watcher.bbseen.add(f) continue if 'ENOSPC' in str(e): providerlog.error("No space left on device or exceeds fs.inotify.max_user_watches?")
Signature · 1 line
2.21.1
PATCH 2/2 cooker: Ignore notifications about changes to the sanity_info file
Since this file is written during recipe parsing, having a watch on it (or actually the directory that contains it) will trigger a re-parse the next time bitbake is run and the resident bitbake server is enabled. This causes the sanity_info file to be updated again, which triggers a new re-parse the next time bitbake is run ad infinitum.
Also generalize the ignoring of bitbake-cookerdaemon.log and bitbake.lock to all log files and lock files.
Signed-off-by Peter Kjellerstedt <[email protected]>
--- bitbake/lib/bb/cooker.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-)
bitbake/lib/bb/cooker.py +3 -2
@@ -267,8 +267,9 @@ class BBCooker: self.parsecache_valid = False bb.parse.clear_cache() return - if event.pathname.endswith("bitbake-cookerdaemon.log") \ - or event.pathname.endswith("bitbake.lock"): + if event.pathname.endswith(".log") \ + or event.pathname.endswith(".lock") \ + or event.pathname.endswith("sanity_info"): return if not event.pathname in self.inotify_modified_files: self.inotify_modified_files.append(event.pathname)
Signature · 1 line
2.21.1