Skip to contentSkip to conversation

cooker: Keep track of watched files using a set instead of a list

PATCH Superseded Peter Kjellerstedt 1 participant 2 messages Jan 7, 2020

v1 of 2 Superseded Latest is v2 →

lore
Share & tools
lore
#0 lore raw

PATCH 1/2 cooker: Keep track of watched files using a set instead of a list

When there are many watched files, keeping track of them using lists is suboptimal. Using sets improves the performance considerably.

---
 bitbake/lib/bb/cooker.py | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)
bitbake/lib/bb/cooker.py +8 -8
index e6442bff93..b74affa7ec 100644
@@ -169,8 +169,8 @@ class BBCooker:
         bb.debug(1, "BBCooker pyinotify1 %s" % time.time())
         sys.stdout.flush()
 
-        self.configwatcher.bbseen = []
-        self.configwatcher.bbwatchedfiles = []
+        self.configwatcher.bbseen = set()
+        self.configwatcher.bbwatchedfiles = set()
         self.confignotifier = pyinotify.Notifier(self.configwatcher, self.config_notifications)
         bb.debug(1, "BBCooker pyinotify2 %s" % time.time())
         sys.stdout.flush()
@@ -180,8 +180,8 @@ class BBCooker:
         self.watcher = pyinotify.WatchManager()
         bb.debug(1, "BBCooker pyinotify3 %s" % time.time())
         sys.stdout.flush()
-        self.watcher.bbseen = []
-        self.watcher.bbwatchedfiles = []
+        self.watcher.bbseen = set()
+        self.watcher.bbwatchedfiles = set()
         self.notifier = pyinotify.Notifier(self.watcher, self.notifications)
 
         bb.debug(1, "BBCooker pyinotify complete %s" % time.time())
@@ -278,14 +278,14 @@ class BBCooker:
         if not watcher:
             watcher = self.watcher
         for i in deps:
-            watcher.bbwatchedfiles.append(i[0])
+            watcher.bbwatchedfiles.add(i[0])
             if dirs:
                 f = i[0]
             else:
                 f = os.path.dirname(i[0])
             if f in watcher.bbseen:
                 continue
-            watcher.bbseen.append(f)
+            watcher.bbseen.add(f)
             watchtarget = None
             while True:
                 # We try and add watches for files that don't exist but if they did, would influence
@@ -294,7 +294,7 @@ class BBCooker:
                 try:
                     watcher.add_watch(f, self.watchmask, quiet=False)
                     if watchtarget:
-                        watcher.bbwatchedfiles.append(watchtarget)
+                        watcher.bbwatchedfiles.add(watchtarget)
                     break
                 except pyinotify.WatchManagerError as e:
                     if 'ENOENT' in str(e):
@@ -302,7 +302,7 @@ class BBCooker:
                         f = os.path.dirname(f)
                         if f in watcher.bbseen:
                             break
-                        watcher.bbseen.append(f)
+                        watcher.bbseen.add(f)
                         continue
                     if 'ENOSPC' in str(e):
                         providerlog.error("No space left on device or exceeds fs.inotify.max_user_watches?")
Signature · 1 line
2.21.1
#1 lore raw

PATCH 2/2 cooker: Ignore notifications about changes to the sanity_info file

Since this file is written during recipe parsing, having a watch on it (or actually the directory that contains it) will trigger a re-parse the next time bitbake is run and the resident bitbake server is enabled. This causes the sanity_info file to be updated again, which triggers a new re-parse the next time bitbake is run ad infinitum.

Also generalize the ignoring of bitbake-cookerdaemon.log and bitbake.lock to all log files and lock files.

---
 bitbake/lib/bb/cooker.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)
bitbake/lib/bb/cooker.py +3 -2
index b74affa7ec..74c9fbc4c0 100644
@@ -267,8 +267,9 @@ class BBCooker:
             self.parsecache_valid = False
             bb.parse.clear_cache()
             return
-        if event.pathname.endswith("bitbake-cookerdaemon.log") \
-                or event.pathname.endswith("bitbake.lock"):
+        if event.pathname.endswith(".log") \
+                or event.pathname.endswith(".lock") \
+                or event.pathname.endswith("sanity_info"):
             return
         if not event.pathname in self.inotify_modified_files:
             self.inotify_modified_files.append(event.pathname)
Signature · 1 line
2.21.1