Skip to contentSkip to conversation

sstate: cleanup the sstate mirror progress check

PATCH New Jose Quaresma 1 participant 3 messages Nov 9, 2021

lore
Share & tools
lore
#0 lore raw

PATCH 1/3 sstate: cleanup the sstate mirror progress check

We only have the progress bar when there are more than 100 objects. So we can check for this and store the result to show the progress bar.

---
 meta/classes/sstate.bbclass | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)
meta/classes/sstate.bbclass +5 -4
index 849723d4dc..3f6a1fe99f 100644
@@ -996,11 +996,11 @@ def sstate_checkhashes(sq_data, d, siginfo=False, currentcount=0, summary=True,
                 bb.debug(2, "SState: Unsuccessful fetch test for %s (%s)" % (srcuri, e))
             except Exception as e:
                 bb.error("SState: cannot test %s: %s" % (srcuri, e))
-            if len(tasklist) >= min_tasks:
+
+            if progress:
                 bb.event.fire(bb.event.ProcessProgress(msg, len(tasklist) - thread_worker.tasks.qsize()), d)
 
         tasklist = []
-        min_tasks = 100
         for tid in sq_data['hash']:
             if tid in found:
                 continue
@@ -1011,7 +1011,8 @@ def sstate_checkhashes(sq_data, d, siginfo=False, currentcount=0, summary=True,
         if tasklist:
             nproc = min(int(d.getVar("BB_NUMBER_THREADS")), len(tasklist))
 
-            if len(tasklist) >= min_tasks:
+            progress = len(tasklist) >= 100
+            if progress:
                 msg = "Checking sstate mirror object availability"
                 bb.event.fire(bb.event.ProcessStarted(msg, len(tasklist)), d)
 
@@ -1024,7 +1025,7 @@ def sstate_checkhashes(sq_data, d, siginfo=False, currentcount=0, summary=True,
             pool.wait_completion()
             bb.event.disable_threadlock()
 
-            if len(tasklist) >= min_tasks:
+            if progress:
                 bb.event.fire(bb.event.ProcessFinished(msg), d)
 
     inheritlist = d.getVar("INHERIT")
Signature · 1 line
2.33.1
#1 lore raw

PATCH 2/3 sstate: only search on the mirrors for the missing files

The existence of missed files on the local sstate cache is the condition to search for it on the sstate mirror.
The missed collection are populated when we search the files on the local sstate cache.

- When we have sstate mirrors we don't need to iterate all files again because we already know what's missing.

- When we found a file on the sstate mirror is because of they are missing on the local sstate mirror.
So avoid checking if the hash exists on the missed collection.

- When we can't find the file on the sstate mirror we don't need to add the hash again to the missed collection as it already exists.

---
 meta/classes/sstate.bbclass | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)
meta/classes/sstate.bbclass +2 -6
index 3f6a1fe99f..88bd84a06c 100644
@@ -989,10 +989,8 @@ def sstate_checkhashes(sq_data, d, siginfo=False, currentcount=0, summary=True,
                 fetcher.checkstatus()
                 bb.debug(2, "SState: Successful fetch test for %s" % srcuri)
                 found.add(tid)
-                if tid in missed:
-                    missed.remove(tid)
+                missed.remove(tid)
             except bb.fetch2.FetchError as e:
-                missed.add(tid)
                 bb.debug(2, "SState: Unsuccessful fetch test for %s (%s)" % (srcuri, e))
             except Exception as e:
                 bb.error("SState: cannot test %s: %s" % (srcuri, e))
@@ -1001,9 +999,7 @@ def sstate_checkhashes(sq_data, d, siginfo=False, currentcount=0, summary=True,
                 bb.event.fire(bb.event.ProcessProgress(msg, len(tasklist) - thread_worker.tasks.qsize()), d)
 
         tasklist = []
-        for tid in sq_data['hash']:
-            if tid in found:
-                continue
+        for tid in missed:
             spec, extrapath, tname = getpathcomponents(tid, d)
             sstatefile = d.expand(extrapath + generate_sstatefn(spec, gethash(tid), tname, siginfo, d))
             tasklist.append((tid, sstatefile))
Signature · 1 line
2.33.1
#2 lore raw

PATCH 3/3 sstate: add a function to get the sstate file name in sstate_checkhashes

In reply to Jose Quaresma
---
 meta/classes/sstate.bbclass | 16 +++++++---------
 1 file changed, 7 insertions(+), 9 deletions(-)
meta/classes/sstate.bbclass +7 -9
index 88bd84a06c..0a78a2fb69 100644
@@ -934,12 +934,13 @@ def sstate_checkhashes(sq_data, d, siginfo=False, currentcount=0, summary=True,
 
         return spec, extrapath, tname
 
+    def getsstatefile(tid, siginfo, d):
+        spec, extrapath, tname = getpathcomponents(tid, d)
+        return extrapath + generate_sstatefn(spec, gethash(tid), tname, siginfo, d)
 
     for tid in sq_data['hash']:
 
-        spec, extrapath, tname = getpathcomponents(tid, d)
-
-        sstatefile = d.expand("${SSTATE_DIR}/" + extrapath + generate_sstatefn(spec, gethash(tid), tname, siginfo, d))
+        sstatefile = d.expand("${SSTATE_DIR}/" + getsstatefile(tid, siginfo, d))
 
         if os.path.exists(sstatefile):
             found.add(tid)
@@ -1000,8 +1001,7 @@ def sstate_checkhashes(sq_data, d, siginfo=False, currentcount=0, summary=True,
 
         tasklist = []
         for tid in missed:
-            spec, extrapath, tname = getpathcomponents(tid, d)
-            sstatefile = d.expand(extrapath + generate_sstatefn(spec, gethash(tid), tname, siginfo, d))
+            sstatefile = d.expand(getsstatefile(tid, siginfo, d))
             tasklist.append((tid, sstatefile))
 
         if tasklist:
@@ -1028,12 +1028,10 @@ def sstate_checkhashes(sq_data, d, siginfo=False, currentcount=0, summary=True,
     if "toaster" in inheritlist:
         evdata = {'missed': [], 'found': []};
         for tid in missed:
-            spec, extrapath, tname = getpathcomponents(tid, d)
-            sstatefile = d.expand(extrapath + generate_sstatefn(spec, gethash(tid), tname, False, d))
+            sstatefile = d.expand(getsstatefile(tid, False, d))
             evdata['missed'].append((bb.runqueue.fn_from_tid(tid), bb.runqueue.taskname_from_tid(tid), gethash(tid), sstatefile ) )
         for tid in found:
-            spec, extrapath, tname = getpathcomponents(tid, d)
-            sstatefile = d.expand(extrapath + generate_sstatefn(spec, gethash(tid), tname, False, d))
+            sstatefile = d.expand(getsstatefile(tid, False, d))
             evdata['found'].append((bb.runqueue.fn_from_tid(tid), bb.runqueue.taskname_from_tid(tid), gethash(tid), sstatefile ) )
         bb.event.fire(bb.event.MetadataEvent("MissedSstate", evdata), d)
 
Signature · 1 line
2.33.1