Skip to contentSkip to conversation

sstate.bbclass: count the files on mirrors using the pre local files

PATCH In review Jose Quaresma 1 participant 2 messages Oct 3, 2021 – Oct 21, 2021

lore
Share & tools
lore
#0 lore raw

PATCH 1/1 sstate.bbclass: count the files on mirrors using the pre local files

We don't need extra python collections to count the found files on the sstate cache and sstate mirrors.

The main found collections provides all the files that were found. Then we only need to store the nunber of files that found on the local sstate cache and the files that found on the sstate cache mirror is derived from that.

---
 meta/classes/sstate.bbclass | 11 ++++-------
 1 file changed, 4 insertions(+), 7 deletions(-)
meta/classes/sstate.bbclass +4 -7
index 92a73114bb..f06778a3ee 100644
@@ -893,8 +893,6 @@ BB_HASHCHECK_FUNCTION = "sstate_checkhashes"
 
 def sstate_checkhashes(sq_data, d, siginfo=False, currentcount=0, summary=True, **kwargs):
     found = set()
-    foundLocal = set()
-    foundNet = set()
     missed = set()
 
     def gethash(task):
@@ -925,14 +923,13 @@ def sstate_checkhashes(sq_data, d, siginfo=False, currentcount=0, summary=True,
         sstatefile = d.expand("${SSTATE_DIR}/" + extrapath + generate_sstatefn(spec, gethash(tid), tname, siginfo, d))
 
         if os.path.exists(sstatefile):
-            bb.debug(2, "SState: Found valid sstate file %s" % sstatefile)
             found.add(tid)
-            foundLocal.add(tid)
-            continue
+            bb.debug(2, "SState: Found valid sstate file %s" % sstatefile)
         else:
             missed.add(tid)
             bb.debug(2, "SState: Looked for but didn't find file %s" % sstatefile)
 
+    foundLocal = len(found)
     mirrors = d.getVar("SSTATE_MIRRORS")
     if mirrors:
         # Copy the data object and override DL_DIR and SRC_URI
@@ -973,7 +970,6 @@ def sstate_checkhashes(sq_data, d, siginfo=False, currentcount=0, summary=True,
                 fetcher.checkstatus()
                 bb.debug(2, "SState: Successful fetch test for %s" % srcuri)
                 found.add(tid)
-                foundNet.add(tid)
                 if tid in missed:
                     missed.remove(tid)
             except:
@@ -1034,7 +1030,8 @@ def sstate_checkhashes(sq_data, d, siginfo=False, currentcount=0, summary=True,
         match = 0
         if total:
             match = len(found) / total * 100
-        bb.plain("Sstate summary: Wanted %d Local %d Network %d Missed %d Current %d (%d%% match, %d%% complete)" % (total, len(foundLocal), len(foundNet),len(missed), currentcount, match, complete))
+        bb.plain("Sstate summary: Wanted %d Local %d Mirrors %d Missed %d Current %d (%d%% match, %d%% complete)" %
+            (total, foundLocal, len(found)-foundLocal, len(missed), currentcount, match, complete))
 
     if hasattr(bb.parse.siggen, "checkhashes"):
         bb.parse.siggen.checkhashes(sq_data, missed, found, d)
Signature · 1 line
2.33.0
#1 lore raw

ping

Jose Quaresma <[email protected]> escreveu no dia domingo, 3/10/2021 à(s) 22:54:

Quoted 76 lines

We don't need extra python collections to count the found files on the sstate cache and sstate mirrors.

The main found collections provides all the files that were found. Then we only need to store the nunber of files that found on the local sstate cache and the files that found on the sstate cache mirror is derived from that.

---
 meta/classes/sstate.bbclass | 11 ++++-------
 1 file changed, 4 insertions(+), 7 deletions(-)
meta/classes/sstate.bbclass +0 -0
index 92a73114bb..f06778a3ee 100644
@@ -893,8 +893,6 @@ BB_HASHCHECK_FUNCTION = "sstate_checkhashes"
 
 def sstate_checkhashes(sq_data, d, siginfo=False, currentcount=0,

summary=True, **kwargs):
found = set()

meta/classes/sstate.bbclass +2 -5
-    foundLocal = set()
-    foundNet = set()
     missed = set()
 
     def gethash(task):
@@ -925,14 +923,13 @@ def sstate_checkhashes(sq_data, d, siginfo=False,
 currentcount=0, summary=True,
         sstatefile = d.expand("${SSTATE_DIR}/" + extrapath +
 generate_sstatefn(spec, gethash(tid), tname, siginfo, d))
 
         if os.path.exists(sstatefile):
-            bb.debug(2, "SState: Found valid sstate file %s" % sstatefile)
             found.add(tid)
-            foundLocal.add(tid)
-            continue
+            bb.debug(2, "SState: Found valid sstate file %s" % sstatefile)
         else:
             missed.add(tid)
             bb.debug(2, "SState: Looked for but didn't find file %s" %
 sstatefile)
 
+    foundLocal = len(found)

mirrors = d.getVar("SSTATE_MIRRORS")
if mirrors:
# Copy the data object and override DL_DIR and SRC_URI

meta/classes/sstate.bbclass +0 -1
@@ -973,7 +970,6 @@ def sstate_checkhashes(sq_data, d, siginfo=False,
 currentcount=0, summary=True,
                 fetcher.checkstatus()
                 bb.debug(2, "SState: Successful fetch test for %s" %
 srcuri)
                 found.add(tid)
-                foundNet.add(tid)
                 if tid in missed:

missed.remove(tid)
except:

meta/classes/sstate.bbclass +1 -1
@@ -1034,7 +1030,8 @@ def sstate_checkhashes(sq_data, d, siginfo=False,
 currentcount=0, summary=True,
         match = 0
         if total:
             match = len(found) / total * 100
-        bb.plain("Sstate summary: Wanted %d Local %d Network %d Missed %d
 Current %d (%d%% match, %d%% complete)" % (total, len(foundLocal),
 len(foundNet),len(missed), currentcount, match, complete))
+        bb.plain("Sstate summary: Wanted %d Local %d Mirrors %d Missed %d

Current %d (%d%% match, %d%% complete)" %
+ (total, foundLocal, len(found)-foundLocal, len(missed),
currentcount, match, complete))

if hasattr(bb.parse.siggen, "checkhashes"):
bb.parse.siggen.checkhashes(sq_data, missed, found, d)
--
2.33.0

Signature · 3 lines
Best regards,

José Quaresma