Skip to contentSkip to conversation

sstate.bbclass: track found files on mirrors with a counter

PATCH In review Jose Quaresma 2 participants 5 messages Sep 19, 2021 – Oct 3, 2021

lore
Share & tools
lore
#0 lore raw

PATCH 1/2 sstate.bbclass: track found files on mirrors with a counter

We don't need extra python collections to count the found files on the sstate cache and sstate mirrors.
The main found collections provides all the files that were found, then we only need to count the files on sstate mirror

---
 meta/classes/sstate.bbclass | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)
meta/classes/sstate.bbclass +5 -6
index 29679e6a5e..fc156ac81a 100644
@@ -893,8 +893,6 @@ BB_HASHCHECK_FUNCTION = "sstate_checkhashes"
 
 def sstate_checkhashes(sq_data, d, siginfo=False, currentcount=0, summary=True, **kwargs):
     found = set()
-    foundLocal = set()
-    foundNet = set()
     missed = set()
 
     def gethash(task):
@@ -927,12 +925,12 @@ def sstate_checkhashes(sq_data, d, siginfo=False, currentcount=0, summary=True,
         if os.path.exists(sstatefile):
             bb.debug(2, "SState: Found valid sstate file %s" % sstatefile)
             found.add(tid)
-            foundLocal.add(tid)
             continue
         else:
-            missed.add(tid)
             bb.debug(2, "SState: Looked for but didn't find file %s" % sstatefile)
+            missed.add(tid)
 
+    foundMirrors = 0
     mirrors = d.getVar("SSTATE_MIRRORS")
     if mirrors:
         # Copy the data object and override DL_DIR and SRC_URI
@@ -972,8 +970,8 @@ def sstate_checkhashes(sq_data, d, siginfo=False, currentcount=0, summary=True,
                             connection_cache=thread_worker.connection_cache)
                 fetcher.checkstatus()
                 bb.debug(2, "SState: Successful fetch test for %s" % srcuri)
+                foundMirrors += 1
                 found.add(tid)
-                foundNet.add(tid)
                 if tid in missed:
                     missed.remove(tid)
             except:
@@ -1034,7 +1032,8 @@ def sstate_checkhashes(sq_data, d, siginfo=False, currentcount=0, summary=True,
         match = 0
         if total:
             match = len(found) / total * 100
-        bb.plain("Sstate summary: Wanted %d Local %d Network %d Missed %d Current %d (%d%% match, %d%% complete)" % (total, len(foundLocal), len(foundNet),len(missed), currentcount, match, complete))
+        bb.plain("Sstate summary: Wanted %d Local %d Mirrors %d Missed %d Current %d (%d%% match, %d%% complete)" %
+            (total, len(found)-foundMirrors, foundMirrors, len(missed), currentcount, match, complete))
 
     if hasattr(bb.parse.siggen, "checkhashes"):
         bb.parse.siggen.checkhashes(sq_data, missed, found, d)
Signature · 1 line
2.33.0
#1 lore raw

PATCH 2/2 sstate.bbclass: cleanup the sstate mirror progress check

We only has the progress bar when we have more than 100 objects. So check for this and store the result to show the progress bar.

---
 meta/classes/sstate.bbclass | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)
meta/classes/sstate.bbclass +8 -4
index fc156ac81a..1be20e4055 100644
@@ -978,11 +978,11 @@ def sstate_checkhashes(sq_data, d, siginfo=False, currentcount=0, summary=True,
                 missed.add(tid)
                 bb.debug(2, "SState: Unsuccessful fetch test for %s" % srcuri)
                 pass
-            if len(tasklist) >= min_tasks:
+
+            if progress:
                 bb.event.fire(bb.event.ProcessProgress(msg, len(tasklist) - thread_worker.tasks.qsize()), d)
 
         tasklist = []
-        min_tasks = 100
         for tid in sq_data['hash']:
             if tid in found:
                 continue
@@ -993,7 +993,11 @@ def sstate_checkhashes(sq_data, d, siginfo=False, currentcount=0, summary=True,
         if tasklist:
             nproc = min(int(d.getVar("BB_NUMBER_THREADS")), len(tasklist))
 
-            if len(tasklist) >= min_tasks:
+            progress = False
+            if len(tasklist) >= 100:
+                progress = True
+
+            if progress:
                 msg = "Checking sstate mirror object availability"
                 bb.event.fire(bb.event.ProcessStarted(msg, len(tasklist)), d)
 
@@ -1006,7 +1010,7 @@ def sstate_checkhashes(sq_data, d, siginfo=False, currentcount=0, summary=True,
             pool.wait_completion()
             bb.event.disable_threadlock()
 
-            if len(tasklist) >= min_tasks:
+            if progress:
                 bb.event.fire(bb.event.ProcessFinished(msg), d)
 
     inheritlist = d.getVar("INHERIT")
Signature · 1 line
2.33.0
#2 lore raw
In reply to Jose Quaresma

Hello Jose,

Quoted 5 lines · Jose Quaresma

We don't need extra python collections to count the found files on the sstate cache and sstate mirrors.
The main found collections provides all the files that were found, then we only need to count the files on sstate mirror

I believe this is the cause of:
https://autobuilder.yoctoproject.org/typhoon/#/builders/63/builds/3999
https://autobuilder.yoctoproject.org/typhoon/#/builders/53/builds/4039
https://autobuilder.yoctoproject.org/typhoon/#/builders/42/builds/4019
https://autobuilder.yoctoproject.org/typhoon/#/builders/97/builds/3338
https://autobuilder.yoctoproject.org/typhoon/#/builders/60/builds/4012
https://autobuilder.yoctoproject.org/typhoon/#/builders/74/builds/4004
https://autobuilder.yoctoproject.org/typhoon/#/builders/59/builds/4009
https://autobuilder.yoctoproject.org/typhoon/#/builders/73/builds/4004

I removed both patches and all the builds passed

Quoted 60 lines
---
 meta/classes/sstate.bbclass | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)
meta/classes/sstate.bbclass +5 -6
index 29679e6a5e..fc156ac81a 100644
@@ -893,8 +893,6 @@ BB_HASHCHECK_FUNCTION = "sstate_checkhashes"
 
 def sstate_checkhashes(sq_data, d, siginfo=False, currentcount=0, summary=True, **kwargs):
     found = set()
-    foundLocal = set()
-    foundNet = set()
     missed = set()
 
     def gethash(task):
@@ -927,12 +925,12 @@ def sstate_checkhashes(sq_data, d, siginfo=False, currentcount=0, summary=True,
         if os.path.exists(sstatefile):
             bb.debug(2, "SState: Found valid sstate file %s" % sstatefile)
             found.add(tid)
-            foundLocal.add(tid)
             continue
         else:
-            missed.add(tid)
             bb.debug(2, "SState: Looked for but didn't find file %s" % sstatefile)
+            missed.add(tid)
 
+    foundMirrors = 0
     mirrors = d.getVar("SSTATE_MIRRORS")
     if mirrors:
         # Copy the data object and override DL_DIR and SRC_URI
@@ -972,8 +970,8 @@ def sstate_checkhashes(sq_data, d, siginfo=False, currentcount=0, summary=True,
                             connection_cache=thread_worker.connection_cache)
                 fetcher.checkstatus()
                 bb.debug(2, "SState: Successful fetch test for %s" % srcuri)
+                foundMirrors += 1
                 found.add(tid)
-                foundNet.add(tid)
                 if tid in missed:
                     missed.remove(tid)
             except:
@@ -1034,7 +1032,8 @@ def sstate_checkhashes(sq_data, d, siginfo=False, currentcount=0, summary=True,
         match = 0
         if total:
             match = len(found) / total * 100
-        bb.plain("Sstate summary: Wanted %d Local %d Network %d Missed %d Current %d (%d%% match, %d%% complete)" % (total, len(foundLocal), len(foundNet),len(missed), currentcount, match, complete))
+        bb.plain("Sstate summary: Wanted %d Local %d Mirrors %d Missed %d Current %d (%d%% match, %d%% complete)" %
+            (total, len(found)-foundMirrors, foundMirrors, len(missed), currentcount, match, complete))
 
     if hasattr(bb.parse.siggen, "checkhashes"):
         bb.parse.siggen.checkhashes(sq_data, missed, found, d)
Signature · 1 line
2.33.0
Signature · 3 lines
Alexandre Belloni, co-owner and COO, Bootlin
Embedded Linux and Kernel engineering
https://bootlin.com
#3 lore raw

Hi Alexandre,

I've had some problems in the past with this patchset and this time I removed what I considered the most problematic.
I can't see the problems that are present here. Any help in the review is welcome.

Jose

Alexandre Belloni <[email protected]> escreveu no dia terça, 21/09/2021 à(s) 11:43:

Quoted 89 lines

Hello Jose,

Quoted 5 lines · Jose Quaresma

We don't need extra python collections to count the found files on the sstate cache and sstate mirrors.
The main found collections provides all the files that were found, then we only need to count the files on sstate mirror

I believe this is the cause of:
https://autobuilder.yoctoproject.org/typhoon/#/builders/63/builds/3999
https://autobuilder.yoctoproject.org/typhoon/#/builders/53/builds/4039
https://autobuilder.yoctoproject.org/typhoon/#/builders/42/builds/4019
https://autobuilder.yoctoproject.org/typhoon/#/builders/97/builds/3338
https://autobuilder.yoctoproject.org/typhoon/#/builders/60/builds/4012
https://autobuilder.yoctoproject.org/typhoon/#/builders/74/builds/4004
https://autobuilder.yoctoproject.org/typhoon/#/builders/59/builds/4009
https://autobuilder.yoctoproject.org/typhoon/#/builders/73/builds/4004

I removed both patches and all the builds passed

Quoted 54 lines
---
 meta/classes/sstate.bbclass | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)
meta/classes/sstate.bbclass +5 -7
index 29679e6a5e..fc156ac81a 100644
@@ -893,8 +893,6 @@ BB_HASHCHECK_FUNCTION = "sstate_checkhashes"
 
 def sstate_checkhashes(sq_data, d, siginfo=False, currentcount=0, summary=True, **kwargs):
     found = set()
-    foundLocal = set()
-    foundNet = set()
     missed = set()
 
     def gethash(task):
@@ -927,12 +925,12 @@ def sstate_checkhashes(sq_data, d, siginfo=False, currentcount=0, summary=True,
         if os.path.exists(sstatefile):
             bb.debug(2, "SState: Found valid sstate file %s" % sstatefile)
             found.add(tid)
-            foundLocal.add(tid)
             continue
         else:
-            missed.add(tid)
             bb.debug(2, "SState: Looked for but didn't find file %s" % sstatefile)
+            missed.add(tid)
 
+    foundMirrors = 0
     mirrors = d.getVar("SSTATE_MIRRORS")
     if mirrors:
         # Copy the data object and override DL_DIR and SRC_URI
@@ -972,8 +970,8 @@ def sstate_checkhashes(sq_data, d, siginfo=False, currentcount=0, summary=True,
                             connection_cache=thread_worker.connection_cache)
                 fetcher.checkstatus()
                 bb.debug(2, "SState: Successful fetch test for %s" % srcuri)
+                foundMirrors += 1
                 found.add(tid)
-                foundNet.add(tid)
                 if tid in missed:
                     missed.remove(tid)
             except:
@@ -1034,7 +1032,8 @@ def sstate_checkhashes(sq_data, d, siginfo=False, currentcount=0, summary=True,
         match = 0
         if total:
             match = len(found) / total * 100
-        bb.plain("Sstate summary: Wanted %d Local %d Network %d Missed %d Current %d (%d%% match, %d%% complete)" % (total, len(foundLocal), len(foundNet),len(missed), currentcount, match, complete))
+        bb.plain("Sstate summary: Wanted %d Local %d Mirrors %d Missed %d Current %d (%d%% match, %d%% complete)" %
+            (total, len(found)-foundMirrors, foundMirrors, len(missed), currentcount, match, complete))
 
     if hasattr(bb.parse.siggen, "checkhashes"):
         bb.parse.siggen.checkhashes(sq_data, missed, found, d)
--
Quoted 6 lines

2.33.0

--
Alexandre Belloni, co-owner and COO, Bootlin
Embedded Linux and Kernel engineering
https://bootlin.com

Signature · 3 lines
Best regards,

José Quaresma
#4 lore raw
In reply to Alexandre Belloni

Hi Alexandre,

Alexandre Belloni <[email protected]> escreveu no dia terça, 21/09/2021 à(s) 11:43:

Quoted 62 lines

Hello Jose,

Quoted 5 lines · Jose Quaresma

We don't need extra python collections to count the found files on the sstate cache and sstate mirrors.
The main found collections provides all the files that were found, then we only need to count the files on sstate mirror

I believe this is the cause of:
https://autobuilder.yoctoproject.org/typhoon/#/builders/63/builds/3999
https://autobuilder.yoctoproject.org/typhoon/#/builders/53/builds/4039
https://autobuilder.yoctoproject.org/typhoon/#/builders/42/builds/4019
https://autobuilder.yoctoproject.org/typhoon/#/builders/97/builds/3338
https://autobuilder.yoctoproject.org/typhoon/#/builders/60/builds/4012
https://autobuilder.yoctoproject.org/typhoon/#/builders/74/builds/4004
https://autobuilder.yoctoproject.org/typhoon/#/builders/59/builds/4009
https://autobuilder.yoctoproject.org/typhoon/#/builders/73/builds/4004

I removed both patches and all the builds passed

meta/classes/sstate.bbclass BB_HASHCHECK_FUNCTION = "sstate_checkhashes" · line 894

Quoted 12 lines
---
 meta/classes/sstate.bbclass | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)
meta/classes/sstate.bbclass +0 -0
index 29679e6a5e..fc156ac81a 100644
@@ -893,8 +893,6 @@ BB_HASHCHECK_FUNCTION = "sstate_checkhashes"
 
 def sstate_checkhashes(sq_data, d, siginfo=False, currentcount=0,

summary=True, **kwargs):

meta/classes/sstate.bbclass def sstate_checkhashes(sq_data, d, siginfo=False,

Quoted 7 lines
meta/classes/sstate.bbclass +0 -2
     found = set()
-    foundLocal = set()
-    foundNet = set()
     missed = set()
 
     def gethash(task):
@@ -927,12 +925,12 @@ def sstate_checkhashes(sq_data, d, siginfo=False,

currentcount=0, summary=True,

meta/classes/sstate.bbclass def sstate_checkhashes(sq_data, d, siginfo=False, · line 926

Quoted 2 lines
meta/classes/sstate.bbclass +0 -0
         if os.path.exists(sstatefile):
             bb.debug(2, "SState: Found valid sstate file %s" %

sstatefile)

meta/classes/sstate.bbclass def sstate_checkhashes(sq_data, d, siginfo=False, currentcount=0, summary=True,

Quoted 6 lines
meta/classes/sstate.bbclass +0 -2
             found.add(tid)
-            foundLocal.add(tid)
             continue
         else:
-            missed.add(tid)
             bb.debug(2, "SState: Looked for but didn't find file %s" %

sstatefile)

meta/classes/sstate.bbclass def sstate_checkhashes(sq_data, d, siginfo=False,

Quoted 7 lines
meta/classes/sstate.bbclass +2 -0
+            missed.add(tid)
 
+    foundMirrors = 0
     mirrors = d.getVar("SSTATE_MIRRORS")
     if mirrors:
         # Copy the data object and override DL_DIR and SRC_URI
@@ -972,8 +970,8 @@ def sstate_checkhashes(sq_data, d, siginfo=False,

currentcount=0, summary=True,

Quoted 1 line
Quoted 6 lines

connection_cache=thread_worker.connection_cache)

Quoted 2 lines

fetcher.checkstatus()
bb.debug(2, "SState: Successful fetch test for %s" %

srcuri)

Quoted 1 line

+ foundMirrors += 1

After looking again at that patch I think the problem may be here as it increments the counter inside of the thread pool without any lock. I sent a new version that uses a different approach to count the files that was found on sstate mirror.

https://lists.openembedded.org/g/openembedded-core/topic/patch_sstate_bbclass_count/86052575?p=,,,20,0,0,0::recentpostdate/sticky,,,20,2,0,86052575,previd=1633298061762412833,nextid=1633195213330301948&previd=1633298061762412833&nextid=1633195213330301948

Quoted 42 lines

meta/classes/sstate.bbclass def sstate_checkhashes(sq_data, d, siginfo=False,

Quoted 6 lines

found.add(tid)

meta/classes/sstate.bbclass +0 -1
-                foundNet.add(tid)
                 if tid in missed:
                     missed.remove(tid)
             except:
@@ -1034,7 +1032,8 @@ def sstate_checkhashes(sq_data, d, siginfo=False,

currentcount=0, summary=True,

meta/classes/sstate.bbclass def sstate_checkhashes(sq_data, d, siginfo=False, currentcount=0, summary=True,

Quoted 4 lines
meta/classes/sstate.bbclass +0 -1
         match = 0
         if total:
             match = len(found) / total * 100
-        bb.plain("Sstate summary: Wanted %d Local %d Network %d Missed

%d Current %d (%d%% match, %d%% complete)" % (total, len(foundLocal), len(foundNet),len(missed), currentcount, match, complete))

meta/classes/sstate.bbclass def sstate_checkhashes(sq_data, d, siginfo=False, · line 1035

Quoted 1 line
meta/classes/sstate.bbclass +1 -0
+        bb.plain("Sstate summary: Wanted %d Local %d Mirrors %d Missed

%d Current %d (%d%% match, %d%% complete)" %

meta/classes/sstate.bbclass def sstate_checkhashes(sq_data, d, siginfo=False, · line 1036

Quoted 1 line
meta/classes/sstate.bbclass +1 -0
+            (total, len(found)-foundMirrors, foundMirrors, len(missed),

currentcount, match, complete))

Quoted 11 lines
meta/classes/sstate.bbclass +0 -1
     if hasattr(bb.parse.siggen, "checkhashes"):
         bb.parse.siggen.checkhashes(sq_data, missed, found, d)
--

2.33.0

-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group. View/Reply Online (#156164):

https://lists.openembedded.org/g/openembedded-core/message/156164

Quoted 3 lines

Mute This Topic: https://lists.openembedded.org/mt/85726949/3617179
Group Owner: [email protected]
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [

[email protected]]

Quoted 2 lines

-=-=-=-=-=-=-=-=-=-=-=-

--
Alexandre Belloni, co-owner and COO, Bootlin
Embedded Linux and Kernel engineering
https://bootlin.com

Signature · 3 lines
Best regards,

José Quaresma