sstate.bbclass: track found files on mirrors with a counter
PATCH 1/2 sstate.bbclass: track found files on mirrors with a counter
We don't need extra python collections to count the found files on the sstate cache and sstate mirrors.
The main found collections provides all the files that were found, then we only need to count the files on sstate mirror
--- meta/classes/sstate.bbclass | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-)
meta/classes/sstate.bbclass +5 -6
@@ -893,8 +893,6 @@ BB_HASHCHECK_FUNCTION = "sstate_checkhashes" def sstate_checkhashes(sq_data, d, siginfo=False, currentcount=0, summary=True, **kwargs): found = set() - foundLocal = set() - foundNet = set() missed = set() def gethash(task):
@@ -927,12 +925,12 @@ def sstate_checkhashes(sq_data, d, siginfo=False, currentcount=0, summary=True, if os.path.exists(sstatefile): bb.debug(2, "SState: Found valid sstate file %s" % sstatefile) found.add(tid) - foundLocal.add(tid) continue else: - missed.add(tid) bb.debug(2, "SState: Looked for but didn't find file %s" % sstatefile) + missed.add(tid) + foundMirrors = 0 mirrors = d.getVar("SSTATE_MIRRORS") if mirrors: # Copy the data object and override DL_DIR and SRC_URI
@@ -972,8 +970,8 @@ def sstate_checkhashes(sq_data, d, siginfo=False, currentcount=0, summary=True, connection_cache=thread_worker.connection_cache) fetcher.checkstatus() bb.debug(2, "SState: Successful fetch test for %s" % srcuri) + foundMirrors += 1 found.add(tid) - foundNet.add(tid) if tid in missed: missed.remove(tid) except:
@@ -1034,7 +1032,8 @@ def sstate_checkhashes(sq_data, d, siginfo=False, currentcount=0, summary=True, match = 0 if total: match = len(found) / total * 100 - bb.plain("Sstate summary: Wanted %d Local %d Network %d Missed %d Current %d (%d%% match, %d%% complete)" % (total, len(foundLocal), len(foundNet),len(missed), currentcount, match, complete)) + bb.plain("Sstate summary: Wanted %d Local %d Mirrors %d Missed %d Current %d (%d%% match, %d%% complete)" % + (total, len(found)-foundMirrors, foundMirrors, len(missed), currentcount, match, complete)) if hasattr(bb.parse.siggen, "checkhashes"): bb.parse.siggen.checkhashes(sq_data, missed, found, d)
Signature · 1 line
2.33.0
PATCH 2/2 sstate.bbclass: cleanup the sstate mirror progress check
We only has the progress bar when we have more than 100 objects. So check for this and store the result to show the progress bar.
--- meta/classes/sstate.bbclass | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-)
meta/classes/sstate.bbclass +8 -4
@@ -978,11 +978,11 @@ def sstate_checkhashes(sq_data, d, siginfo=False, currentcount=0, summary=True, missed.add(tid) bb.debug(2, "SState: Unsuccessful fetch test for %s" % srcuri) pass - if len(tasklist) >= min_tasks: + + if progress: bb.event.fire(bb.event.ProcessProgress(msg, len(tasklist) - thread_worker.tasks.qsize()), d) tasklist = [] - min_tasks = 100 for tid in sq_data['hash']: if tid in found: continue
@@ -993,7 +993,11 @@ def sstate_checkhashes(sq_data, d, siginfo=False, currentcount=0, summary=True, if tasklist: nproc = min(int(d.getVar("BB_NUMBER_THREADS")), len(tasklist)) - if len(tasklist) >= min_tasks: + progress = False + if len(tasklist) >= 100: + progress = True + + if progress: msg = "Checking sstate mirror object availability" bb.event.fire(bb.event.ProcessStarted(msg, len(tasklist)), d)
@@ -1006,7 +1010,7 @@ def sstate_checkhashes(sq_data, d, siginfo=False, currentcount=0, summary=True, pool.wait_completion() bb.event.disable_threadlock() - if len(tasklist) >= min_tasks: + if progress: bb.event.fire(bb.event.ProcessFinished(msg), d) inheritlist = d.getVar("INHERIT")
Signature · 1 line
2.33.0
Hello Jose,
Quoted 5 lines · Jose Quaresma
We don't need extra python collections to count the found files on the sstate cache and sstate mirrors.
The main found collections provides all the files that were found, then we only need to count the files on sstate mirror
I believe this is the cause of:
https://autobuilder.yoctoproject.org/typhoon/#/builders/63/builds/3999
https://autobuilder.yoctoproject.org/typhoon/#/builders/53/builds/4039
https://autobuilder.yoctoproject.org/typhoon/#/builders/42/builds/4019
https://autobuilder.yoctoproject.org/typhoon/#/builders/97/builds/3338
https://autobuilder.yoctoproject.org/typhoon/#/builders/60/builds/4012
https://autobuilder.yoctoproject.org/typhoon/#/builders/74/builds/4004
https://autobuilder.yoctoproject.org/typhoon/#/builders/59/builds/4009
https://autobuilder.yoctoproject.org/typhoon/#/builders/73/builds/4004
I removed both patches and all the builds passed
Quoted 60 lines
Signed-off-by Jose Quaresma <[email protected]>--- meta/classes/sstate.bbclass | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-)meta/classes/sstate.bbclass +5
-6@@ -893,8 +893,6 @@ BB_HASHCHECK_FUNCTION = "sstate_checkhashes" def sstate_checkhashes(sq_data, d, siginfo=False, currentcount=0, summary=True, **kwargs): found = set() - foundLocal = set() - foundNet = set() missed = set() def gethash(task):@@ -927,12 +925,12 @@ def sstate_checkhashes(sq_data, d, siginfo=False, currentcount=0, summary=True, if os.path.exists(sstatefile): bb.debug(2, "SState: Found valid sstate file %s" % sstatefile) found.add(tid) - foundLocal.add(tid) continue else: - missed.add(tid) bb.debug(2, "SState: Looked for but didn't find file %s" % sstatefile) + missed.add(tid) + foundMirrors = 0 mirrors = d.getVar("SSTATE_MIRRORS") if mirrors: # Copy the data object and override DL_DIR and SRC_URI@@ -972,8 +970,8 @@ def sstate_checkhashes(sq_data, d, siginfo=False, currentcount=0, summary=True, connection_cache=thread_worker.connection_cache) fetcher.checkstatus() bb.debug(2, "SState: Successful fetch test for %s" % srcuri) + foundMirrors += 1 found.add(tid) - foundNet.add(tid) if tid in missed: missed.remove(tid) except:@@ -1034,7 +1032,8 @@ def sstate_checkhashes(sq_data, d, siginfo=False, currentcount=0, summary=True, match = 0 if total: match = len(found) / total * 100 - bb.plain("Sstate summary: Wanted %d Local %d Network %d Missed %d Current %d (%d%% match, %d%% complete)" % (total, len(foundLocal), len(foundNet),len(missed), currentcount, match, complete)) + bb.plain("Sstate summary: Wanted %d Local %d Mirrors %d Missed %d Current %d (%d%% match, %d%% complete)" % + (total, len(found)-foundMirrors, foundMirrors, len(missed), currentcount, match, complete)) if hasattr(bb.parse.siggen, "checkhashes"): bb.parse.siggen.checkhashes(sq_data, missed, found, d)Signature · 1 line
2.33.0
Signature · 3 lines
Alexandre Belloni, co-owner and COO, Bootlin Embedded Linux and Kernel engineering https://bootlin.com
Hi Alexandre,
I've had some problems in the past with this patchset and this time I removed what I considered the most problematic.
I can't see the problems that are present here. Any help in the review is welcome.
Jose
Alexandre Belloni <[email protected]> escreveu no dia terça, 21/09/2021 à(s) 11:43:
Quoted 89 lines
Hello Jose,
Quoted 5 lines · Jose Quaresma
We don't need extra python collections to count the found files on the sstate cache and sstate mirrors.
The main found collections provides all the files that were found, then we only need to count the files on sstate mirrorI believe this is the cause of:
https://autobuilder.yoctoproject.org/typhoon/#/builders/63/builds/3999
https://autobuilder.yoctoproject.org/typhoon/#/builders/53/builds/4039
https://autobuilder.yoctoproject.org/typhoon/#/builders/42/builds/4019
https://autobuilder.yoctoproject.org/typhoon/#/builders/97/builds/3338
https://autobuilder.yoctoproject.org/typhoon/#/builders/60/builds/4012
https://autobuilder.yoctoproject.org/typhoon/#/builders/74/builds/4004
https://autobuilder.yoctoproject.org/typhoon/#/builders/59/builds/4009
https://autobuilder.yoctoproject.org/typhoon/#/builders/73/builds/4004I removed both patches and all the builds passed
Quoted 54 lines
Signed-off-by Jose Quaresma <[email protected]>--- meta/classes/sstate.bbclass | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-)meta/classes/sstate.bbclass +5
-7@@ -893,8 +893,6 @@ BB_HASHCHECK_FUNCTION = "sstate_checkhashes" def sstate_checkhashes(sq_data, d, siginfo=False, currentcount=0, summary=True, **kwargs): found = set() - foundLocal = set() - foundNet = set() missed = set() def gethash(task):@@ -927,12 +925,12 @@ def sstate_checkhashes(sq_data, d, siginfo=False, currentcount=0, summary=True, if os.path.exists(sstatefile): bb.debug(2, "SState: Found valid sstate file %s" % sstatefile) found.add(tid) - foundLocal.add(tid) continue else: - missed.add(tid) bb.debug(2, "SState: Looked for but didn't find file %s" % sstatefile) + missed.add(tid) + foundMirrors = 0 mirrors = d.getVar("SSTATE_MIRRORS") if mirrors: # Copy the data object and override DL_DIR and SRC_URI@@ -972,8 +970,8 @@ def sstate_checkhashes(sq_data, d, siginfo=False, currentcount=0, summary=True, connection_cache=thread_worker.connection_cache) fetcher.checkstatus() bb.debug(2, "SState: Successful fetch test for %s" % srcuri) + foundMirrors += 1 found.add(tid) - foundNet.add(tid) if tid in missed: missed.remove(tid) except:@@ -1034,7 +1032,8 @@ def sstate_checkhashes(sq_data, d, siginfo=False, currentcount=0, summary=True, match = 0 if total: match = len(found) / total * 100 - bb.plain("Sstate summary: Wanted %d Local %d Network %d Missed %d Current %d (%d%% match, %d%% complete)" % (total, len(foundLocal), len(foundNet),len(missed), currentcount, match, complete)) + bb.plain("Sstate summary: Wanted %d Local %d Mirrors %d Missed %d Current %d (%d%% match, %d%% complete)" % + (total, len(found)-foundMirrors, foundMirrors, len(missed), currentcount, match, complete)) if hasattr(bb.parse.siggen, "checkhashes"): bb.parse.siggen.checkhashes(sq_data, missed, found, d) --Quoted 6 lines
2.33.0
--
Alexandre Belloni, co-owner and COO, Bootlin
Embedded Linux and Kernel engineering
https://bootlin.com
Signature · 3 lines
Best regards, José Quaresma
Hi Alexandre,
Alexandre Belloni <[email protected]> escreveu no dia terça, 21/09/2021 à(s) 11:43:
Quoted 62 lines
Hello Jose,
Quoted 5 lines · Jose Quaresma
We don't need extra python collections to count the found files on the sstate cache and sstate mirrors.
The main found collections provides all the files that were found, then we only need to count the files on sstate mirrorI believe this is the cause of:
https://autobuilder.yoctoproject.org/typhoon/#/builders/63/builds/3999
https://autobuilder.yoctoproject.org/typhoon/#/builders/53/builds/4039
https://autobuilder.yoctoproject.org/typhoon/#/builders/42/builds/4019
https://autobuilder.yoctoproject.org/typhoon/#/builders/97/builds/3338
https://autobuilder.yoctoproject.org/typhoon/#/builders/60/builds/4012
https://autobuilder.yoctoproject.org/typhoon/#/builders/74/builds/4004
https://autobuilder.yoctoproject.org/typhoon/#/builders/59/builds/4009
https://autobuilder.yoctoproject.org/typhoon/#/builders/73/builds/4004I removed both patches and all the builds passed
meta/classes/sstate.bbclass BB_HASHCHECK_FUNCTION = "sstate_checkhashes" · line 894
Quoted 12 lines
Signed-off-by Jose Quaresma <[email protected]>--- meta/classes/sstate.bbclass | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-)meta/classes/sstate.bbclass +0
-0@@ -893,8 +893,6 @@ BB_HASHCHECK_FUNCTION = "sstate_checkhashes" def sstate_checkhashes(sq_data, d, siginfo=False, currentcount=0,summary=True, **kwargs):
meta/classes/sstate.bbclass def sstate_checkhashes(sq_data, d, siginfo=False,
Quoted 7 lines
meta/classes/sstate.bbclass +0
-2found = set() - foundLocal = set() - foundNet = set() missed = set() def gethash(task):@@ -927,12 +925,12 @@ def sstate_checkhashes(sq_data, d, siginfo=False,currentcount=0, summary=True,
meta/classes/sstate.bbclass def sstate_checkhashes(sq_data, d, siginfo=False, · line 926
Quoted 2 lines
meta/classes/sstate.bbclass +0
-0if os.path.exists(sstatefile): bb.debug(2, "SState: Found valid sstate file %s" %sstatefile)
meta/classes/sstate.bbclass def sstate_checkhashes(sq_data, d, siginfo=False, currentcount=0, summary=True,
Quoted 6 lines
meta/classes/sstate.bbclass +0
-2found.add(tid) - foundLocal.add(tid) continue else: - missed.add(tid) bb.debug(2, "SState: Looked for but didn't find file %s" %sstatefile)
meta/classes/sstate.bbclass def sstate_checkhashes(sq_data, d, siginfo=False,
Quoted 7 lines
meta/classes/sstate.bbclass +2
-0+ missed.add(tid) + foundMirrors = 0 mirrors = d.getVar("SSTATE_MIRRORS") if mirrors: # Copy the data object and override DL_DIR and SRC_URI@@ -972,8 +970,8 @@ def sstate_checkhashes(sq_data, d, siginfo=False,currentcount=0, summary=True,
Quoted 1 line
Quoted 6 lines
connection_cache=thread_worker.connection_cache)
Quoted 2 lines
fetcher.checkstatus()
bb.debug(2, "SState: Successful fetch test for %s" %srcuri)
Quoted 1 line
+ foundMirrors += 1
After looking again at that patch I think the problem may be here as it increments the counter inside of the thread pool without any lock. I sent a new version that uses a different approach to count the files that was found on sstate mirror.
Quoted 42 lines
meta/classes/sstate.bbclass def sstate_checkhashes(sq_data, d, siginfo=False,
Quoted 6 lines
found.add(tid)meta/classes/sstate.bbclass +0
-1- foundNet.add(tid) if tid in missed: missed.remove(tid) except:@@ -1034,7 +1032,8 @@ def sstate_checkhashes(sq_data, d, siginfo=False,currentcount=0, summary=True,
meta/classes/sstate.bbclass def sstate_checkhashes(sq_data, d, siginfo=False, currentcount=0, summary=True,
Quoted 4 lines
meta/classes/sstate.bbclass +0
-1match = 0 if total: match = len(found) / total * 100 - bb.plain("Sstate summary: Wanted %d Local %d Network %d Missed%d Current %d (%d%% match, %d%% complete)" % (total, len(foundLocal), len(foundNet),len(missed), currentcount, match, complete))
meta/classes/sstate.bbclass def sstate_checkhashes(sq_data, d, siginfo=False, · line 1035
Quoted 1 line
meta/classes/sstate.bbclass +1
-0+ bb.plain("Sstate summary: Wanted %d Local %d Mirrors %d Missed%d Current %d (%d%% match, %d%% complete)" %
meta/classes/sstate.bbclass def sstate_checkhashes(sq_data, d, siginfo=False, · line 1036
Quoted 1 line
meta/classes/sstate.bbclass +1
-0+ (total, len(found)-foundMirrors, foundMirrors, len(missed),currentcount, match, complete))
Quoted 11 lines
meta/classes/sstate.bbclass +0
-1if hasattr(bb.parse.siggen, "checkhashes"): bb.parse.siggen.checkhashes(sq_data, missed, found, d) --2.33.0
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group. View/Reply Online (#156164):https://lists.openembedded.org/g/openembedded-core/message/156164
Quoted 3 lines
Mute This Topic: https://lists.openembedded.org/mt/85726949/3617179
Group Owner: [email protected]
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [Quoted 2 lines
-=-=-=-=-=-=-=-=-=-=-=-
--
Alexandre Belloni, co-owner and COO, Bootlin
Embedded Linux and Kernel engineering
https://bootlin.com
Signature · 3 lines
Best regards, José Quaresma