sstate: cleanup the sstate mirror progress check
PATCH 1/3 sstate: cleanup the sstate mirror progress check
We only have the progress bar when there are more than 100 objects. So we can check for this and store the result to show the progress bar.
--- meta/classes/sstate.bbclass | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-)
meta/classes/sstate.bbclass +5 -4
@@ -996,11 +996,11 @@ def sstate_checkhashes(sq_data, d, siginfo=False, currentcount=0, summary=True, bb.debug(2, "SState: Unsuccessful fetch test for %s (%s)" % (srcuri, e)) except Exception as e: bb.error("SState: cannot test %s: %s" % (srcuri, e)) - if len(tasklist) >= min_tasks: + + if progress: bb.event.fire(bb.event.ProcessProgress(msg, len(tasklist) - thread_worker.tasks.qsize()), d) tasklist = [] - min_tasks = 100 for tid in sq_data['hash']: if tid in found: continue
@@ -1011,7 +1011,8 @@ def sstate_checkhashes(sq_data, d, siginfo=False, currentcount=0, summary=True, if tasklist: nproc = min(int(d.getVar("BB_NUMBER_THREADS")), len(tasklist)) - if len(tasklist) >= min_tasks: + progress = len(tasklist) >= 100 + if progress: msg = "Checking sstate mirror object availability" bb.event.fire(bb.event.ProcessStarted(msg, len(tasklist)), d)
@@ -1024,7 +1025,7 @@ def sstate_checkhashes(sq_data, d, siginfo=False, currentcount=0, summary=True, pool.wait_completion() bb.event.disable_threadlock() - if len(tasklist) >= min_tasks: + if progress: bb.event.fire(bb.event.ProcessFinished(msg), d) inheritlist = d.getVar("INHERIT")
Signature · 1 line
2.33.1
PATCH 2/3 sstate: only search on the mirrors for the missing files
The existence of missed files on the local sstate cache is the condition to search for it on the sstate mirror.
The missed collection are populated when we search the files on the local sstate cache.
- When we have sstate mirrors we don't need to iterate all files again because we already know what's missing.
- When we found a file on the sstate mirror is because of they are missing on the local sstate mirror.
So avoid checking if the hash exists on the missed collection.
- When we can't find the file on the sstate mirror we don't need to add the hash again to the missed collection as it already exists.
--- meta/classes/sstate.bbclass | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-)
meta/classes/sstate.bbclass +2 -6
@@ -989,10 +989,8 @@ def sstate_checkhashes(sq_data, d, siginfo=False, currentcount=0, summary=True, fetcher.checkstatus() bb.debug(2, "SState: Successful fetch test for %s" % srcuri) found.add(tid) - if tid in missed: - missed.remove(tid) + missed.remove(tid) except bb.fetch2.FetchError as e: - missed.add(tid) bb.debug(2, "SState: Unsuccessful fetch test for %s (%s)" % (srcuri, e)) except Exception as e: bb.error("SState: cannot test %s: %s" % (srcuri, e))
@@ -1001,9 +999,7 @@ def sstate_checkhashes(sq_data, d, siginfo=False, currentcount=0, summary=True, bb.event.fire(bb.event.ProcessProgress(msg, len(tasklist) - thread_worker.tasks.qsize()), d) tasklist = [] - for tid in sq_data['hash']: - if tid in found: - continue + for tid in missed: spec, extrapath, tname = getpathcomponents(tid, d) sstatefile = d.expand(extrapath + generate_sstatefn(spec, gethash(tid), tname, siginfo, d)) tasklist.append((tid, sstatefile))
Signature · 1 line
2.33.1
PATCH 3/3 sstate: add a function to get the sstate file name in sstate_checkhashes
In reply to Jose Quaresma--- meta/classes/sstate.bbclass | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-)
meta/classes/sstate.bbclass +7 -9
@@ -934,12 +934,13 @@ def sstate_checkhashes(sq_data, d, siginfo=False, currentcount=0, summary=True, return spec, extrapath, tname + def getsstatefile(tid, siginfo, d): + spec, extrapath, tname = getpathcomponents(tid, d) + return extrapath + generate_sstatefn(spec, gethash(tid), tname, siginfo, d) for tid in sq_data['hash']: - spec, extrapath, tname = getpathcomponents(tid, d) - - sstatefile = d.expand("${SSTATE_DIR}/" + extrapath + generate_sstatefn(spec, gethash(tid), tname, siginfo, d)) + sstatefile = d.expand("${SSTATE_DIR}/" + getsstatefile(tid, siginfo, d)) if os.path.exists(sstatefile): found.add(tid)
@@ -1000,8 +1001,7 @@ def sstate_checkhashes(sq_data, d, siginfo=False, currentcount=0, summary=True, tasklist = [] for tid in missed: - spec, extrapath, tname = getpathcomponents(tid, d) - sstatefile = d.expand(extrapath + generate_sstatefn(spec, gethash(tid), tname, siginfo, d)) + sstatefile = d.expand(getsstatefile(tid, siginfo, d)) tasklist.append((tid, sstatefile)) if tasklist:
@@ -1028,12 +1028,10 @@ def sstate_checkhashes(sq_data, d, siginfo=False, currentcount=0, summary=True, if "toaster" in inheritlist: evdata = {'missed': [], 'found': []}; for tid in missed: - spec, extrapath, tname = getpathcomponents(tid, d) - sstatefile = d.expand(extrapath + generate_sstatefn(spec, gethash(tid), tname, False, d)) + sstatefile = d.expand(getsstatefile(tid, False, d)) evdata['missed'].append((bb.runqueue.fn_from_tid(tid), bb.runqueue.taskname_from_tid(tid), gethash(tid), sstatefile ) ) for tid in found: - spec, extrapath, tname = getpathcomponents(tid, d) - sstatefile = d.expand(extrapath + generate_sstatefn(spec, gethash(tid), tname, False, d)) + sstatefile = d.expand(getsstatefile(tid, False, d)) evdata['found'].append((bb.runqueue.fn_from_tid(tid), bb.runqueue.taskname_from_tid(tid), gethash(tid), sstatefile ) ) bb.event.fire(bb.event.MetadataEvent("MissedSstate", evdata), d)
Signature · 1 line
2.33.1