Improve checkstatus using connection cache
v2 cover Improve checkstatus using connection cache
This patch depends on changes did in bitbake [1] and oe/utils.py [2], don't merge before it.
For complete review exist an integration branch at,
http://git.yoctoproject.org/cgit/cgit.cgi/poky-contrib/log/?h=alimon/fetch2-wget-checkstatus
[1] http://lists.openembedded.org/pipermail/bitbake-devel/2015-June/005912.html
[2] http://lists.openembedded.org/pipermail/openembedded-core/2015-June/106430.html
Aníbal Limón (1):
classes/sstate.bbclass: Improve checkstatus using connection cache.
meta/classes/sstate.bbclass | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-)
Signature · 1 line
1.9.1
PATCH v2 1/1 classes/sstate.bbclass: Improve checkstatus using connection cache.
Use FetcherConnectionCache to improve times when do checkstatus over sstate resources.
[YOCTO #7796]
Signed-off-by Aníbal Limón <[email protected]>
--- meta/classes/sstate.bbclass | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-)
meta/classes/sstate.bbclass +12 -2
@@ -739,6 +739,13 @@ def sstate_checkhashes(sq_fn, sq_task, sq_hash, sq_hashfn, d, siginfo=False): if localdata.getVar('BB_NO_NETWORK', True) == "1" and localdata.getVar('SSTATE_MIRROR_ALLOW_NETWORK', True) == "1": localdata.delVar('BB_NO_NETWORK') + from bb.fetch2 import FetchConnectionCache + def checkstatus_init(thread_worker): + thread_worker.connection_cache = FetchConnectionCache() + + def checkstatus_end(thread_worker): + thread_worker.connection_cache.close_connections() + def checkstatus(thread_worker, arg): (task, sstatefile) = arg
@@ -748,7 +755,8 @@ def sstate_checkhashes(sq_fn, sq_task, sq_hash, sq_hashfn, d, siginfo=False): bb.debug(2, "SState: Attempting to fetch %s" % srcuri) try: - fetcher = bb.fetch2.Fetch(srcuri.split(), localdata2) + fetcher = bb.fetch2.Fetch(srcuri.split(), localdata2, + connection_cache=thread_worker.connection_cache) fetcher.checkstatus() bb.debug(2, "SState: Successful fetch test for %s" % srcuri) ret.append(task)
@@ -771,7 +779,9 @@ def sstate_checkhashes(sq_fn, sq_task, sq_hash, sq_hashfn, d, siginfo=False): bb.note("Checking sstate mirror object availability (for %s objects)" % len(tasklist)) import multiprocessing nproc = min(multiprocessing.cpu_count(), len(tasklist)) - pool = oe.utils.ThreadedPool(nproc, len(tasklist)) + + pool = oe.utils.ThreadedPool(nproc, len(tasklist), + worker_init=checkstatus_init, worker_end=checkstatus_end) for t in tasklist: pool.add_task(checkstatus, t) pool.start()
Signature · 1 line
1.9.1