sstate.bbclass: Optimise sstate_hardcode_path
PATCH 1/1 sstate.bbclass: Optimise sstate_hardcode_path
The sstate_hardcode_path() function triggered large numbers of exec() calls when processing packages with large numbers of file relocations (e.g. perl). This patch optimises those calls into longer single commands which make the code significantly more efficient.
This reduced the do_package time for perl by 2 minutes (from 4.75 minutes) for me.
---
meta/classes/sstate.bbclass +16 -11
@@ -317,19 +317,24 @@ def sstate_hardcode_path(d): staging_host = d.getVar('STAGING_DIR_HOST', True) sstate_builddir = d.getVar('SSTATE_BUILDDIR', True) - for i in file_list.split('\n'): - if not i: - continue - if bb.data.inherits_class('native', d) or bb.data.inherits_class('nativesdk', d) or bb.data.inherits_class('crosssdk', d) or bb.data.inherits_class('cross-canadian', d): - cmd = "sed -i -e s:%s:FIXMESTAGINGDIR:g %s" % (staging, i) - elif bb.data.inherits_class('cross', d): - cmd = "sed -i -e s:%s:FIXMESTAGINGDIRTARGET:g %s \ - sed -i -e s:%s:FIXMESTAGINGDIR:g %s" % (staging_target, i, staging, i) - else: - cmd = "sed -i -e s:%s:FIXMESTAGINGDIRHOST:g %s" % (staging_host, i) + files = " ".join(file_list.split('\n')) + if bb.data.inherits_class('native', d) or bb.data.inherits_class('nativesdk', d) or bb.data.inherits_class('crosssdk', d) or bb.data.inherits_class('cross-canadian', d): + cmd = "sed -i -e s:%s:FIXMESTAGINGDIR:g %s" % (staging, files) + elif bb.data.inherits_class('cross', d): + cmd = "sed -i -e s:%s:FIXMESTAGINGDIRTARGET:g %s \ + sed -i -e s:%s:FIXMESTAGINGDIR:g %s" % (staging_target, files, staging, files) + else: + cmd = "sed -i -e s:%s:FIXMESTAGINGDIRHOST:g %s" % (staging_host, files) + + if files: os.system(cmd) - os.system("echo %s | sed -e 's:%s::' >> %sfixmepath" % (i, sstate_builddir, sstate_builddir)) + fix = open("%sfixmepath" % (sstate_builddir), "w") + fixme = [] + for f in file_list.split('\n'): + fixme.append(f.replace(sstate_builddir, "")) + fix.write("\n".join(fixme)) + fix.close() p.close() def sstate_package(ss, d):
Quoted 7 lines · Richard Purdie
The
sstate_hardcode_path()function triggered large numbers ofexec()calls when processing packages with large numbers of file relocations (e.g. perl). This patch optimises those calls into longer single commands which make the code significantly more efficient.This reduced the
do_packagetime for perl by 2 minutes (from 4.75 minutes) for me.
Very nice speed up!
Quoted 1 line
Signed-off-by Richard Purdie<[email protected]>
A minor api/style nit below but:
Quoted 12 lines
---
meta/classes/sstate.bbclass +0
-3@@ -317,19 +317,24 @@ def sstate_hardcode_path(d): staging_host = d.getVar('STAGING_DIR_HOST', True) sstate_builddir = d.getVar('SSTATE_BUILDDIR', True) - for i in file_list.split('\n'): - if not i: - continue
meta/classes/sstate.bbclass def sstate_hardcode_path(d): · line 322
Quoted 10 lines
meta/classes/sstate.bbclass +2
-7- if bb.data.inherits_class('native', d) or bb.data.inherits_class('nativesdk', d) or bb.data.inherits_class('crosssdk', d) or bb.data.inherits_class('cross-canadian', d): - cmd = "sed -i -e s:%s:FIXMESTAGINGDIR:g %s" % (staging, i) - elif bb.data.inherits_class('cross', d): - cmd = "sed -i -e s:%s:FIXMESTAGINGDIRTARGET:g %s \ - sed -i -e s:%s:FIXMESTAGINGDIR:g %s" % (staging_target, i, staging, i) - else: - cmd = "sed -i -e s:%s:FIXMESTAGINGDIRHOST:g %s" % (staging_host, i) + files = " ".join(file_list.split('\n')) + if bb.data.inherits_class('native', d) or bb.data.inherits_class('nativesdk', d) or bb.data.inherits_class('crosssdk', d) or bb.data.inherits_class('cross-canadian', d):
I think this can be replaced by a single call to the inherits method of the oe.utils module:
oe.utils.inherits(d, 'native', 'nativesdk', 'crosssdk', 'cross-canadian')
which makes this a little more concise.
Quoted 26 lines
meta/classes/sstate.bbclass +14
-1+ cmd = "sed -i -e s:%s:FIXMESTAGINGDIR:g %s" % (staging, files) + elif bb.data.inherits_class('cross', d): + cmd = "sed -i -e s:%s:FIXMESTAGINGDIRTARGET:g %s \ + sed -i -e s:%s:FIXMESTAGINGDIR:g %s" % (staging_target, files, staging, files) + else: + cmd = "sed -i -e s:%s:FIXMESTAGINGDIRHOST:g %s" % (staging_host, files) + + if files: os.system(cmd) - os.system("echo %s | sed -e 's:%s::'>> %sfixmepath" % (i, sstate_builddir, sstate_builddir)) + fix = open("%sfixmepath" % (sstate_builddir), "w") + fixme = [] + for f in file_list.split('\n'): + fixme.append(f.replace(sstate_builddir, "")) + fix.write("\n".join(fixme)) + fix.close() p.close() def sstate_package(ss, d):_______________________________________________
Openembedded-core mailing list
[email protected]
http://lists.linuxtogo.org/cgi-bin/mailman/listinfo/openembedded-core
Signature · 3 lines
Joshua Lock
Yocto Project "Johannes factotum"
Intel Open Source Technology Centre