Share gcc work directories
v2 cover Share gcc work directories
Changes of V2:
1) Fix the Makefile.in to read the t-oe in ${B}/gcc/
2) Fix the confiure to read the defaults.in ${B}/gcc/
3) Now the only command that would modify ${S} is gnu-configize, but this is compatible, it would do nothing in the second run once config.sub and config.guess were installed to ${S}.
4 I have renamed ${S}/gcc/defaults.h during my testing to confirm that it
really use the one in ${B}/gcc.
5) Here is the rough improvement data:
gcc-cross, gcc-cross-initial and gcc-cross-intermediate will use the
same ${S} and "do_fetch, do_unpack and do_patch", so:a) $ time bitbake gcc-cross -cpatch -f
real 0m55.742s
so 55s * 2 = 110s will be saved.b) $ du -sh tmp/work-shared/gcc-4.6.0/
795M tmp/work-shared/gcc-4.6.0/
so 795M * 2 = 1590M (about 1.5G) disk space will be saved.
If we also build gcc-crosssdk, then all the 6 buildings will use
the same ${S}, then:
55s * 5(About 275s) will be saved.
795M * 5 = 3975M(About 3.9G) disk space will be saved.The following changes since commit 7aa7673459376aff911cef820c9417c998d1aa96:
meta-yocto/linux-yocto: update to match the renamed linux-yocto recipes (2011-06-14 09:21:32 -0700)
are available in the git repository at:
git://git.pokylinux.org/poky-contrib robert/share_gcc
http://git.pokylinux.org/cgit.cgi/poky-contrib/log/?h=robert/share_gcc
Robert Yang (4):
bitbake: share source directory
Share gcc work directories
gcc-4.5.1: share work directories
gcc-4.6.0: share work directories
bitbake/lib/bb/build.py | 4 +- bitbake/lib/bb/cache.py | 3 + bitbake/lib/bb/runqueue.py | 10 +++ meta/recipes-devtools/gcc/gcc-4.5.1.inc | 1 + .../gcc-4.5.1/use-defaults.h-and-t-oe-in-B.patch | 57 ++++++++++++++ meta/recipes-devtools/gcc/gcc-4.6.0.inc | 1 + .../gcc-4.6.0/use-defaults.h-and-t-oe-in-B.patch | 80 ++++++++++++++++++++ meta/recipes-devtools/gcc/gcc-common.inc | 32 +++++++- meta/recipes-devtools/gcc/gcc-configure-common.inc | 34 ++++----- 9 files changed, 199 insertions(+), 23 deletions(-) create mode 100644 meta/recipes-devtools/gcc/gcc-4.5.1/use-defaults.h-and-t-oe-in-B.patch create mode 100644 meta/recipes-devtools/gcc/gcc-4.6.0/use-defaults.h-and-t-oe-in-B.patch
PATCH v2 1/4 bitbake: share source directory
This patch is derived from Richard, it is a quick proof of concept to show how source code could be shared between recipes which use ${B} to have a separate build directory compared to source directory ${S}.
--- bitbake/lib/bb/build.py | 4 ++-- bitbake/lib/bb/cache.py | 3 +++ bitbake/lib/bb/runqueue.py | 10 ++++++++++ 3 files changed, 15 insertions(+), 2 deletions(-)
bitbake/lib/bb/build.py +2 -2
@@ -383,10 +383,10 @@ def stamp_internal(taskname, d, file_name): taskflagname = taskname.replace("_setscene", "") if file_name: - stamp = d.stamp[file_name] + stamp = d.stamp_base[file_name].get(taskflagname) or d.stamp[file_name] extrainfo = d.stamp_extrainfo[file_name].get(taskflagname) or "" else: - stamp = d.getVar('STAMP', True) + stamp = d.getVarFlag(taskflagname, 'stamp-base', True) or d.getVar('STAMP', True) file_name = d.getVar('BB_FILENAME', True) extrainfo = d.getVarFlag(taskflagname, 'stamp-extra-info', True) or ""
bitbake/lib/bb/cache.py +3 -0
@@ -124,6 +124,7 @@ class CoreRecipeInfo(RecipeInfoCommon): self.broken = self.getvar('BROKEN', metadata) self.not_world = self.getvar('EXCLUDE_FROM_WORLD', metadata) self.stamp = self.getvar('STAMP', metadata) + self.stamp_base = self.flaglist('stamp-base', self.tasks, metadata) self.stamp_extrainfo = self.flaglist('stamp-extra-info', self.tasks, metadata) self.packages_dynamic = self.listvar('PACKAGES_DYNAMIC', metadata) self.depends = self.depvar('DEPENDS', metadata)
@@ -151,6 +152,7 @@ class CoreRecipeInfo(RecipeInfoCommon): cachedata.pkg_dp = {} cachedata.stamp = {} + cachedata.stamp_base = {} cachedata.stamp_extrainfo = {} cachedata.fn_provides = {} cachedata.pn_provides = defaultdict(list)
@@ -183,6 +185,7 @@ class CoreRecipeInfo(RecipeInfoCommon): cachedata.pkg_pepvpr[fn] = (self.pe, self.pv, self.pr) cachedata.pkg_dp[fn] = self.defaultpref cachedata.stamp[fn] = self.stamp + cachedata.stamp_base[fn] = self.stamp_base cachedata.stamp_extrainfo[fn] = self.stamp_extrainfo provides = [self.pn]
bitbake/lib/bb/runqueue.py +10 -0
@@ -105,6 +105,11 @@ class RunQueueScheduler(object): if self.rq.runq_running[taskid] == 1: continue if self.rq.runq_buildable[taskid] == 1: + fn = self.rqdata.taskData.fn_index[self.rqdata.runq_fnid[taskid]] + taskname = self.rqdata.runq_task[taskid] + stamp = bb.build.stampfile(taskname, self.rqdata.dataCache, fn) + if stamp in self.rq.build_stamps.values(): + continue return taskid def next(self):
@@ -1009,6 +1014,7 @@ class RunQueueExecute: self.runq_complete = [] self.build_pids = {} self.build_pipes = {} + self.build_stamps = {} self.failed_fnids = [] def runqueue_process_waitpid(self):
@@ -1023,6 +1029,9 @@ class RunQueueExecute: del self.build_pids[result[0]] self.build_pipes[result[0]].close() del self.build_pipes[result[0]] + # self.build_stamps[result[0]] may not exist when use shared work directory. + if result[0] in self.build_stamps.keys(): + del self.build_stamps[result[0]] if result[1] != 0: self.task_fail(task, result[1]>>8) else:
@@ -1319,6 +1328,7 @@ class RunQueueExecuteTasks(RunQueueExecute): self.build_pids[pid] = task self.build_pipes[pid] = runQueuePipe(pipein, pipeout, self.cfgData) + self.build_stamps[pid] = bb.build.stampfile(taskname, self.rqdata.dataCache, fn) self.runq_running[task] = 1 self.stats.taskActive() if self.stats.active < self.number_tasks:
Signature · 1 line
1.7.1
PATCH v2 2/4 Share gcc work directories
In reply to Robert YangThis patched is derived from Richard, make gcc use the shared source directory during the different building:
1) Make gcc-cross, gcc-cross-initial, gcc-cross-intermediate and gcc-runtime share the same source directory.
2) The source directory is ${TMPDIR}/work-shared/gcc-${PV}, for example:
tmp/work-shared/gcc-4.6.0
3) Fix do_clean to clean the shared source directory and stamps
4) gcc uses sed and creates config files against ${S} which means the directory should not be shared. Change the way to make it work:
* The configure option --with-build-sysroot=${STAGING_DIR_TARGET} can replace the sed command.
* Move t-oe from ${T} to ${B}/gcc, so that the patched Makefile.in can read it easily, please see the commit for gcc-4.5.1 and gcc-4.6.0.
* Use the defaults.h in ${B}/gcc instead of ${S}/gcc, and the patched configure.ac(configure) can read it correctly, please see the commit for gcc-4.5.1 and gcc-4.6.0.
--- meta/recipes-devtools/gcc/gcc-common.inc | 32 +++++++++++++++++- meta/recipes-devtools/gcc/gcc-configure-common.inc | 34 +++++++++----------- 2 files changed, 45 insertions(+), 21 deletions(-)
meta/recipes-devtools/gcc/gcc-common.inc +30 -2
@@ -37,10 +37,38 @@ ${GNU_MIRROR}/gcc/ http://gcc.get-software.com/releases/ \n \ # gcclibdir = "${libdir}/gcc" BINV = "${PV}" -S = "${WORKDIR}/gcc-${PV}" -B = "${S}/build.${HOST_SYS}.${TARGET_SYS}" +#S = "${WORKDIR}/gcc-${PV}" +S = "${TMPDIR}/work-shared/gcc-${PV}/gcc-${PV}" +B = "${WORKDIR}/gcc-${PV}/build.${HOST_SYS}.${TARGET_SYS}" + +# SS means Shared Stamps directory +SS = "${TMPDIR}/stamps/work-shared/gcc-${PV}" +do_fetch[stamp-base] = "${SS}" +do_unpack[stamp-base] = "${SS}" +do_patch[stamp-base] = "${SS}" + +# SW means Shared Work directory +SW = "${TMPDIR}/work-shared/gcc-${PV}" +WORKDIR_task-unpack = "${SW}" +WORKDIR_task-patch = "${SW}" target_includedir ?= "${includedir}" target_libdir ?= "${libdir}" target_base_libdir ?= "${base_libdir}" target_prefix ?= "${prefix}" + +CLEANFUNCS += "workshared_clean" +# The do_clean should be exclusive since share ${S} +do_clean[lockfiles] = "${TMPDIR}/stamps/work-shared/gcc-${PV}.clean.lock" + +python workshared_clean () { + """clear the source directory""" + dir = bb.data.expand("${SW}", d) + bb.note("Removing " + dir) + oe.path.remove(dir) + + """clear the the stamps in work-shared""" + dir = "%s.*" % bb.data.expand(d.getVarFlag('do_fetch', 'stamp-base', True), d) + bb.note("Removing " + dir) + oe.path.remove(dir) +}
meta/recipes-devtools/gcc/gcc-configure-common.inc +15 -19
@@ -29,6 +29,7 @@ GCCMULTILIB = "--disable-multilib" EXTRA_OECONF = "${@['--enable-clocale=generic', ''][bb.data.getVar('USE_NLS', d, 1) != 'no']} \ --with-gnu-ld \ + --with-build-sysroot=${STAGING_DIR_TARGET} \ --enable-shared \ --enable-languages=${LANGUAGES} \ --enable-threads=posix \
@@ -88,25 +89,20 @@ do_configure () { (cd ${S} && gnu-configize) || die "failure running gnu-configize" # teach gcc to find correct target includedir when checking libc ssp support - sed -i 's:^\([ ]*\)glibc_header_dir=\"${with_build_sysroot}/usr/include\":\1glibc_header_dir=\"${with_build_sysroot}${SYSTEMHEADERS}\":g' ${S}/gcc/configure.ac - sed -i 's:^\([ ]*\)glibc_header_dir=\"${with_build_sysroot}/usr/include\":\1glibc_header_dir=\"${with_build_sysroot}${SYSTEMHEADERS}\":g' ${S}/gcc/configure - - # splice our idea of where the headers live into gcc's world - echo "NATIVE_SYSTEM_HEADER_DIR = ${SYSTEMHEADERS}" > ${T}/t-oe - sed 's%^tmake_file=.*$%& ${T}/t-oe%' < ${S}/gcc/Makefile.in >${S}/gcc/Makefile.in.new - mv ${S}/gcc/Makefile.in.new ${S}/gcc/Makefile.in - cat ${S}/gcc/defaults.h | grep -v "\#endif.*GCC_DEFAULTS_H" > ${S}/gcc/defaults.h.new - echo "#ifndef STANDARD_INCLUDE_DIR" >> ${S}/gcc/defaults.h.new - echo "#define STANDARD_INCLUDE_DIR \"${SYSTEMHEADERS}\"" >> ${S}/gcc/defaults.h.new - echo "#endif" >> ${S}/gcc/defaults.h.new - echo "#ifndef STANDARD_STARTFILE_PREFIX_1" >> ${S}/gcc/defaults.h.new - echo "#define STANDARD_STARTFILE_PREFIX_1 \"${SYSTEMLIBS}\"" >> ${S}/gcc/defaults.h.new - echo "#endif" >> ${S}/gcc/defaults.h.new - echo "#ifndef STANDARD_STARTFILE_PREFIX_2" >> ${S}/gcc/defaults.h.new - echo "#define STANDARD_STARTFILE_PREFIX_2 \"${SYSTEMLIBS1}\"" >> ${S}/gcc/defaults.h.new - echo "#endif" >> ${S}/gcc/defaults.h.new - echo "#endif /* ! GCC_DEFAULTS_H */" >> ${S}/gcc/defaults.h.new - mv ${S}/gcc/defaults.h.new ${S}/gcc/defaults.h + mkdir -p ${B}/gcc + echo "NATIVE_SYSTEM_HEADER_DIR = ${SYSTEMHEADERS}" > ${B}/gcc/t-oe + cat ${S}/gcc/defaults.h | grep -v "\#endif.*GCC_DEFAULTS_H" > ${B}/gcc/defaults.h.new + echo "#ifndef STANDARD_INCLUDE_DIR" >> ${B}/gcc/defaults.h.new + echo "#define STANDARD_INCLUDE_DIR \"${SYSTEMHEADERS}\"" >> ${B}/gcc/defaults.h.new + echo "#endif" >> ${B}/gcc/defaults.h.new + echo "#ifndef STANDARD_STARTFILE_PREFIX_1" >> ${B}/gcc/defaults.h.new + echo "#define STANDARD_STARTFILE_PREFIX_1 \"${SYSTEMLIBS}\"" >> ${B}/gcc/defaults.h.new + echo "#endif" >> ${B}/gcc/defaults.h.new + echo "#ifndef STANDARD_STARTFILE_PREFIX_2" >> ${B}/gcc/defaults.h.new + echo "#define STANDARD_STARTFILE_PREFIX_2 \"${SYSTEMLIBS1}\"" >> ${B}/gcc/defaults.h.new + echo "#endif" >> ${B}/gcc/defaults.h.new + echo "#endif /* ! GCC_DEFAULTS_H */" >> ${B}/gcc/defaults.h.new + mv ${B}/gcc/defaults.h.new ${B}/gcc/defaults.h oe_runconf }
Signature · 1 line
1.7.1
Hi Robert,
This is looking good, just one further problem:
Quoted 2 lines · Robert Yang
* The configure option --with-build-sysroot=${STAGING_DIR_TARGET} can replace the sed command.
Sadly I'm not sure it can.
meta/recipes-devtools/gcc/gcc-configure-common.inc do_configure () { · line 92
Quoted 13 lines
EXTRA_OECONF = "${@['--enable-clocale=generic', ''][bb.data.getVar('USE_NLS', d, 1) != 'no']} \ --with-gnu-ld \meta/recipes-devtools/gcc/gcc-configure-common.inc +1
-2+ --with-build-sysroot=${STAGING_DIR_TARGET} \ --enable-shared \ --enable-languages=${LANGUAGES} \ --enable-threads=posix \@@ -88,25 +89,20 @@ do_configure () { (cd ${S} && gnu-configize) || die "failure running gnu-configize" # teach gcc to find correct target includedir when checking libc ssp support - sed -i 's:^\([ ]*\)glibc_header_dir=\"${with_build_sysroot}/usr/include\":\1glibc_header_dir=\"${with_build_sysroot}${SYSTEMHEADERS}\":g' ${S}/gcc/configure.ac - sed -i 's:^\([ ]*\)glibc_header_dir=\"${with_build_sysroot}/usr/include\":\1glibc_header_dir=\"${with_build_sysroot}${SYSTEMHEADERS}\":g' ${S}/gcc/configure
Here, a hardcoded "/usr/include" in the gcc makefiles is being replaced by ${SYSTEMHEADERS} which could be /some/other/path/include. These two are therefore not equivalent. Did you try a "bitbake meta-toolchain" with this patch series?
I suspect configure needs to use ${includedir} to build this path instead of hardcoding it.
Cheers,
Richard
Quoted 20 lines · Richard Purdie
Hi Robert,
This is looking good, just one further problem:
Quoted 2 lines · Robert Yang
* The configure option --with-build-sysroot=${STAGING_DIR_TARGET} can replace the sed command.
Sadly I'm not sure it can.
Quoted 10 lines
EXTRA_OECONF = "${@['--enable-clocale=generic', ''][bb.data.getVar('USE_NLS', d, 1) != 'no']} \ --with-gnu-ld \meta/recipes-devtools/gcc/gcc-configure-common.inc +1
-0+ --with-build-sysroot=${STAGING_DIR_TARGET} \ --enable-shared \ --enable-languages=${LANGUAGES} \ --enable-threads=posix \@@ -88,25 +89,20 @@ do_configure () { (cd ${S}&& gnu-configize) || die "failure running gnu-configize"
Quoted 9 lines
meta/recipes-devtools/gcc/gcc-configure-common.inc do_configure () { · line 92
Quoted 3 lines
meta/recipes-devtools/gcc/gcc-configure-common.inc +0
-2# teach gcc to find correct target includedir when checking libc ssp support - sed -i 's:^\([ ]*\)glibc_header_dir=\"${with_build_sysroot}/usr/include\":\1glibc_header_dir=\"${with_build_sysroot}${SYSTEMHEADERS}\":g' ${S}/gcc/configure.ac - sed -i 's:^\([ ]*\)glibc_header_dir=\"${with_build_sysroot}/usr/include\":\1glibc_header_dir=\"${with_build_sysroot}${SYSTEMHEADERS}\":g' ${S}/gcc/configureHere, a hardcoded "
/usr/include" in the gcc makefiles is being replaced by ${SYSTEMHEADERS} which could be /some/other/path/include. These two are therefore not equivalent. Did you try a "bitbake meta-toolchain" with this patch series?
I'm sorry, because the "bitbake meta-toolchain" didn't work recently(without these patches), I forgot to build it(just built gcc-cross and gcc-crosssdk), I will go on working on it and send the V3 later.
Thanks
// Robert
Quoted 15 lines
I suspect configure needs to use ${includedir} to build this path instead of hardcoding it.
Cheers,
Richard
_______________________________________________
Openembedded-core mailing list
[email protected]
http://lists.linuxtogo.org/cgi-bin/mailman/listinfo/openembedded-core
PATCH v2 3/4 gcc-4.5.1: share work directories
In reply to Robert YangFix configure and Makefile to read the defaults.h and t-oe from ${B}, so that the ${S} can be shared.
--- meta/recipes-devtools/gcc/gcc-4.5.1.inc | 1 + .../gcc-4.5.1/use-defaults.h-and-t-oe-in-B.patch | 57 ++++++++++++++++++++ 2 files changed, 58 insertions(+), 0 deletions(-) create mode 100644 meta/recipes-devtools/gcc/gcc-4.5.1/use-defaults.h-and-t-oe-in-B.patch
meta/recipes-devtools/gcc/gcc-4.5.1.inc +1 -0
@@ -57,6 +57,7 @@ SRC_URI = "${GNU_MIRROR}/gcc/gcc-${PV}/gcc-${PV}.tar.bz2 \ file://gcc-poison-parameters.patch \ file://GPLUSPLUS_INCLUDE_DIR_with_sysroot.patch \ file://COLLECT_GCC_OPTIONS.patch \ + file://use-defaults.h-and-t-oe-in-B.patch \ " SRC_URI_append_sh3 = " file://sh3-installfix-fixheaders.patch;patch=1 "
meta/recipes-devtools/gcc/gcc-4.5.1/use-defaults.h-and-t-oe-in-B.patch new file +57 -0
57 lines not on this page · raw message
Signature · 1 line
1.7.1
PATCH v2 4/4 gcc-4.6.0: share work directories
In reply to Robert YangFix configure and Makefile to read the defaults.h and t-oe from ${B}, so that the ${S} can be shared.
--- meta/recipes-devtools/gcc/gcc-4.6.0.inc | 1 + .../gcc-4.6.0/use-defaults.h-and-t-oe-in-B.patch | 80 ++++++++++++++++++++ 2 files changed, 81 insertions(+), 0 deletions(-) create mode 100644 meta/recipes-devtools/gcc/gcc-4.6.0/use-defaults.h-and-t-oe-in-B.patch
meta/recipes-devtools/gcc/gcc-4.6.0.inc +1 -0
@@ -54,6 +54,7 @@ SRC_URI = "${GNU_MIRROR}/gcc/gcc-${PV}/gcc-${PV}.tar.bz2 \ file://GPLUSPLUS_INCLUDE_DIR_with_sysroot.patch \ file://COLLECT_GCC_OPTIONS.patch \ file://volatile_access_backport.patch \ + file://use-defaults.h-and-t-oe-in-B.patch \ " SRC_URI_append_sh3 = " file://sh3-installfix-fixheaders.patch;patch=1 "
meta/recipes-devtools/gcc/gcc-4.6.0/use-defaults.h-and-t-oe-in-B.patch new file +80 -0
80 lines not on this page · raw message
Signature · 1 line
1.7.1