Skip to contentSkip to conversation

bitbake: share source directory

PATCH Superseded Robert Yang 2 participants 3 messages Jun 11, 2011 1 message not in the archive

v1 of 4 Superseded Latest is v4 →

lore
Share & tools
lore

A message that is not in the archive (sent off-list or not yet received): [email protected]

#1 lore raw

PATCH 1/2 bitbake: share source directory

This patch is derived from Richard, it is a quick proof of concept to show how source code could be shared between recipes which use ${B} to have a separate build directory compared to source directory ${S}.

---
 bitbake/lib/bb/build.py    |    4 ++--
 bitbake/lib/bb/cache.py    |    3 +++
 bitbake/lib/bb/runqueue.py |   10 ++++++++++
 3 files changed, 15 insertions(+), 2 deletions(-)
bitbake/lib/bb/build.py +2 -2
index 7947906..052a3bf 100644
@@ -383,10 +383,10 @@ def stamp_internal(taskname, d, file_name):
         taskflagname = taskname.replace("_setscene", "")
 
     if file_name:
-        stamp = d.stamp[file_name]
+        stamp = d.stamp_base[file_name].get(taskflagname) or d.stamp[file_name]
         extrainfo = d.stamp_extrainfo[file_name].get(taskflagname) or ""
     else:
-        stamp = d.getVar('STAMP', True)
+        stamp = d.getVarFlag(taskflagname, 'stamp-base', True) or d.getVar('STAMP', True)
         file_name = d.getVar('BB_FILENAME', True)
         extrainfo = d.getVarFlag(taskflagname, 'stamp-extra-info', True) or ""
 
bitbake/lib/bb/cache.py +3 -0
index d4a16ad..10ef235 100644
@@ -124,6 +124,7 @@ class CoreRecipeInfo(RecipeInfoCommon):
         self.broken = self.getvar('BROKEN', metadata)
         self.not_world = self.getvar('EXCLUDE_FROM_WORLD', metadata)
         self.stamp = self.getvar('STAMP', metadata)
+        self.stamp_base = self.flaglist('stamp-base', self.tasks, metadata)
         self.stamp_extrainfo = self.flaglist('stamp-extra-info', self.tasks, metadata)
         self.packages_dynamic = self.listvar('PACKAGES_DYNAMIC', metadata)
         self.depends          = self.depvar('DEPENDS', metadata)
@@ -151,6 +152,7 @@ class CoreRecipeInfo(RecipeInfoCommon):
         cachedata.pkg_dp = {}
 
         cachedata.stamp = {}
+        cachedata.stamp_base = {}
         cachedata.stamp_extrainfo = {}
         cachedata.fn_provides = {}
         cachedata.pn_provides = defaultdict(list)
@@ -183,6 +185,7 @@ class CoreRecipeInfo(RecipeInfoCommon):
         cachedata.pkg_pepvpr[fn] = (self.pe, self.pv, self.pr)
         cachedata.pkg_dp[fn] = self.defaultpref
         cachedata.stamp[fn] = self.stamp
+        cachedata.stamp_base[fn] = self.stamp_base
         cachedata.stamp_extrainfo[fn] = self.stamp_extrainfo
 
         provides = [self.pn]
bitbake/lib/bb/runqueue.py +10 -0
index af21eae..5ec69af 100644
@@ -105,6 +105,11 @@ class RunQueueScheduler(object):
             if self.rq.runq_running[taskid] == 1:
                 continue
             if self.rq.runq_buildable[taskid] == 1:
+                fn = self.rqdata.taskData.fn_index[self.rqdata.runq_fnid[taskid]]
+                taskname = self.rqdata.runq_task[taskid]
+                stamp = bb.build.stampfile(taskname, self.rqdata.dataCache, fn)
+                if stamp in self.rq.build_stamps.values():
+                    continue
                 return taskid
 
     def next(self):
@@ -1009,6 +1014,7 @@ class RunQueueExecute:
         self.runq_complete = []
         self.build_pids = {}
         self.build_pipes = {}
+        self.build_stamps = {}
         self.failed_fnids = []
 
     def runqueue_process_waitpid(self):
@@ -1023,6 +1029,9 @@ class RunQueueExecute:
         del self.build_pids[result[0]]
         self.build_pipes[result[0]].close()
         del self.build_pipes[result[0]]
+        # self.build_stamps[result[0]] may not exist when use shared work directory.
+        if result[0] in self.build_stamps.keys():
+            del self.build_stamps[result[0]]
         if result[1] != 0:
             self.task_fail(task, result[1]>>8)
         else:
@@ -1319,6 +1328,7 @@ class RunQueueExecuteTasks(RunQueueExecute):
 
             self.build_pids[pid] = task
             self.build_pipes[pid] = runQueuePipe(pipein, pipeout, self.cfgData)
+            self.build_stamps[pid] = bb.build.stampfile(taskname, self.rqdata.dataCache, fn)
             self.runq_running[task] = 1
             self.stats.taskActive()
             if self.stats.active < self.number_tasks:
Signature · 1 line
1.7.1
#2 lore raw

PATCH 2/2 Share gcc work directories

In reply to a message not in the archive

This patched is derived from Richard, make gcc use the shared source directory during the different building:

1) Make gcc-cross, gcc-cross-initial, gcc-cross-intermediate and gcc-runtime share the same source directory.

2) The source directory is ${TMPDIR}/work-shared/gcc-${PV}, for example:
tmp/work-shared/gcc-4.6.0

3) gcc uses sed and creates config files against ${S} which means the directory should not be shared. Change the way to make it work.

4) Fix do_clean to clean the shared source directory and stamps

---
 meta/recipes-devtools/gcc/gcc-common.inc           |   32 ++++++++++++++++++-
 meta/recipes-devtools/gcc/gcc-configure-common.inc |   10 +++++-
 2 files changed, 39 insertions(+), 3 deletions(-)
meta/recipes-devtools/gcc/gcc-common.inc +30 -2
index a3fa234..7bf036c 100644
@@ -37,10 +37,38 @@ ${GNU_MIRROR}/gcc/	http://gcc.get-software.com/releases/ \n \
 #
 gcclibdir = "${libdir}/gcc"
 BINV = "${PV}"
-S = "${WORKDIR}/gcc-${PV}"
-B = "${S}/build.${HOST_SYS}.${TARGET_SYS}"
+#S = "${WORKDIR}/gcc-${PV}"
+S = "${TMPDIR}/work-shared/gcc-${PV}/gcc-${PV}"
+B = "${WORKDIR}/gcc-${PV}/build.${HOST_SYS}.${TARGET_SYS}"
+
+# SS means Shared Stamps directory
+SS = "${TMPDIR}/stamps/work-shared/gcc-${PV}"
+do_fetch[stamp-base] = "${SS}"
+do_unpack[stamp-base] = "${SS}"
+do_patch[stamp-base] = "${SS}"
+
+# SW means Shared Work directory
+SW = "${TMPDIR}/work-shared/gcc-${PV}"
+WORKDIR_task-unpack = "${SW}"
+WORKDIR_task-patch = "${SW}"
 
 target_includedir ?= "${includedir}"
 target_libdir ?= "${libdir}"
 target_base_libdir ?= "${base_libdir}"
 target_prefix ?= "${prefix}"
+
+CLEANFUNCS += "workshared_clean"
+# The do_clean should be exclusive since share ${S}
+do_clean[lockfiles] = "${TMPDIR}/stamps/work-shared/gcc-${PV}.clean.lock"
+
+python workshared_clean () {
+	"""clear the source directory"""
+	dir = bb.data.expand("${SW}", d)
+	bb.note("Removing " + dir)
+	oe.path.remove(dir)
+
+	"""clear the the stamps in work-shared"""
+	dir = "%s.*" % bb.data.expand(d.getVarFlag('do_fetch', 'stamp-base', True), d)
+	bb.note("Removing " + dir)
+	oe.path.remove(dir)
+}
meta/recipes-devtools/gcc/gcc-configure-common.inc +9 -1
index f7b5836..c32242d 100644
@@ -87,7 +87,15 @@ do_configure () {
 	export ARCH_FLAGS_FOR_TARGET="${ARCH_FLAGS_FOR_TARGET}"
 	(cd ${S} && gnu-configize) || die "failure running gnu-configize"
 	
-	# teach gcc to find correct target includedir when checking libc ssp support
+	# teach gcc to find correct target includedir when checking libc ssp support,
+	# save the files before hack them, so that we can always hack the fresh one.
+	for i in configure.ac Makefile.in defaults.h configure; do
+		[ ! -f ${S}/gcc/$i.orig ] && cp ${S}/gcc/$i ${S}/gcc/$i.orig
+		cp ${S}/gcc/$i.orig ${S}/gcc/$i
+	done
+	# Make sure that ${S}/configure is newer than ${S}/gcc/configure to prevent
+	# auto run autoconf.
+	touch ${S}/configure
 	sed -i 's:^\([ 	]*\)glibc_header_dir=\"${with_build_sysroot}/usr/include\":\1glibc_header_dir=\"${with_build_sysroot}${SYSTEMHEADERS}\":g' ${S}/gcc/configure.ac
 	sed -i 's:^\([ 	]*\)glibc_header_dir=\"${with_build_sysroot}/usr/include\":\1glibc_header_dir=\"${with_build_sysroot}${SYSTEMHEADERS}\":g' ${S}/gcc/configure
 
Signature · 1 line
1.7.1
#3 lore raw

Re: [PATCH 2/2] Share gcc work directories

Quoted 76 lines · Robert Yang

This patched is derived from Richard, make gcc use the shared source directory during the different building:

1) Make gcc-cross, gcc-cross-initial, gcc-cross-intermediate and gcc-runtime share the same source directory.

2) The source directory is ${TMPDIR}/work-shared/gcc-${PV}, for example:

   tmp/work-shared/gcc-4.6.0

3) gcc uses sed and creates config files against ${S} which means the directory should not be shared. Change the way to make it work.

4) Fix do_clean to clean the shared source directory and stamps

---
 meta/recipes-devtools/gcc/gcc-common.inc           |   32 ++++++++++++++++++-
 meta/recipes-devtools/gcc/gcc-configure-common.inc |   10 +++++-
 2 files changed, 39 insertions(+), 3 deletions(-)
meta/recipes-devtools/gcc/gcc-common.inc +30 -2
index a3fa234..7bf036c 100644
@@ -37,10 +37,38 @@ ${GNU_MIRROR}/gcc/	http://gcc.get-software.com/releases/ \n \
 #
 gcclibdir = "${libdir}/gcc"
 BINV = "${PV}"
-S = "${WORKDIR}/gcc-${PV}"
-B = "${S}/build.${HOST_SYS}.${TARGET_SYS}"
+#S = "${WORKDIR}/gcc-${PV}"
+S = "${TMPDIR}/work-shared/gcc-${PV}/gcc-${PV}"
+B = "${WORKDIR}/gcc-${PV}/build.${HOST_SYS}.${TARGET_SYS}"
+
+# SS means Shared Stamps directory
+SS = "${TMPDIR}/stamps/work-shared/gcc-${PV}"
+do_fetch[stamp-base] = "${SS}"
+do_unpack[stamp-base] = "${SS}"
+do_patch[stamp-base] = "${SS}"
+
+# SW means Shared Work directory
+SW = "${TMPDIR}/work-shared/gcc-${PV}"
+WORKDIR_task-unpack = "${SW}"
+WORKDIR_task-patch = "${SW}"
 
 target_includedir ?= "${includedir}"
 target_libdir ?= "${libdir}"
 target_base_libdir ?= "${base_libdir}"
 target_prefix ?= "${prefix}"
+
+CLEANFUNCS += "workshared_clean"
+# The do_clean should be exclusive since share ${S}
+do_clean[lockfiles] = "${TMPDIR}/stamps/work-shared/gcc-${PV}.clean.lock"
+
+python workshared_clean () {
+	"""clear the source directory"""
+	dir = bb.data.expand("${SW}", d)
+	bb.note("Removing " + dir)
+	oe.path.remove(dir)
+
+	"""clear the the stamps in work-shared"""
+	dir = "%s.*" % bb.data.expand(d.getVarFlag('do_fetch', 'stamp-base', True), d)
+	bb.note("Removing " + dir)
+	oe.path.remove(dir)
+}
meta/recipes-devtools/gcc/gcc-configure-common.inc +2 -1
index f7b5836..c32242d 100644
@@ -87,7 +87,15 @@ do_configure () {
 	export ARCH_FLAGS_FOR_TARGET="${ARCH_FLAGS_FOR_TARGET}"
 	(cd ${S} && gnu-configize) || die "failure running gnu-configize"
 
-	# teach gcc to find correct target includedir when checking libc ssp support
+	# teach gcc to find correct target includedir when checking libc ssp support,
+	# save the files before hack them, so that we can always hack the fresh one.

meta/recipes-devtools/gcc/gcc-configure-common.inc do_configure () { · line 101

Quoted 10 lines
meta/recipes-devtools/gcc/gcc-configure-common.inc +7 -0
+	for i in configure.ac Makefile.in defaults.h configure; do
+		[ ! -f ${S}/gcc/$i.orig ] && cp ${S}/gcc/$i ${S}/gcc/$i.orig
+		cp ${S}/gcc/$i.orig ${S}/gcc/$i
+	done
+	# Make sure that ${S}/configure is newer than ${S}/gcc/configure to prevent
+	# auto run autoconf.
+	touch ${S}/configure
 	sed -i 's:^\([ 	]*\)glibc_header_dir=\"${with_build_sysroot}/usr/include\":\1glibc_header_dir=\"${with_build_sysroot}${SYSTEMHEADERS}\":g' ${S}/gcc/configure.ac
 	sed -i 's:^\([ 	]*\)glibc_header_dir=\"${with_build_sysroot}/usr/include\":\1glibc_header_dir=\"${with_build_sysroot}${SYSTEMHEADERS}\":g' ${S}/gcc/configure
 

If I run something like a meta-toolchain build at the same time as a target toolchain build, this isn't going to work. It will also break when we start doing multilib builds with multiple toolchains as multiple builds will use ${S} at once.

I think we're going to need to be able to influence these things we currently set directly either from the configure commandline, or from config files in ${B} instead of ${S}.

Cheers,

Richard