package/ccache: bump to version 3.2.1
PATCH 1/2 package/ccache: bump to version 3.2.1
--- package/ccache/ccache.hash | 4 ++-- package/ccache/ccache.mk | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-)
package/ccache/ccache.hash +2 -2
@@ -1,2 +1,2 @@ -# Verified key http://samba.org/ftp/ccache/ccache-3.1.11.tar.xz.asc - sha256 computed locally -sha256 60e9f3f1e60f528cfce910f3224bcc698029320f6ab77ec09a7b46bda2bc678e ccache-3.1.11.tar.xz +# Verified key http://samba.org/ftp/ccache/ccache-3.2.1.tar.xz.asc - sha256 computed locally +sha256 872af34e1d2cef18694001d29db7f167d644e7602ca32cd6c74b28d81458bd9d ccache-3.2.1.tar.xz
package/ccache/ccache.mk +5 -5
@@ -4,7 +4,7 @@ # ################################################################################ -CCACHE_VERSION = 3.1.11 +CCACHE_VERSION = 3.2.1 CCACHE_SITE = http://samba.org/ftp/ccache CCACHE_SOURCE = ccache-$(CCACHE_VERSION).tar.xz CCACHE_LICENSE = GPLv3+, others
@@ -19,7 +19,7 @@ CCACHE_LICENSE_FILES = LICENSE.txt GPL-3.0.txt # to use HOSTCC_NOCCACHE as the compiler. Instead, we take the easy # path: tell ccache to use its internal copy of zlib, so that ccache # has zero dependency besides the C library. -HOST_CCACHE_CONF_OPTS += ccache_cv_zlib_1_2_3=no +HOST_CCACHE_CONF_OPTS += --with-bundled-zlib # Patch host-ccache as follows: # - Use BR_CACHE_DIR instead of CCACHE_DIR, because CCACHE_DIR
@@ -34,11 +34,11 @@ HOST_CCACHE_CONF_OPTS += ccache_cv_zlib_1_2_3=no # the need to specify BR_CACHE_DIR when invoking ccache directly. define HOST_CCACHE_PATCH_CONFIGURATION sed -i 's,getenv("CCACHE_DIR"),getenv("BR_CACHE_DIR"),' $(@D)/ccache.c - sed -i 's,getenv("CCACHE_COMPILERCHECK"),"none",' $(@D)/ccache.c - sed -i 's,"%s/.ccache","$(BR_CACHE_DIR)",' $(@D)/ccache.c + sed -i 's,conf->compiler_check = x_strdup("mtime"),conf->compiler_check = x_strdup("none"),' $(@D)/conf.c + sed -i 's,"%s/.ccache","$(BR_CACHE_DIR)",' $(@D)/conf.c endef -HOST_CCACHE_POST_CONFIGURE_HOOKS += HOST_CCACHE_PATCH_CONFIGURATION +HOST_CCACHE_POST_PATCH_HOOKS += HOST_CCACHE_PATCH_CONFIGURATION define HOST_CCACHE_MAKE_CACHE_DIR mkdir -p $(BR_CACHE_DIR)
Signature · 1 line
1.7.10.4
PATCH 2/2 package/ccache: add wrapper for internal toolchain
This patch will make ccache to use mtime for compiler checks except for BR's internal toolchain, where using mtime would make ccache useless, because of the recompilations of the binaries. In this case we use a wrapper script and an md5 hash file with the BR .config options as suggested previously by Arnout.
--- Config.in | 7 ------- package/ccache/ccache.mk | 25 ++++++++++++++++++++----- 2 files changed, 20 insertions(+), 12 deletions(-)
Config.in +0 -7
@@ -257,13 +257,6 @@ config BR2_CCACHE up future builds. By default, the cache is stored in $HOME/.buildroot-ccache. - Note that Buildroot does not try to invalidate the cache - contents when the compiler changes in an incompatible - way. Therefore, if you make a change to the compiler version - and/or configuration, you are responsible for purging the - ccache cache by removing the $HOME/.buildroot-ccache - directory. - if BR2_CCACHE config BR2_CCACHE_DIR
package/ccache/ccache.mk +20 -5
@@ -26,15 +26,10 @@ HOST_CCACHE_CONF_OPTS += --with-bundled-zlib # is already used by autotargets for the ccache package. # BR_CACHE_DIR is exported by Makefile based on config option # BR2_CCACHE_DIR. -# - ccache shouldn't use the compiler binary mtime to detect a change in -# the compiler, because in the context of Buildroot, that completely -# defeats the purpose of ccache. Of course, that leaves the user -# responsible for purging its cache when the compiler changes. # - Change hard-coded last-ditch default to match path in .config, to avoid # the need to specify BR_CACHE_DIR when invoking ccache directly. define HOST_CCACHE_PATCH_CONFIGURATION sed -i 's,getenv("CCACHE_DIR"),getenv("BR_CACHE_DIR"),' $(@D)/ccache.c - sed -i 's,conf->compiler_check = x_strdup("mtime"),conf->compiler_check = x_strdup("none"),' $(@D)/conf.c sed -i 's,"%s/.ccache","$(BR_CACHE_DIR)",' $(@D)/conf.c endef
@@ -58,6 +53,26 @@ endef HOST_CCACHE_POST_INSTALL_HOOKS += HOST_CCACHE_DO_INITIAL_SETUP endif +# When using an internal toolchain, disable mtime and use the hash of the toolchain options instead +define HOST_CCACHE_INSTALL_WRAPPER + mkdir -p $(HOST_DIR)/usr/bin/ccache.bin + mv $(HOST_DIR)/usr/bin/ccache $(HOST_DIR)/usr/bin/ccache.bin/ccache + echo "#!/bin/sh" > $(HOST_DIR)/usr/bin/ccache + echo "if [ \"$$"'1'"\" = \"$(TARGET_CC_NOCCACHE)\" ] || \\" >> $(HOST_DIR)/usr/bin/ccache + echo " [ \"$$"'1'"\" = \"$(TARGET_CXX_NOCCACHE)\" ]; then" >> $(HOST_DIR)/usr/bin/ccache + echo " export CCACHE_COMPILERCHECK=none CCACHE_EXTRAFILES=$(HOST_DIR)/ccache_toolchain.hash" >> $(HOST_DIR)/usr/bin/ccache + echo "fi" >> $(HOST_DIR)/usr/bin/ccache + echo "$(HOST_DIR)/usr/bin/ccache.bin/ccache \"$$"'@'"\"" >> $(HOST_DIR)/usr/bin/ccache + grep "BR2_ARCH\|BR2_TOOLCHAIN\|BR2_GCC\|BR2_BINUTILS\|BR2_UCLIBC\|BR2_GLIBC\|BR2_EGLIBC" $(BR2_CONFIG) | \ + grep -v "#" | md5sum > $(HOST_DIR)/ccache_toolchain.hash + chmod 755 $(HOST_DIR)/usr/bin/ccache +endef + +# Install wrapper script only if internal toolchain is used +ifeq ($(BR2_TOOLCHAIN_BUILDROOT),y) +HOST_CCACHE_POST_INSTALL_HOOKS += HOST_CCACHE_INSTALL_WRAPPER +endif + $(eval $(host-autotools-package)) ifeq ($(BR2_CCACHE),y)
Signature · 1 line
1.7.10.4
PATCH 2/2 package/ccache: add wrapper for internal toolchain
Dear Karoly Kasza,
Quoted 5 lines · Karoly Kasza
This patch will make ccache to use mtime for compiler checks except for BR's internal toolchain, where using mtime would make ccache useless, because of the recompilations of the binaries. In this case we use a wrapper script and an md5 hash file with the BR .config options as suggested previously by Arnout.
Well, even for external toolchains using the mtime on the compiler in $(HOST_DIR)/usr/bin will not work. Because the compiler in $(HOST_DIR)/usr/bin is not actually the compiler, but a binary wrapper built by Buildroot, for which the mtime will change at each build.
I must say I don't quite like the fact that the solution used here is different between internal and external toolchain, but I haven't thought too much of the problem.
How would ccache work with external toolchains?
Best regards,
Thomas
Signature · 3 lines
Thomas Petazzoni, CTO, Free Electrons Embedded Linux, Kernel and Android engineering http://free-electrons.com
PATCH 2/2 package/ccache: add wrapper for internal toolchain
Hello Thomas,
Well, even for external toolchains using the mtime on the compiler in
Quoted 4 lines
$(HOST_DIR)/usr/bin will not work. Because the compiler in $(HOST_DIR)/usr/bin is not actually the compiler, but a binary wrapper built by Buildroot, for which the mtime will change at each build.
That is true, so the wrapper for external toolchains also needs the extrafiles/hash solution. I simply missed this.
Quoted 4 lines
I must say I don't quite like the fact that the solution used here is different between internal and external toolchain, but I haven't thought too much of the problem.
I thought this would be more efficient, but the above statement kills it.
Quoted 2 lines
How would ccache work with external toolchains?
Well, using mtime is out of question because of the BR built wrapper, so it requires the
same solution: a wrapper for ccache and using extrafiles/hash as for the internal.
This means a wrapper will always be needed, and has to recognize the toolchain - mtime for HOST_CC and EXTRAFILES for TARGET_CC always.
Kind regards,
Karoly
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.busybox.net/pipermail/buildroot/attachments/20150426/a70247b4/attachment.html>
PATCH 2/2 package/ccache: add wrapper for internal toolchain
Dear K?roly Kasza,
Quoted 2 lines · K?roly Kasza
That is true, so the wrapper for external toolchains also needs the extrafiles/hash solution. I simply missed this.
No problem, but it would indeed be nice to have a solution that covers both cases.
Quoted 5 lines
Quoted 3 lines
I must say I don't quite like the fact that the solution used here is different between internal and external toolchain, but I haven't thought too much of the problem.
I thought this would be more efficient, but the above statement kills it.
I certainly don't want to kill anything. This ccache problem has been around for a long time, and it's definitely great to see someone taking care of this relatively complicated issue.
Quoted 7 lines
Quoted 2 lines
How would ccache work with external toolchains?
Well, using mtime is out of question because of the BR built wrapper, so it requires the
same solution: a wrapper for ccache and using extrafiles/hash as for the internal.
Indeed. But maybe the existing external toolchain wrapper can be used. We probably don't want to have a shell wrapper call a C wrapper calling the compiler.
Thomas
Signature · 3 lines
Thomas Petazzoni, CTO, Free Electrons Embedded Linux, Kernel and Android engineering http://free-electrons.com
PATCH 2/2 package/ccache: add wrapper for internal toolchain
Dear Thomas,
No problem, but it would indeed be nice to have a solution that covers
Quoted 1 line
both cases.
Strongly agree.
Quoted 4 lines
Indeed. But maybe the existing external toolchain wrapper can be used. We probably don't want to have a shell wrapper call a C wrapper calling the compiler.
I'll look into it!
Until then, I'll just resend the version bump with a more extensive commit message, as Baruch suggested.
Regards,
Karoly
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.busybox.net/pipermail/buildroot/attachments/20150426/29b6bc2c/attachment.html>
PATCH 2/2 package/ccache: add wrapper for internal toolchain
Hello Thomas, all,
Quoted 9 lines
Quoted 5 lines
Indeed. But maybe the existing external toolchain wrapper can be used. We probably don't want to have a shell wrapper call a C wrapper calling the compiler.
I'll look into it!
Some overview as I understand:
By default ccache works by doing a hash on the compiler, it's arguments, and the source file to choose a previously cached result instead of compiling it. If no fitting hash found, it just falls back compiling (and storing the result in cache).
In Buildroot, we can use 3+1 different compilers:
- The host compiler (mostly /usr/bin/gcc or whatever) - this changes only if the host is updated, the default ccache checking method (mtime) can efficiently used with it. No deep changes needed to make this work, only applies for host-* packages. If we decide to use a ccache wrapper script (see below), than this can be called from it. If we want to skip the wrapper script here, then all occurrences of the $(CCACHE) variable everywhere in BR needs to reviewed/modified, as $(CCACHE) $(HOSTCC) and $(CCACHE) $(TARGET_CC) will have to be separated.
- The external toolchain target compiler - this is the same in terms of binary, but the problem as ThomasP. pointed out, that it is called by a wrapper that is recompiled every time, thus invalidating the cache if mtime is used on it. Ccache can only hash it's first argument ($1) with mtime, so the solution is to turn off this kind of hashing (compiler_check=none) and use another file (extra_files) as the source of the hash. This file can be an extract of the BR .config file, which only contains compiler/toolchain related settings, thus recompiling different packages won't invalidate the compiler's identity.
After roughly understanding the ext-toolchain-wrapper, I think there are two paths to follow:
1) Use a wrapper script to call ccache, which will call the wrapper
binary for the compiler. In this script, extrafiles can be set up as an environment variable. This is far more easier, requires a lot less modifications, but should (may) be a bit slower, as we have to call a shell script before every compiler invocation. If we want a universal solution for internal and external toolchains, this can be used.
2) Modify BR to let the wrapper binary decide whether to use ccache
before calling the real compiler. This requires tampering with the TARGET_CC and TARGET_CC_NOCCACHE variables in BR at least and modifying the source code of ext-toolchain-wrapper, but should be a bit more faster, as it won't call an additional shell script every time gcc/g++ is invoked. This won't be universal naturally.
- The internal toolchain - this is recompiled every time, so mtime is out of question. The external toolchain solution is also valid here, using method 1) - ccache wrapper script. I could think of no other viable solutions.
+1 the internal toolchain gcc-initial stage, for which the simple internal toolchain's extrafiles setting should work.
My questions are:
- Did I miss something?
- Anybody has other solutions for hashing the external toolchain?
- If not, which should be implemented?
- Do we want an universal solution (wrapper script) or independent
solutions for every case?
Best regards,
Karoly
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.busybox.net/pipermail/buildroot/attachments/20150427/df1eb388/attachment.html>
PATCH 1/2 package/ccache: bump to version 3.2.1
In reply to Karoly KaszaHi Karoly,
Quoted 10 lines · Karoly Kasza
package/ccache/ccache.mk +1
-1@@ -19,7 +19,7 @@ CCACHE_LICENSE_FILES = LICENSE.txt GPL-3.0.txt # to use HOSTCC_NOCCACHE as the compiler. Instead, we take the easy # path: tell ccache to use its internal copy of zlib, so that ccache # has zero dependency besides the C library. -HOST_CCACHE_CONF_OPTS += ccache_cv_zlib_1_2_3=no +HOST_CCACHE_CONF_OPTS += --with-bundled-zlib # Patch host-ccache as follows: # - Use BR_CACHE_DIR instead of CCACHE_DIR, because CCACHE_DIR
package/ccache/ccache.mk HOST_CCACHE_CONF_OPTS += ccache_cv_zlib_1_2_3=no · line 39
Quoted 8 lines
package/ccache/ccache.mk +2
-2@@ -34,11 +34,11 @@ HOST_CCACHE_CONF_OPTS += ccache_cv_zlib_1_2_3=no # the need to specify BR_CACHE_DIR when invoking ccache directly. define HOST_CCACHE_PATCH_CONFIGURATION sed -i 's,getenv("CCACHE_DIR"),getenv("BR_CACHE_DIR"),' $(@D)/ccache.c - sed -i 's,getenv("CCACHE_COMPILERCHECK"),"none",' $(@D)/ccache.c - sed -i 's,"%s/.ccache","$(BR_CACHE_DIR)",' $(@D)/ccache.c + sed -i 's,conf->compiler_check = x_strdup("mtime"),conf->compiler_check = x_strdup("none"),' $(@D)/conf.c + sed -i 's,"%s/.ccache","$(BR_CACHE_DIR)",' $(@D)/conf.c endef
You are doing non trivial changes here and above. Please explain in a commit log what you are doing, and why this is needed as part of the version bump.
baruch
Signature · 3 lines
http://baruch.siach.name/blog/ ~. .~ Tk Open Systems
=}------------------------------------------------ooO--U--Ooo------------{=
- baruch at tkos.co.il - tel: +972.2.679.5364, http://www.tkos.co.il -PATCH 1/2 package/ccache: bump to version 3.2.1
Hello Baruch,
Quoted 8 lines
-HOST_CCACHE_CONF_OPTS += ccache_cv_zlib_1_2_3=no
Review
Quoted 7 lines
(quoted diff) +2
-2+HOST_CCACHE_CONF_OPTS += --with-bundled-zlib - sed -i 's,getenv("CCACHE_COMPILERCHECK"),"none",' $(@D)/ccache.c - sed -i 's,"%s/.ccache","$(BR_CACHE_DIR)",' $(@D)/ccache.c + sed -i 's,conf->compiler_check =
Quoted 9 lines
x_strdup("mtime"),conf->compiler_check =
x_strdup("none"),' $(@D)/conf.cQuoted 1 line
+ sed -i 's,"%s/.ccache","$(BR_CACHE_DIR)",' $(@D)/conf.c
You are doing non trivial changes here and above. Please explain in a commit
log what you are doing, and why this is needed as part of the version bump.
As you probably see, those lines do the exact same as before, just the structure of the ccache sources changed with the major version bump. I'll wait for some comments on the second patch regarding the introduction of a wrapper script and resend with a more informative git log.
Kind regards,
Karoly
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.busybox.net/pipermail/buildroot/attachments/20150426/6a315ca3/attachment.html>
PATCH 1/2 package/ccache: bump to version 3.2.1
Hi K?roly,
Quoted 6 lines · K?roly Kasza
Quoted 6 lines
package/ccache/ccache.mk HOST_CCACHE_CONF_OPTS += ccache_cv_zlib_1_2_3=no
Quoted 3 lines
package/ccache/ccache.mk +1
-2- sed -i 's,getenv("CCACHE_COMPILERCHECK"),"none",' $(@D)/ccache.c - sed -i 's,"%s/.ccache","$(BR_CACHE_DIR)",' $(@D)/ccache.c + sed -i 's,conf->compiler_check =x_strdup("mtime"),conf->compiler_check =
x_strdup("none"),' $(@D)/conf.cQuoted 1 line
+ sed -i 's,"%s/.ccache","$(BR_CACHE_DIR)",' $(@D)/conf.c
Quoted 8 lines
Quoted 3 lines
You are doing non trivial changes here and above. Please explain in a commit log what you are doing, and why this is needed as part of the version bump.
As you probably see, those lines do the exact same as before, just the structure of the ccache sources changed with the major version bump. I'll wait for some comments on the second patch regarding the introduction of a wrapper script and resend with a more informative git log.
Thanks. When doing so please comment also on the _HOOKS change.
baruch
Signature · 3 lines
http://baruch.siach.name/blog/ ~. .~ Tk Open Systems
=}------------------------------------------------ooO--U--Ooo------------{=
- baruch at tkos.co.il - tel: +972.2.679.5364, http://www.tkos.co.il -