Skip to contentSkip to conversation

toolchain-external: support downloading custom toolchains + related misc. fixes

PATCH New Thomas De Schampheleire 3 participants 9 messages Aug 2, 2011 – Aug 19, 2011

lore
Share & tools
lore

Thomas De Schampheleire

author
#0 lore raw

PATCH 1/1 toolchain-external: support downloading custom toolchains + related misc. fixes

This patch series originates from the need to be able to use a custom external (buildroot) toolchain, downloaded from an scp location. Support for scp is not included in this series (it was submitted previously, but needs to be retriggered) but the prerequisites relevant for toolchain-external are.

While I was at it, a few CodeSourcery typos were fixed as well.

Signed-off-by Thomas De Schampheleire <[email protected]>

---
For the patches related to host-tar, I would like to get some feedback regarding the best way to handle this. I found an earlier thread (http://lists.busybox.net/pipermail/buildroot/2011-March/042191.html - alsa-utils and nctp troubles with 2011.02) where the tar issue was left to rest because it was attributed to an old distribution. However, RedHat/CentOS still ship these old versions of tar, so I think the problem remains. The proposed changes do work and fix my problems, but may need refinement (as described in the patch comment).

 package/Makefile.package.in              |  9 ++++++++-
 package/tar/tar.mk                       |  3 +++
 toolchain/toolchain-external/Config.in   |  6 +++---
 toolchain/toolchain-external/ext-tool.mk |  8 ++++----
 4 files changed, 18 insertions(+), 8 deletions(-)
author
#1 lore raw

PATCH 1/1 toolchain-external: allow downloading a custom toolchain

Custom external toolchains currently already have to be extracted by the user. This patch adds support for downloading a custom toolchain in the form of a tarball, which will be extracted by buildroot.

Signed-off-by Thomas De Schampheleire <[email protected]>
toolchain/toolchain-external/Config.in +12 -1
@@ -189,12 +189,23 @@
 config BR2_TOOLCHAIN_EXTERNAL_DOWNLOAD
 	bool "Download toolchain automatically"
 	default y
-	depends on !BR2_TOOLCHAIN_EXTERNAL_CUSTOM
 	help
 	  When enabled, Buildroot will automatically download and
 	  install the selected external toolchain. When disabled,
 	  Buildroot will use a pre-installed toolchain.
 
+config BR2_TOOLCHAIN_EXTERNAL_SITE
+	string "Toolchain download site"
+	depends on BR2_TOOLCHAIN_EXTERNAL_CUSTOM && BR2_TOOLCHAIN_EXTERNAL_DOWNLOAD
+	help
+	  Base URL at which the custom toolchain can be downloaded.
+
+config BR2_TOOLCHAIN_EXTERNAL_SOURCE
+	string "Toolchain filename"
+	depends on BR2_TOOLCHAIN_EXTERNAL_CUSTOM && BR2_TOOLCHAIN_EXTERNAL_DOWNLOAD
+	help
+	  Filename of the toolchain package to download.
+
 config BR2_TOOLCHAIN_EXTERNAL_PATH
 	string "Toolchain path"
 	default "/path/to/toolchain/usr"
toolchain/toolchain-external/ext-tool.mk +9 -0
@@ -160,6 +160,10 @@
 TOOLCHAIN_EXTERNAL_DEPENDENCIES = $(STAMP_DIR)/ext-toolchain-checked
 endif
 
+ifeq ($(BR2_TOOLCHAIN_EXTERNAL_CUSTOM),y)
+TOOLCHAIN_EXTERNAL_DEPENDENCIES += $(STAMP_DIR)/ext-toolchain-checked
+endif
+
 ifeq ($(BR2_TOOLCHAIN_EXTERNAL_CODESOURCERY_ARM2009Q1),y)
 TOOLCHAIN_EXTERNAL_SITE=http://www.codesourcery.com/sgpp/lite/arm/portal/package4571/public/arm-none-linux-gnueabi/
 TOOLCHAIN_EXTERNAL_SOURCE=arm-2009q1-203-arm-none-linux-gnueabi-i686-pc-linux-gnu.tar.bz2

@@ -197,11 +201,16 @@
 TOOLCHAIN_EXTERNAL_SOURCE_2 = blackfin-toolchain-uclibc-full-2010R1-RC4.i386.tar.bz2
 TOOLCHAIN_EXTERNAL_SOURCE   = $(TOOLCHAIN_EXTERNAL_SOURCE_1) $(TOOLCHAIN_EXTERNAL_SOURCE_2)
 else
+# Custom toolchain
+TOOLCHAIN_EXTERNAL_SITE=$(call qstrip,$(BR2_TOOLCHAIN_EXTERNAL_SITE))
+TOOLCHAIN_EXTERNAL_SOURCE=$(call qstrip,$(BR2_TOOLCHAIN_EXTERNAL_SOURCE))
 # A value must be set (even if unused), otherwise the
 # $(DL_DIR)/$(TOOLCHAIN_EXTERNAL_SOURCE) rule would override the main
 # $(DL_DIR) rule
+ifeq (,$(TOOLCHAIN_EXTERNAL_SOURCE))
 TOOLCHAIN_EXTERNAL_SOURCE=none
 endif
+endif
 
 # Special handling for Blackfin toolchain, because of the split in two
 # tarballs, and the organization of tarball contents. The tarballs
author
#2 lore raw

PATCH 1/1 Support URI schemes in DOWNLOAD function, when not using gentargets

In reply to Thomas De Schampheleire

When using the gentargets/autotools/cmake infrastructure, the DOWNLOAD function uses an appropriate download method based on $(PKG)_SITE_METHOD, which is autodetected based on the URI if none was explicitly set. When the DOWNLOAD function is called directly, when not using gentargets, the SITE_METHOD is not autodetected, and thus the download defaults to wget.

This patch adds URI scheme detection directly to the DOWNLOAD method, in case no SITE_METHOD was detected yet.

Signed-off-by Thomas De Schampheleire <[email protected]>
package/Makefile.package.in +8 -1
@@ -184,12 +184,19 @@
 # $(call DOWNLOAD,$(FOO_SITE),$(FOO_SOURCE))
 ################################################################################
 
+geturischeme=$(firstword $(subst ://, ,$(call qstrip,$(1))))
+
 define DOWNLOAD
 	$(Q)if test -n "$(call qstrip,$(BR2_PRIMARY_SITE))" ; then \
 		$(call $(DL_MODE)_WGET,$(BR2_PRIMARY_SITE),$(2)) && exit ; \
 	fi ; \
 	if test -n "$(1)" ; then \
-		case "$($(PKG)_SITE_METHOD)" in \
+		if test -z "$($(PKG)_SITE_METHOD)" ; then \
+			scheme="$(call geturischeme,$(1))" ; \
+		else \
+			scheme="$($(PKG)_SITE_METHOD)" ; \
+		fi ; \
+		case "$$scheme" in \
 			git) $($(DL_MODE)_GIT) && exit ;; \
 			svn) $($(DL_MODE)_SVN) && exit ;; \
 			bzr) $($(DL_MODE)_BZR) && exit ;; \
author
#3 lore raw

PATCH 1/1 toolchain-external: fix some typos CodeSoucery -> CodeSourcery

In reply to Thomas De Schampheleire
Signed-off-by Thomas De Schampheleire <[email protected]>
toolchain/toolchain-external/Config.in +3 -3
@@ -120,7 +120,7 @@
             - MIPS32 little endian soft float uclibc
 
 config BR2_TOOLCHAIN_EXTERNAL_CODESOURCERY_POWERPC201009
-	bool "CodeSoucery PowerPC 2010.09"
+	bool "CodeSourcery PowerPC 2010.09"
 	depends on BR2_powerpc
 	select BR2_TOOLCHAIN_EXTERNAL_GLIBC
 	select BR2_INSTALL_LIBSTDCPP

@@ -138,7 +138,7 @@
             - 970 glibc hard-float, 64 bits
 
 config BR2_TOOLCHAIN_EXTERNAL_CODESOURCERY_SH201009
-	bool "CodeSoucery SH 2010.09"
+	bool "CodeSourcery SH 2010.09"
 	depends on BR2_sh4a || BR2_sh4aeb
 	select BR2_TOOLCHAIN_EXTERNAL_GLIBC
 	select BR2_INSTALL_LIBSTDCPP

@@ -153,7 +153,7 @@
             - SH4A, uClibc, big endian
 
 config BR2_TOOLCHAIN_EXTERNAL_CODESOURCERY_SH2A_201009
-	bool "CodeSoucery SH 2010.09"
+	bool "CodeSourcery SH 2010.09"
 	depends on BR2_sh2a
 	select BR2_TOOLCHAIN_EXTERNAL_UCLIBC
 	select BR2_INSTALL_LIBSTDCPP
#4 lore raw

PATCH 1/1 toolchain-external: fix some typos CodeSoucery -> CodeSourcery

Le Tue, 02 Aug 2011 20:33:17 +0200,
Thomas De Schampheleire <[email protected]> a ?crit :

Quoted 1 line
Signed-off-by Thomas De Schampheleire <[email protected]>
Acked-by Thomas Petazzoni <[email protected]>

Peter: this one should go in 2011.08, it's just fixes. The other patches need discussion, so I'd prefer to handle them post 2011.08.

Thomas

Signature · 4 lines
Thomas Petazzoni, Free Electrons
Kernel, drivers, real-time and embedded Linux
development, consulting, training and support.
http://free-electrons.com
author
#6 lore raw

PATCH 1/1 tar: create a host version

In reply to Thomas De Schampheleire

Some systems, like RedHat/CentOS, are even today shipped with ancient versions of GNU tar, like 1.15. This can cause trouble, for example when unpacking archives containing hardlinks with the --strip-components option.

To avoid such problems, let buildroot create a host-tar that can be used when unpacking.

Signed-off-by Thomas De Schampheleire <[email protected]>
package/tar/tar.mk +3 -0
@@ -8,3 +8,6 @@
 TAR_SITE = $(BR2_GNU_MIRROR)/tar
 
 $(eval $(call AUTOTARGETS,package,tar))
+$(eval $(call AUTOTARGETS,package,tar,host))
+
+HOST_TAR = $(HOST_DIR)/usr/bin/tar
author
#7 lore raw

PATCH 1/1 toolchain-external: use host-tar instead of tar to unpack toolchains

In reply to Thomas De Schampheleire

Using the system tar with --strip-components causes problems when its version is too old (<1.17). Even recent releases of RedHat/CentOS still ship with tar 1.15. To avoid such problems, always use host-tar instead of tar to unpack toolchains.

Signed-off-by Thomas De Schampheleire <[email protected]>

---
To be honest, I'm not sure what the best way is to handle this. The same problems with tar could occur in other places than just external toolchains. So maybe we should use host-tar in more places. On the other hand, most users have a recent, working, tar version on there system. It may thus make more sense to detect the system tar version and only use host-tar if it's too old.

toolchain/toolchain-external/ext-tool.mk +4 -4
@@ -155,7 +155,7 @@
 endif
 
 ifeq ($(BR2_TOOLCHAIN_EXTERNAL_DOWNLOAD),y)
-TOOLCHAIN_EXTERNAL_DEPENDENCIES = $(TOOLCHAIN_EXTERNAL_DIR)/.extracted
+TOOLCHAIN_EXTERNAL_DEPENDENCIES = host-tar $(TOOLCHAIN_EXTERNAL_DIR)/.extracted
 else
 TOOLCHAIN_EXTERNAL_DEPENDENCIES = $(STAMP_DIR)/ext-toolchain-checked
 endif

@@ -227,9 +227,9 @@
 $(TOOLCHAIN_EXTERNAL_DIR)/.extracted: $(DL_DIR)/$(TOOLCHAIN_EXTERNAL_SOURCE_1) $(DL_DIR)/$(TOOLCHAIN_EXTERNAL_SOURCE_2)
 	mkdir -p $(@D)
 	$(INFLATE$(suffix $(TOOLCHAIN_EXTERNAL_SOURCE_1))) $(DL_DIR)/$(TOOLCHAIN_EXTERNAL_SOURCE_1) | \
-		$(TAR) $(TAR_STRIP_COMPONENTS)=3 --hard-dereference -C $(@D) $(TAR_OPTIONS) -
+		$(HOST_TAR) $(TAR_STRIP_COMPONENTS)=3 --hard-dereference -C $(@D) $(TAR_OPTIONS) -
 	$(INFLATE$(suffix $(TOOLCHAIN_EXTERNAL_SOURCE_2))) $(DL_DIR)/$(TOOLCHAIN_EXTERNAL_SOURCE_2) | \
-		$(TAR) $(TAR_STRIP_COMPONENTS)=3 --hard-dereference -C $(@D) $(TAR_OPTIONS) -
+		$(HOST_TAR) $(TAR_STRIP_COMPONENTS)=3 --hard-dereference -C $(@D) $(TAR_OPTIONS) -
 ifeq ($(TOOLCHAIN_EXTERNAL_PREFIX),bfin-uclinux)
 	rm -rf $(TOOLCHAIN_EXTERNAL_DIR)/bfin-linux-uclibc
 	mv $(TOOLCHAIN_EXTERNAL_DIR)/bfin-uclinux $(TOOLCHAIN_EXTERNAL_DIR)/tmp

@@ -249,7 +249,7 @@
 
 $(TOOLCHAIN_EXTERNAL_DIR)/.extracted: $(DL_DIR)/$(TOOLCHAIN_EXTERNAL_SOURCE)
 	mkdir -p $(@D)
-	$(INFLATE$(suffix $(TOOLCHAIN_EXTERNAL_SOURCE))) $^ | $(TAR) $(TAR_STRIP_COMPONENTS)=1 -C $(@D) $(TAR_OPTIONS) -
+	$(INFLATE$(suffix $(TOOLCHAIN_EXTERNAL_SOURCE))) $^ | $(HOST_TAR) $(TAR_STRIP_COMPONENTS)=1 -C $(@D) $(TAR_OPTIONS) -
 	$(Q)touch $@
 endif
author
#8 lore raw

PATCH 1/1 toolchain-external: use host-tar instead of tar to unpack toolchains

Quoted 15 lines · Thomas De Schampheleire

Using the system tar with --strip-components causes problems when its version is too old (<1.17). Even recent releases of RedHat/CentOS still ship with tar 1.15. To avoid such problems, always use host-tar instead of tar to unpack toolchains.

Signed-off-by Thomas De Schampheleire <[email protected]>

---
To be honest, I'm not sure what the best way is to handle this. The same problems with tar could occur in other places than just external toolchains. So maybe we should use host-tar in more places. On the other hand, most users have a recent, working, tar version on there system. It may thus make more sense to detect the system tar version and only use host-tar if it's too old.

toolchain/toolchain-external/ext-tool.mk +0 -0
@@ -155,7 +155,7 @@

?endif

?ifeq ($(BR2_TOOLCHAIN_EXTERNAL_DOWNLOAD),y)

toolchain/toolchain-external/ext-tool.mk line 158

Quoted 2 lines
toolchain/toolchain-external/ext-tool.mk +1 -1
-TOOLCHAIN_EXTERNAL_DEPENDENCIES = $(TOOLCHAIN_EXTERNAL_DIR)/.extracted
+TOOLCHAIN_EXTERNAL_DEPENDENCIES = host-tar $(TOOLCHAIN_EXTERNAL_DIR)/.extracted

It turns out that this has the annoying side-effect of re-running the 'ext-toolchain-installed' step each time, because host-tar is not an existing file. One can workaround this as follows:

include package/tar/tar.mk $(STAMP_DIR)/tar-installed:
$(BUILD_DIR)/host-tar-$(TAR_VERSION)/.stamp_host_installed

	$(Q)touch $@

TOOLCHAIN_EXTERNAL_DEPENDENCIES = $(STAMP_DIR)tar-installed $(TOOLCHAIN_EXTERNAL_DIR)/.extracted

This is not very beautiful because you need to include package/tar/tar.mk to get to $(TAR_VERSION). Is there a better way?

Thanks,
Thomas

Quoted 25 lines

?else
?TOOLCHAIN_EXTERNAL_DEPENDENCIES = $(STAMP_DIR)/ext-toolchain-checked ?endif

toolchain/toolchain-external/ext-tool.mk +2 -2
@@ -227,9 +227,9 @@
 ?$(TOOLCHAIN_EXTERNAL_DIR)/.extracted: $(DL_DIR)/$(TOOLCHAIN_EXTERNAL_SOURCE_1) $(DL_DIR)/$(TOOLCHAIN_EXTERNAL_SOURCE_2)
 ? ? ? ?mkdir -p $(@D)
 ? ? ? ?$(INFLATE$(suffix $(TOOLCHAIN_EXTERNAL_SOURCE_1))) $(DL_DIR)/$(TOOLCHAIN_EXTERNAL_SOURCE_1) | \
- ? ? ? ? ? ? ? $(TAR) $(TAR_STRIP_COMPONENTS)=3 --hard-dereference -C $(@D) $(TAR_OPTIONS) -
+ ? ? ? ? ? ? ? $(HOST_TAR) $(TAR_STRIP_COMPONENTS)=3 --hard-dereference -C $(@D) $(TAR_OPTIONS) -
 ? ? ? ?$(INFLATE$(suffix $(TOOLCHAIN_EXTERNAL_SOURCE_2))) $(DL_DIR)/$(TOOLCHAIN_EXTERNAL_SOURCE_2) | \
- ? ? ? ? ? ? ? $(TAR) $(TAR_STRIP_COMPONENTS)=3 --hard-dereference -C $(@D) $(TAR_OPTIONS) -
+ ? ? ? ? ? ? ? $(HOST_TAR) $(TAR_STRIP_COMPONENTS)=3 --hard-dereference -C $(@D) $(TAR_OPTIONS) -
 ?ifeq ($(TOOLCHAIN_EXTERNAL_PREFIX),bfin-uclinux)

? ? ? ?rm -rf $(TOOLCHAIN_EXTERNAL_DIR)/bfin-linux-uclibc
? ? ? ?mv $(TOOLCHAIN_EXTERNAL_DIR)/bfin-uclinux $(TOOLCHAIN_EXTERNAL_DIR)/tmp

toolchain/toolchain-external/ext-tool.mk +1 -1
@@ -249,7 +249,7 @@
 
 ?$(TOOLCHAIN_EXTERNAL_DIR)/.extracted: $(DL_DIR)/$(TOOLCHAIN_EXTERNAL_SOURCE)
 ? ? ? ?mkdir -p $(@D)
- ? ? ? $(INFLATE$(suffix $(TOOLCHAIN_EXTERNAL_SOURCE))) $^ | $(TAR) $(TAR_STRIP_COMPONENTS)=1 -C $(@D) $(TAR_OPTIONS) -
+ ? ? ? $(INFLATE$(suffix $(TOOLCHAIN_EXTERNAL_SOURCE))) $^ | $(HOST_TAR) $(TAR_STRIP_COMPONENTS)=1 -C $(@D) $(TAR_OPTIONS) -

? ? ? ?$(Q)touch $@
?endif