Introduce suitable-extractor and add host-xzcat dependency
PATCH v3 1/1 Introduce suitable-extractor and add host-xzcat dependency
v3: fix $(firstword) usage
v2: update based on Thomas Petazzoni's comments.
PATCH v3 1/1 infra: introduce suitable-extractor helper function
In order to simplify determining the right extractor tool for a given file type, this patch introduces a make function 'suitable-extractor'. Its usage is $(call suitable-extractor,filename), and it returns the path to the suitable extractor.
---
v3: Correct $(firstword) usage
(v2): new patch in series
Note: one remaining direct usage of the INFLATE variables is in package/pkg-generic.mk, but it's in the inner block where dollar signs are thrown around your ears. I'm not sure if it's possible to use the function there...
package/lsof/lsof.mk | 2 +- package/perl/perl.mk | 2 +- package/pkg-generic.mk | 4 +++- package/pkg-utils.mk | 2 ++ package/tar/tar.mk | 2 +- toolchain/toolchain-external/ext-tool.mk | 6 +++--- 6 files changed, 11 insertions(+), 7 deletions(-)
package/lsof/lsof.mk +1 -1
@@ -41,7 +41,7 @@ endif # The .tar.bz2 contains another .tar, which contains the source code. define LSOF_EXTRACT_CMDS - $(INFLATE.bz2) $(DL_DIR)/$(LSOF_SOURCE) | \ + $(call suitable-extractor,$(LSOF_SOURCE)) $(DL_DIR)/$(LSOF_SOURCE) | \ $(TAR) -O $(TAR_OPTIONS) - lsof_$(LSOF_VERSION)/lsof_$(LSOF_VERSION)_src.tar | \ $(TAR) $(TAR_STRIP_COMPONENTS)=1 -C $(LSOF_DIR) $(TAR_OPTIONS) - endef
package/perl/perl.mk +1 -1
@@ -30,7 +30,7 @@ endef PERL_POST_DOWNLOAD_HOOKS += PERL_CROSS_DOWNLOAD define PERL_CROSS_EXTRACT - $(INFLATE$(suffix $(PERL_CROSS_SOURCE))) $(DL_DIR)/$(PERL_CROSS_SOURCE) | \ + $(call suitable-extractor,$(PERL_CROSS_SOURCE)) $(DL_DIR)/$(PERL_CROSS_SOURCE) | \ $(TAR) $(TAR_STRIP_COMPONENTS)=1 -C $(@D) $(TAR_OPTIONS) - endef PERL_POST_EXTRACT_HOOKS += PERL_CROSS_EXTRACT
package/pkg-generic.mk +3 -1
@@ -539,7 +539,9 @@ else ifeq ($$($(2)_SITE_METHOD),hg) DL_TOOLS_DEPENDENCIES += hg endif # SITE_METHOD -DL_TOOLS_DEPENDENCIES += $(firstword $(INFLATE$(suffix $($(2)_SOURCE)))) +# $(firstword) is used here because the extractor can have arguments, like +# ZCAT="gzip -d -c", and to check for the dependency we only want 'gzip'. +DL_TOOLS_DEPENDENCIES += $(firstword $(call suitable-extractor,$($(2)_SOURCE))) endif # $(2)_KCONFIG_VAR endef # inner-generic-package
package/pkg-utils.mk +2 -0
@@ -62,6 +62,8 @@ INFLATE.tbz2 = $(BZCAT) INFLATE.tgz = $(ZCAT) INFLATE.xz = $(XZCAT) INFLATE.tar = cat +# suitable-extractor(filename): returns extractor based on suffix +suitable-extractor = $(INFLATE$(suffix $(1))) # MESSAGE Macro -- display a message in bold type MESSAGE = echo "$(TERM_BOLD)>>> $($(PKG)_NAME) $($(PKG)_VERSION) $(1)$(TERM_RESET)"
package/tar/tar.mk +1 -1
@@ -23,7 +23,7 @@ HOST_TAR_SOURCE = tar-$(TAR_VERSION).cpi define HOST_TAR_EXTRACT_CMDS mkdir -p $(@D) cd $(@D) && \ - $(INFLATE.gz) $(DL_DIR)/$(HOST_TAR_SOURCE) | cpio -i + $(call suitable-extractor,$(HOST_TAR_SOURCE)) $(DL_DIR)/$(HOST_TAR_SOURCE) | cpio -i mv $(@D)/tar-$(TAR_VERSION)/* $(@D) rmdir $(@D)/tar-$(TAR_VERSION) endef
toolchain/toolchain-external/ext-tool.mk +3 -3
@@ -337,9 +337,9 @@ ifeq ($(BR2_TOOLCHAIN_EXTERNAL_BLACKFIN_ $(TOOLCHAIN_EXTERNAL_DIR)/.extracted: $(DL_DIR)/$(TOOLCHAIN_EXTERNAL_SOURCE_1) $(DL_DIR)/$(TOOLCHAIN_EXTERNAL_SOURCE_2) mkdir -p $(@D) - $(INFLATE$(suffix $(TOOLCHAIN_EXTERNAL_SOURCE_1))) $(DL_DIR)/$(TOOLCHAIN_EXTERNAL_SOURCE_1) | \ + $(call suitable-extractor,$(TOOLCHAIN_EXTERNAL_SOURCE_1)) $(DL_DIR)/$(TOOLCHAIN_EXTERNAL_SOURCE_1) | \ $(TAR) $(TAR_STRIP_COMPONENTS)=3 --hard-dereference -C $(@D) $(TAR_OPTIONS) - - $(INFLATE$(suffix $(TOOLCHAIN_EXTERNAL_SOURCE_2))) $(DL_DIR)/$(TOOLCHAIN_EXTERNAL_SOURCE_2) | \ + $(call suitable-extractor,$(TOOLCHAIN_EXTERNAL_SOURCE_2)) $(DL_DIR)/$(TOOLCHAIN_EXTERNAL_SOURCE_2) | \ $(TAR) $(TAR_STRIP_COMPONENTS)=3 --hard-dereference -C $(@D) $(TAR_OPTIONS) - $(Q)touch $@ else
@@ -349,7 +349,7 @@ else $(TOOLCHAIN_EXTERNAL_DIR)/.extracted: $(DL_DIR)/$(TOOLCHAIN_EXTERNAL_SOURCE) mkdir -p $(@D) - $(INFLATE$(suffix $(TOOLCHAIN_EXTERNAL_SOURCE))) $^ | \ + $(call suitable-extractor,$(TOOLCHAIN_EXTERNAL_SOURCE)) $^ | \ $(TAR) $(TAR_STRIP_COMPONENTS)=1 --exclude='usr/lib/locale/*' -C $(@D) $(TAR_OPTIONS) - $(TOOLCHAIN_EXTERNAL_FIXUP_CMDS) $(Q)touch $@
PATCH v3 1/1 infra: introduce suitable-extractor helper function
Quoted 102 lines · Thomas De Schampheleire
In order to simplify determining the right extractor tool for a given file type, this patch introduces a make function 'suitable-extractor'. Its usage is $(call suitable-extractor,filename), and it returns the path to the suitable extractor.
Signed-off-by Thomas De Schampheleire <[email protected]>---
v3: Correct $(firstword) usage
(v2): new patch in seriesNote: one remaining direct usage of the INFLATE variables is in
package/pkg-generic.mk, but it's in the inner block where dollar signs are thrown around your ears. I'm not sure if it's possible to use the function there...package/lsof/lsof.mk | 2 +- package/perl/perl.mk | 2 +- package/pkg-generic.mk | 4 +++- package/pkg-utils.mk | 2 ++ package/tar/tar.mk | 2 +- toolchain/toolchain-external/ext-tool.mk | 6 +++--- 6 files changed, 11 insertions(+), 7 deletions(-)package/lsof/lsof.mk +1
-1@@ -41,7 +41,7 @@ endif # The .tar.bz2 contains another .tar, which contains the source code. define LSOF_EXTRACT_CMDS - $(INFLATE.bz2) $(DL_DIR)/$(LSOF_SOURCE) | \ + $(call suitable-extractor,$(LSOF_SOURCE)) $(DL_DIR)/$(LSOF_SOURCE) | \ $(TAR) -O $(TAR_OPTIONS) - lsof_$(LSOF_VERSION)/lsof_$(LSOF_VERSION)_src.tar | \ $(TAR) $(TAR_STRIP_COMPONENTS)=1 -C $(LSOF_DIR) $(TAR_OPTIONS) - endefpackage/perl/perl.mk +1
-1@@ -30,7 +30,7 @@ endef PERL_POST_DOWNLOAD_HOOKS += PERL_CROSS_DOWNLOAD define PERL_CROSS_EXTRACT - $(INFLATE$(suffix $(PERL_CROSS_SOURCE))) $(DL_DIR)/$(PERL_CROSS_SOURCE) | \ + $(call suitable-extractor,$(PERL_CROSS_SOURCE)) $(DL_DIR)/$(PERL_CROSS_SOURCE) | \ $(TAR) $(TAR_STRIP_COMPONENTS)=1 -C $(@D) $(TAR_OPTIONS) - endef PERL_POST_EXTRACT_HOOKS += PERL_CROSS_EXTRACTpackage/pkg-generic.mk +3
-1@@ -539,7 +539,9 @@ else ifeq ($$($(2)_SITE_METHOD),hg) DL_TOOLS_DEPENDENCIES += hg endif # SITE_METHOD -DL_TOOLS_DEPENDENCIES += $(firstword $(INFLATE$(suffix $($(2)_SOURCE)))) +# $(firstword) is used here because the extractor can have arguments, like +# ZCAT="gzip -d -c", and to check for the dependency we only want 'gzip'. +DL_TOOLS_DEPENDENCIES += $(firstword $(call suitable-extractor,$($(2)_SOURCE))) endif # $(2)_KCONFIG_VAR endef # inner-generic-packagepackage/pkg-utils.mk +2
-0@@ -62,6 +62,8 @@ INFLATE.tbz2 = $(BZCAT) INFLATE.tgz = $(ZCAT) INFLATE.xz = $(XZCAT) INFLATE.tar = cat +# suitable-extractor(filename): returns extractor based on suffix +suitable-extractor = $(INFLATE$(suffix $(1))) # MESSAGE Macro -- display a message in bold type MESSAGE = echo "$(TERM_BOLD)>>> $($(PKG)_NAME) $($(PKG)_VERSION) $(1)$(TERM_RESET)"package/tar/tar.mk +1
-1@@ -23,7 +23,7 @@ HOST_TAR_SOURCE = tar-$(TAR_VERSION).cpi define HOST_TAR_EXTRACT_CMDS mkdir -p $(@D) cd $(@D) && \ - $(INFLATE.gz) $(DL_DIR)/$(HOST_TAR_SOURCE) | cpio -i + $(call suitable-extractor,$(HOST_TAR_SOURCE)) $(DL_DIR)/$(HOST_TAR_SOURCE) | cpio -i mv $(@D)/tar-$(TAR_VERSION)/* $(@D) rmdir $(@D)/tar-$(TAR_VERSION) endeftoolchain/toolchain-external/ext-tool.mk +2
-2@@ -337,9 +337,9 @@ ifeq ($(BR2_TOOLCHAIN_EXTERNAL_BLACKFIN_ $(TOOLCHAIN_EXTERNAL_DIR)/.extracted: $(DL_DIR)/$(TOOLCHAIN_EXTERNAL_SOURCE_1) $(DL_DIR)/$(TOOLCHAIN_EXTERNAL_SOURCE_2) mkdir -p $(@D) - $(INFLATE$(suffix $(TOOLCHAIN_EXTERNAL_SOURCE_1))) $(DL_DIR)/$(TOOLCHAIN_EXTERNAL_SOURCE_1) | \ + $(call suitable-extractor,$(TOOLCHAIN_EXTERNAL_SOURCE_1)) $(DL_DIR)/$(TOOLCHAIN_EXTERNAL_SOURCE_1) | \ $(TAR) $(TAR_STRIP_COMPONENTS)=3 --hard-dereference -C $(@D) $(TAR_OPTIONS) - - $(INFLATE$(suffix $(TOOLCHAIN_EXTERNAL_SOURCE_2))) $(DL_DIR)/$(TOOLCHAIN_EXTERNAL_SOURCE_2) | \ + $(call suitable-extractor,$(TOOLCHAIN_EXTERNAL_SOURCE_2)) $(DL_DIR)/$(TOOLCHAIN_EXTERNAL_SOURCE_2) | \ $(TAR) $(TAR_STRIP_COMPONENTS)=3 --hard-dereference -C $(@D) $(TAR_OPTIONS) - $(Q)touch $@ else
toolchain/toolchain-external/ext-tool.mk else · line 355
Quoted 8 lines
toolchain/toolchain-external/ext-tool.mk +1
-1@@ -349,7 +349,7 @@ else $(TOOLCHAIN_EXTERNAL_DIR)/.extracted: $(DL_DIR)/$(TOOLCHAIN_EXTERNAL_SOURCE) mkdir -p $(@D) - $(INFLATE$(suffix $(TOOLCHAIN_EXTERNAL_SOURCE))) $^ | \ + $(call suitable-extractor,$(TOOLCHAIN_EXTERNAL_SOURCE)) $^ | \ $(TAR) $(TAR_STRIP_COMPONENTS)=1 --exclude='usr/lib/locale/*' -C $(@D) $(TAR_OPTIONS) - $(TOOLCHAIN_EXTERNAL_FIXUP_CMDS) $(Q)touch $@
As the next branch has been opened now, could we apply this series now?
PATCH v3 1/1 infra: introduce suitable-extractor helper function
Quoted 105 lines · Thomas De Schampheleire
Quoted 103 lines · Thomas De Schampheleire
In order to simplify determining the right extractor tool for a given file type, this patch introduces a make function 'suitable-extractor'. Its usage is $(call suitable-extractor,filename), and it returns the path to the suitable extractor.
Signed-off-by Thomas De Schampheleire <[email protected]>---
v3: Correct $(firstword) usage
(v2): new patch in seriesNote: one remaining direct usage of the INFLATE variables is in
package/pkg-generic.mk, but it's in the inner block where dollar signs are thrown around your ears. I'm not sure if it's possible to use the function there...package/lsof/lsof.mk | 2 +- package/perl/perl.mk | 2 +- package/pkg-generic.mk | 4 +++- package/pkg-utils.mk | 2 ++ package/tar/tar.mk | 2 +- toolchain/toolchain-external/ext-tool.mk | 6 +++--- 6 files changed, 11 insertions(+), 7 deletions(-)package/lsof/lsof.mk +1
-1@@ -41,7 +41,7 @@ endif # The .tar.bz2 contains another .tar, which contains the source code. define LSOF_EXTRACT_CMDS - $(INFLATE.bz2) $(DL_DIR)/$(LSOF_SOURCE) | \ + $(call suitable-extractor,$(LSOF_SOURCE)) $(DL_DIR)/$(LSOF_SOURCE) | \ $(TAR) -O $(TAR_OPTIONS) - lsof_$(LSOF_VERSION)/lsof_$(LSOF_VERSION)_src.tar | \ $(TAR) $(TAR_STRIP_COMPONENTS)=1 -C $(LSOF_DIR) $(TAR_OPTIONS) - endefpackage/perl/perl.mk +1
-1@@ -30,7 +30,7 @@ endef PERL_POST_DOWNLOAD_HOOKS += PERL_CROSS_DOWNLOAD define PERL_CROSS_EXTRACT - $(INFLATE$(suffix $(PERL_CROSS_SOURCE))) $(DL_DIR)/$(PERL_CROSS_SOURCE) | \ + $(call suitable-extractor,$(PERL_CROSS_SOURCE)) $(DL_DIR)/$(PERL_CROSS_SOURCE) | \ $(TAR) $(TAR_STRIP_COMPONENTS)=1 -C $(@D) $(TAR_OPTIONS) - endef PERL_POST_EXTRACT_HOOKS += PERL_CROSS_EXTRACTpackage/pkg-generic.mk +3
-1@@ -539,7 +539,9 @@ else ifeq ($$($(2)_SITE_METHOD),hg) DL_TOOLS_DEPENDENCIES += hg endif # SITE_METHOD -DL_TOOLS_DEPENDENCIES += $(firstword $(INFLATE$(suffix $($(2)_SOURCE)))) +# $(firstword) is used here because the extractor can have arguments, like +# ZCAT="gzip -d -c", and to check for the dependency we only want 'gzip'. +DL_TOOLS_DEPENDENCIES += $(firstword $(call suitable-extractor,$($(2)_SOURCE))) endif # $(2)_KCONFIG_VAR endef # inner-generic-packagepackage/pkg-utils.mk +2
-0@@ -62,6 +62,8 @@ INFLATE.tbz2 = $(BZCAT) INFLATE.tgz = $(ZCAT) INFLATE.xz = $(XZCAT) INFLATE.tar = cat +# suitable-extractor(filename): returns extractor based on suffix +suitable-extractor = $(INFLATE$(suffix $(1))) # MESSAGE Macro -- display a message in bold type MESSAGE = echo "$(TERM_BOLD)>>> $($(PKG)_NAME) $($(PKG)_VERSION) $(1)$(TERM_RESET)"package/tar/tar.mk +1
-1@@ -23,7 +23,7 @@ HOST_TAR_SOURCE = tar-$(TAR_VERSION).cpi define HOST_TAR_EXTRACT_CMDS mkdir -p $(@D) cd $(@D) && \ - $(INFLATE.gz) $(DL_DIR)/$(HOST_TAR_SOURCE) | cpio -i + $(call suitable-extractor,$(HOST_TAR_SOURCE)) $(DL_DIR)/$(HOST_TAR_SOURCE) | cpio -i mv $(@D)/tar-$(TAR_VERSION)/* $(@D) rmdir $(@D)/tar-$(TAR_VERSION) endeftoolchain/toolchain-external/ext-tool.mk +2
-2@@ -337,9 +337,9 @@ ifeq ($(BR2_TOOLCHAIN_EXTERNAL_BLACKFIN_ $(TOOLCHAIN_EXTERNAL_DIR)/.extracted: $(DL_DIR)/$(TOOLCHAIN_EXTERNAL_SOURCE_1) $(DL_DIR)/$(TOOLCHAIN_EXTERNAL_SOURCE_2) mkdir -p $(@D) - $(INFLATE$(suffix $(TOOLCHAIN_EXTERNAL_SOURCE_1))) $(DL_DIR)/$(TOOLCHAIN_EXTERNAL_SOURCE_1) | \ + $(call suitable-extractor,$(TOOLCHAIN_EXTERNAL_SOURCE_1)) $(DL_DIR)/$(TOOLCHAIN_EXTERNAL_SOURCE_1) | \ $(TAR) $(TAR_STRIP_COMPONENTS)=3 --hard-dereference -C $(@D) $(TAR_OPTIONS) - - $(INFLATE$(suffix $(TOOLCHAIN_EXTERNAL_SOURCE_2))) $(DL_DIR)/$(TOOLCHAIN_EXTERNAL_SOURCE_2) | \ + $(call suitable-extractor,$(TOOLCHAIN_EXTERNAL_SOURCE_2)) $(DL_DIR)/$(TOOLCHAIN_EXTERNAL_SOURCE_2) | \ $(TAR) $(TAR_STRIP_COMPONENTS)=3 --hard-dereference -C $(@D) $(TAR_OPTIONS) - $(Q)touch $@ else@@ -349,7 +349,7 @@ else
Quoted 9 lines
toolchain/toolchain-external/ext-tool.mk else · line 355
Quoted 7 lines
toolchain/toolchain-external/ext-tool.mk +1
-1$(TOOLCHAIN_EXTERNAL_DIR)/.extracted: $(DL_DIR)/$(TOOLCHAIN_EXTERNAL_SOURCE) mkdir -p $(@D) - $(INFLATE$(suffix $(TOOLCHAIN_EXTERNAL_SOURCE))) $^ | \ + $(call suitable-extractor,$(TOOLCHAIN_EXTERNAL_SOURCE)) $^ | \ $(TAR) $(TAR_STRIP_COMPONENTS)=1 --exclude='usr/lib/locale/*' -C $(@D) $(TAR_OPTIONS) - $(TOOLCHAIN_EXTERNAL_FIXUP_CMDS) $(Q)touch $@As the next branch has been opened now, could we apply this series now?
ping?
PATCH v3 1/1 infra: introduce suitable-extractor helper function
In reply to Thomas De SchampheleireQuoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
"Thomas" == Thomas De Schampheleire <[email protected]> writes:
Thomas> In order to simplify determining the right extractor tool for a given Thomas> file type, this patch introduces a make function 'suitable-extractor'. Thomas> Its usage is $(call suitable-extractor,filename), and it returns the Thomas> path to the suitable extractor.
Thomas> Signed-off-by: Thomas De Schampheleire <[email protected]>
Thomas> ---
Thomas> v3: Correct $(firstword) usage
Thomas> (v2): new patch in series
Committed, thanks.
Signature · 1 line
Bye, Peter Korsgaard
PATCH v3 1/1 xzcat: treat as host prerequisite and build if needed
In reply to Thomas De SchampheleireIf xzcat is not present on the host system, buildroot bails out early asking the developer to install it (xzcat is now a DL_TOOLS_DEPENDENCY) Conversely, when BR2_TARGET_ROOTFS_CPIO_XZ is enabled, then host-xz is a build dependency, and no manual action is required from the developer.
Because the second approach is nicer, also build host-xz when xzcat is not available, using the host-prerequisite and suitable-host-pkg mechanisms, already used for tar.
---
v3: update after $(firstword) change
v2: avoid filtering out xzcat from DL_TOOLS_DEPENDENCIES (comment ThomasP)
package/pkg-generic.mk | 4 ++++ support/dependencies/check-host-xzcat.mk | 7 +++++++ support/dependencies/check-host-xzcat.sh | 14 ++++++++++++++ 3 files changed, 25 insertions(+), 0 deletions(-)
package/pkg-generic.mk +4 -0
@@ -541,7 +541,11 @@ endif # SITE_METHOD # $(firstword) is used here because the extractor can have arguments, like # ZCAT="gzip -d -c", and to check for the dependency we only want 'gzip'. +# Do not add xzcat to the list of required dependencies, as it gets built +# automatically if it isn't found. +ifneq ($(call suitable-extractor,$($(2)_SOURCE)),$(XZCAT)) DL_TOOLS_DEPENDENCIES += $(firstword $(call suitable-extractor,$($(2)_SOURCE))) +endif endif # $(2)_KCONFIG_VAR endef # inner-generic-package
support/dependencies/check-host-xzcat.mk new file +7 -0
@@ -0,0 +1,7 @@ +# XZCAT is taken from BR2_XZCAT (defaults to 'xzcat') (see Makefile) +# If it is not present, build our own host-xzcat + +ifeq (,$(call suitable-host-package,xzcat,$(XZCAT))) + DEPENDENCIES_HOST_PREREQ += host-xz + XZCAT = $(HOST_DIR)/usr/bin/xzcat +endif
support/dependencies/check-host-xzcat.sh new file +14 -0
@@ -0,0 +1,14 @@ +#!/bin/sh + +candidate="$1" + +xzcat=`which $candidate 2>/dev/null` +if [ ! -x "$xzcat" ]; then + xzcat=`which xzcat 2>/dev/null` + if [ ! -x "$xzcat" ]; then + # echo nothing: no suitable xzcat found + exit 1 + fi +fi + +echo $xzcat
PATCH v3 1/1 xzcat: treat as host prerequisite and build if needed
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
"Thomas" == Thomas De Schampheleire <[email protected]> writes:
Thomas> If xzcat is not present on the host system, buildroot bails out early asking Thomas> the developer to install it (xzcat is now a DL_TOOLS_DEPENDENCY) Thomas> Conversely, when BR2_TARGET_ROOTFS_CPIO_XZ is enabled, then host-xz is a Thomas> build dependency, and no manual action is required from the developer.
Thomas> Because the second approach is nicer, also build host-xz when xzcat is not Thomas> available, using the host-prerequisite and suitable-host-pkg mechanisms, Thomas> already used for tar.
Thomas> Signed-off-by: Thomas De Schampheleire <[email protected]>
Thomas> ---
Thomas> v3: update after $(firstword) change
Thomas> v2: avoid filtering out xzcat from DL_TOOLS_DEPENDENCIES (comment ThomasP)
Committed, thanks.
Signature · 1 line
Bye, Peter Korsgaard