Skip to contentSkip to conversation

core: add generic support for lz archives

PATCH New Baruch Siach 3 participants 11 messages Feb 10, 2017 – Feb 12, 2017 1 message not in the archive Kbuild

lore
Share & tools
lore
#0 lore raw

PATCH 1/4 core: add generic support for lz archives

This commit teaches the generic code how to extract .tar.lz archives. When lzip is not installed on the host, host-lzip gets built automatically.

To avoid the dependency checker complain about missing host lzip (in addition to xzcat) add a new host-extractor function that only returns extractors we expect to be host installed. Use host-extractor output to feed the dependency checker.

Cc Thomas De Schampheleire <[email protected]>
Cc Peter Seiderer <[email protected]>

---
v2: add host-extractor to fix host lzip check

---
 Config.in                               |  7 +++++++
 Makefile                                |  1 +
 package/pkg-generic.mk                  |  6 +-----
 package/pkg-utils.mk                    |  5 +++++
 support/dependencies/check-host-lzip.mk |  4 ++++
 support/dependencies/check-host-lzip.sh | 14 ++++++++++++++
 6 files changed, 32 insertions(+), 5 deletions(-)
 create mode 100644 support/dependencies/check-host-lzip.mk
 create mode 100755 support/dependencies/check-host-lzip.sh
Config.in +7 -0
index ccd777e8bb00..bd8f0d1a10ee 100644
@@ -158,6 +158,13 @@ config BR2_XZCAT
 	  Command to be used to extract a xz'ed file to stdout.
 	  Default is "xzcat"
 
+config BR2_LZCAT
+	string "lzcat command"
+	default "lzip -d -c"
+	help
+	  Command to be used to extract a lzip'ed file to stdout.
+	  Default is "lzip -d -c"
+
 config BR2_TAR_OPTIONS
 	string "Tar options"
 	default ""
Makefile +1 -0
index df3b64eb03ec..b4550e098958 100644
@@ -431,6 +431,7 @@ KERNEL_ARCH := $(shell echo "$(ARCH)" | sed -e "s/-.*//" \
 ZCAT := $(call qstrip,$(BR2_ZCAT))
 BZCAT := $(call qstrip,$(BR2_BZCAT))
 XZCAT := $(call qstrip,$(BR2_XZCAT))
+LZCAT := $(call qstrip,$(BR2_LZCAT))
 TAR_OPTIONS = $(call qstrip,$(BR2_TAR_OPTIONS)) -xf
 
 # packages compiled for the host go here
package/pkg-generic.mk +1 -5
index 3ca71b03b9df..b45d1109ca4f 100644
@@ -928,11 +928,7 @@ endif # SITE_METHOD
 
 # $(firstword) is used here because the extractor can have arguments, like
 # ZCAT="gzip -d -c", and to check for the dependency we only want 'gzip'.
-# Do not add xzcat to the list of required dependencies, as it gets built
-# automatically if it isn't found.
-ifneq ($$(call suitable-extractor,$$($(2)_SOURCE)),$$(XZCAT))
-DL_TOOLS_DEPENDENCIES += $$(firstword $$(call suitable-extractor,$$($(2)_SOURCE)))
-endif
+DL_TOOLS_DEPENDENCIES += $$(firstword $$(call host-extractor,$$($(2)_SOURCE)))
 
 # Ensure all virtual targets are PHONY. Listed alphabetically.
 .PHONY:	$(1) \
package/pkg-utils.mk +5 -0
index c5d4080c72f4..1c091a43d36c 100644
@@ -36,6 +36,7 @@ pkgname = $(lastword $(subst /, ,$(pkgdir)))
 # Define extractors for different archive suffixes
 INFLATE.bz2  = $(BZCAT)
 INFLATE.gz   = $(ZCAT)
+INFLATE.lz   = $(LZCAT)
 INFLATE.lzma = $(XZCAT)
 INFLATE.tbz  = $(BZCAT)
 INFLATE.tbz2 = $(BZCAT)
@@ -45,6 +46,10 @@ INFLATE.tar  = cat
 # suitable-extractor(filename): returns extractor based on suffix
 suitable-extractor = $(INFLATE$(suffix $(1)))
 
+# host-extractor(filename): same as suitable-extractor, but filter out
+# extractors we build when the host lacks one.
+host-extractor = $(INFLATE$(filter-out .lz .lzma .xz,$(suffix $(1))))
+
 # check-deprecated-variable -- throw an error on deprecated variables
 # example:
 #   $(eval $(call check-deprecated-variable,FOO_MAKE_OPT,FOO_MAKE_OPTS))
support/dependencies/check-host-lzip.mk new file +4 -0
new file mode 100644
index 000000000000..32ab9f4daffe
@@ -0,0 +1,4 @@
+ifeq (,$(call suitable-host-package,lzip,$(LZCAT)))
+DEPENDENCIES_HOST_PREREQ += host-lzip
+LZCAT = $(HOST_DIR)/usr/bin/lzip -d -c
+endif
support/dependencies/check-host-lzip.sh new file +14 -0
new file mode 100755
index 000000000000..4f8a2ba3de5b
@@ -0,0 +1,14 @@
+#!/bin/sh
+
+candidate="$1"
+
+lzip=`which $candidate 2>/dev/null`
+if [ ! -x "$lzip" ]; then
+	lzip=`which lzip 2>/dev/null`
+	if [ ! -x "$lzip" ]; then
+		# echo nothing: no suitable lzip found
+		exit 1
+	fi
+fi
+
+echo $lzip
Signature · 1 line
2.11.0
#1 lore raw

PATCH 2/4 ed: use generic extract command

The generic code now knows how to extract .tar.lz archives. Remove the local extract code.

---
 package/ed/ed.mk | 6 ------
 1 file changed, 6 deletions(-)
package/ed/ed.mk +0 -6
index 50adeb4ec5b4..6b9f65ec5280 100644
@@ -10,15 +10,9 @@ ED_SOURCE = ed-$(ED_VERSION).tar.lz
 ED_CONF_OPTS = \
 	CC="$(TARGET_CC)" CFLAGS="$(TARGET_CFLAGS)" \
 	LDFLAGS="$(TARGET_LDFLAGS)"
-ED_DEPENDENCIES = host-lzip
 ED_LICENSE = GPLv3+
 ED_LICENSE_FILES = COPYING
 
-define ED_EXTRACT_CMDS
-	$(HOST_DIR)/usr/bin/lzip -d -c $(DL_DIR)/$(ED_SOURCE) | \
-		tar --strip-components=1 -C $(@D) $(TAR_OPTIONS) -
-endef
-
 define ED_CONFIGURE_CMDS
 	(cd $(@D); \
 		$(TARGET_MAKE_ENV) ./configure \
Signature · 1 line
2.11.0
#2 lore raw

PATCH 2/4 ed: use generic extract command

Quoted 18 lines · Baruch Siach

The generic code now knows how to extract .tar.lz archives. Remove the local extract code.

---
 package/ed/ed.mk | 6 ------
 1 file changed, 6 deletions(-)
package/ed/ed.mk +0 -1
index 50adeb4ec5b4..6b9f65ec5280 100644
@@ -10,15 +10,9 @@ ED_SOURCE = ed-$(ED_VERSION).tar.lz
 ED_CONF_OPTS = \
 	CC="$(TARGET_CC)" CFLAGS="$(TARGET_CFLAGS)" \
 	LDFLAGS="$(TARGET_LDFLAGS)"
-ED_DEPENDENCIES = host-lzip
 ED_LICENSE = GPLv3+

package/ed/ed.mk ED_SOURCE = ed-$(ED_VERSION).tar.lz · line 18

Quoted 10 lines
package/ed/ed.mk +0 -5
 ED_LICENSE_FILES = COPYING
 
-define ED_EXTRACT_CMDS
-	$(HOST_DIR)/usr/bin/lzip -d -c $(DL_DIR)/$(ED_SOURCE) | \
-		tar --strip-components=1 -C $(@D) $(TAR_OPTIONS) -
-endef
-
 define ED_CONFIGURE_CMDS
 	(cd $(@D); \
 		$(TARGET_MAKE_ENV) ./configure \
#3 lore raw

PATCH 3/4 ddrescue: use generic extract command

In reply to Baruch Siach

The generic code now knows how to extract .tar.lz archives. Remove the local extract code

---
 package/ddrescue/ddrescue.mk | 6 ------
 1 file changed, 6 deletions(-)
package/ddrescue/ddrescue.mk +0 -6
index 145d0520d66f..4e244e6bdaef 100644
@@ -9,12 +9,6 @@ DDRESCUE_SOURCE = ddrescue-$(DDRESCUE_VERSION).tar.lz
 DDRESCUE_SITE = http://download.savannah.gnu.org/releases/ddrescue
 DDRESCUE_LICENSE = GPLv2+
 DDRESCUE_LICENSE_FILES = COPYING
-DDRESCUE_DEPENDENCIES = host-lzip
-
-define DDRESCUE_EXTRACT_CMDS
-	$(HOST_DIR)/usr/bin/lzip -d -c $(DL_DIR)/$(DDRESCUE_SOURCE) | \
-		tar --strip-components=1 -C $(@D) $(TAR_OPTIONS) -
-endef
 
 define DDRESCUE_CONFIGURE_CMDS
 	(cd $(@D); \
Signature · 1 line
2.11.0
#4 lore raw

PATCH 3/4 ddrescue: use generic extract command

Quoted 15 lines · Baruch Siach

The generic code now knows how to extract .tar.lz archives. Remove the local extract code

---
 package/ddrescue/ddrescue.mk | 6 ------
 1 file changed, 6 deletions(-)
package/ddrescue/ddrescue.mk +0 -0
index 145d0520d66f..4e244e6bdaef 100644
@@ -9,12 +9,6 @@ DDRESCUE_SOURCE = ddrescue-$(DDRESCUE_VERSION).tar.lz
 DDRESCUE_SITE = http://download.savannah.gnu.org/releases/ddrescue
 DDRESCUE_LICENSE = GPLv2+

package/ddrescue/ddrescue.mk DDRESCUE_SOURCE = ddrescue-$(DDRESCUE_VERSION).tar.lz · line 14

Quoted 10 lines
package/ddrescue/ddrescue.mk +0 -6
 DDRESCUE_LICENSE_FILES = COPYING
-DDRESCUE_DEPENDENCIES = host-lzip
-
-define DDRESCUE_EXTRACT_CMDS
-	$(HOST_DIR)/usr/bin/lzip -d -c $(DL_DIR)/$(DDRESCUE_SOURCE) | \
-		tar --strip-components=1 -C $(@D) $(TAR_OPTIONS) -
-endef
 
 define DDRESCUE_CONFIGURE_CMDS
 	(cd $(@D); \
#5 lore raw

PATCH 4/4 ocrad: use generic extract command

In reply to Baruch Siach

The generic code now knows how to extract .tar.lz archives. Remove the local extract code

---
 package/ocrad/ocrad.mk | 6 ------
 1 file changed, 6 deletions(-)
package/ocrad/ocrad.mk +0 -6
index d19e83ef4279..54b607c79193 100644
@@ -10,12 +10,6 @@ OCRAD_SITE = $(BR2_GNU_MIRROR)/ocrad
 OCRAD_LICENSE = GPLv3+
 OCRAD_LICENSE_FILES = COPYING
 OCRAD_INSTALL_STAGING = YES
-OCRAD_DEPENDENCIES = host-lzip
-
-define OCRAD_EXTRACT_CMDS
-	$(HOST_DIR)/usr/bin/lzip -d -c $(DL_DIR)/$(OCRAD_SOURCE) | \
-		tar --strip-components=1 -C $(@D) $(TAR_OPTIONS) -
-endef
 
 # This is not a true autotools package.
 define OCRAD_CONFIGURE_CMDS
Signature · 1 line
2.11.0
#6 lore raw

PATCH 4/4 ocrad: use generic extract command

Quoted 15 lines · Baruch Siach

The generic code now knows how to extract .tar.lz archives. Remove the local extract code

---
 package/ocrad/ocrad.mk | 6 ------
 1 file changed, 6 deletions(-)
package/ocrad/ocrad.mk +0 -0
index d19e83ef4279..54b607c79193 100644
@@ -10,12 +10,6 @@ OCRAD_SITE = $(BR2_GNU_MIRROR)/ocrad
 OCRAD_LICENSE = GPLv3+
 OCRAD_LICENSE_FILES = COPYING

package/ocrad/ocrad.mk OCRAD_SITE = $(BR2_GNU_MIRROR)/ocrad · line 15

Quoted 10 lines
package/ocrad/ocrad.mk +0 -6
 OCRAD_INSTALL_STAGING = YES
-OCRAD_DEPENDENCIES = host-lzip
-
-define OCRAD_EXTRACT_CMDS
-	$(HOST_DIR)/usr/bin/lzip -d -c $(DL_DIR)/$(OCRAD_SOURCE) | \
-		tar --strip-components=1 -C $(@D) $(TAR_OPTIONS) -
-endef
 
 # This is not a true autotools package.
 define OCRAD_CONFIGURE_CMDS
#7 lore raw

PATCH 1/4 core: add generic support for lz archives

In reply to Baruch Siach

Hello Baruch,

Quoted 114 lines · Baruch Siach

This commit teaches the generic code how to extract .tar.lz archives. When lzip is not installed on the host, host-lzip gets built automatically.

To avoid the dependency checker complain about missing host lzip (in addition to xzcat) add a new host-extractor function that only returns extractors we expect to be host installed. Use host-extractor output to feed the dependency checker.

Cc Thomas De Schampheleire <[email protected]>
Cc Peter Seiderer <[email protected]>

---
v2: add host-extractor to fix host lzip check

---
 Config.in                               |  7 +++++++
 Makefile                                |  1 +
 package/pkg-generic.mk                  |  6 +-----
 package/pkg-utils.mk                    |  5 +++++
 support/dependencies/check-host-lzip.mk |  4 ++++
 support/dependencies/check-host-lzip.sh | 14 ++++++++++++++
 6 files changed, 32 insertions(+), 5 deletions(-)
 create mode 100644 support/dependencies/check-host-lzip.mk
 create mode 100755 support/dependencies/check-host-lzip.sh
Config.in +7 -0
index ccd777e8bb00..bd8f0d1a10ee 100644
@@ -158,6 +158,13 @@ config BR2_XZCAT
 	  Command to be used to extract a xz'ed file to stdout.
 	  Default is "xzcat"
 
+config BR2_LZCAT
+	string "lzcat command"
+	default "lzip -d -c"
+	help
+	  Command to be used to extract a lzip'ed file to stdout.
+	  Default is "lzip -d -c"
+
 config BR2_TAR_OPTIONS
 	string "Tar options"
 	default ""
Makefile +1 -0
index df3b64eb03ec..b4550e098958 100644
@@ -431,6 +431,7 @@ KERNEL_ARCH := $(shell echo "$(ARCH)" | sed -e "s/-.*//" \
 ZCAT := $(call qstrip,$(BR2_ZCAT))
 BZCAT := $(call qstrip,$(BR2_BZCAT))
 XZCAT := $(call qstrip,$(BR2_XZCAT))
+LZCAT := $(call qstrip,$(BR2_LZCAT))
 TAR_OPTIONS = $(call qstrip,$(BR2_TAR_OPTIONS)) -xf
 
 # packages compiled for the host go here
package/pkg-generic.mk +1 -5
index 3ca71b03b9df..b45d1109ca4f 100644
@@ -928,11 +928,7 @@ endif # SITE_METHOD
 
 # $(firstword) is used here because the extractor can have arguments, like
 # ZCAT="gzip -d -c", and to check for the dependency we only want 'gzip'.
-# Do not add xzcat to the list of required dependencies, as it gets built
-# automatically if it isn't found.
-ifneq ($$(call suitable-extractor,$$($(2)_SOURCE)),$$(XZCAT))
-DL_TOOLS_DEPENDENCIES += $$(firstword $$(call suitable-extractor,$$($(2)_SOURCE)))
-endif
+DL_TOOLS_DEPENDENCIES += $$(firstword $$(call host-extractor,$$($(2)_SOURCE)))
 
 # Ensure all virtual targets are PHONY. Listed alphabetically.
 .PHONY:	$(1) \
package/pkg-utils.mk +5 -0
index c5d4080c72f4..1c091a43d36c 100644
@@ -36,6 +36,7 @@ pkgname = $(lastword $(subst /, ,$(pkgdir)))
 # Define extractors for different archive suffixes
 INFLATE.bz2  = $(BZCAT)
 INFLATE.gz   = $(ZCAT)
+INFLATE.lz   = $(LZCAT)
 INFLATE.lzma = $(XZCAT)
 INFLATE.tbz  = $(BZCAT)
 INFLATE.tbz2 = $(BZCAT)
@@ -45,6 +46,10 @@ INFLATE.tar  = cat
 # suitable-extractor(filename): returns extractor based on suffix
 suitable-extractor = $(INFLATE$(suffix $(1)))
 
+# host-extractor(filename): same as suitable-extractor, but filter out
+# extractors we build when the host lacks one.
+host-extractor = $(INFLATE$(filter-out .lz .lzma .xz,$(suffix $(1))))
+
 # check-deprecated-variable -- throw an error on deprecated variables
 # example:
 #   $(eval $(call check-deprecated-variable,FOO_MAKE_OPT,FOO_MAKE_OPTS))
support/dependencies/check-host-lzip.mk new file +4 -0
new file mode 100644
index 000000000000..32ab9f4daffe
@@ -0,0 +1,4 @@
+ifeq (,$(call suitable-host-package,lzip,$(LZCAT)))
+DEPENDENCIES_HOST_PREREQ += host-lzip
+LZCAT = $(HOST_DIR)/usr/bin/lzip -d -c
+endif
support/dependencies/check-host-lzip.sh new file +4 -0
new file mode 100755
index 000000000000..4f8a2ba3de5b
@@ -0,0 +1,14 @@
+#!/bin/sh
+
+candidate="$1"
+

support/dependencies/check-host-lzip.sh line 14

Quoted 10 lines
support/dependencies/check-host-lzip.sh new file +10 -0
+lzip=`which $candidate 2>/dev/null`
+if [ ! -x "$lzip" ]; then
+	lzip=`which lzip 2>/dev/null`
+	if [ ! -x "$lzip" ]; then
+		# echo nothing: no suitable lzip found
+		exit 1
+	fi
+fi
+
+echo $lzip

Works for my testcase/ddrescue, you can add my

Regards,
Peter

#8 lore raw

PATCH 1/4 core: add generic support for lz archives

In reply to Baruch Siach

Hi Baruch,

Quoted 84 lines · Baruch Siach

This commit teaches the generic code how to extract .tar.lz archives. When lzip is not installed on the host, host-lzip gets built automatically.

To avoid the dependency checker complain about missing host lzip (in addition to xzcat) add a new host-extractor function that only returns extractors we expect to be host installed. Use host-extractor output to feed the dependency checker.

Cc Thomas De Schampheleire <[email protected]>
Cc Peter Seiderer <[email protected]>

---
v2: add host-extractor to fix host lzip check

---
 Config.in                               |  7 +++++++
 Makefile                                |  1 +
 package/pkg-generic.mk                  |  6 +-----
 package/pkg-utils.mk                    |  5 +++++
 support/dependencies/check-host-lzip.mk |  4 ++++
 support/dependencies/check-host-lzip.sh | 14 ++++++++++++++
 6 files changed, 32 insertions(+), 5 deletions(-)
 create mode 100644 support/dependencies/check-host-lzip.mk
 create mode 100755 support/dependencies/check-host-lzip.sh
Config.in +7 -0
index ccd777e8bb00..bd8f0d1a10ee 100644
@@ -158,6 +158,13 @@ config BR2_XZCAT
          Command to be used to extract a xz'ed file to stdout.
          Default is "xzcat"
 
+config BR2_LZCAT
+       string "lzcat command"
+       default "lzip -d -c"
+       help
+         Command to be used to extract a lzip'ed file to stdout.
+         Default is "lzip -d -c"
+
 config BR2_TAR_OPTIONS
        string "Tar options"
        default ""
Makefile +1 -0
index df3b64eb03ec..b4550e098958 100644
@@ -431,6 +431,7 @@ KERNEL_ARCH := $(shell echo "$(ARCH)" | sed -e "s/-.*//" \
 ZCAT := $(call qstrip,$(BR2_ZCAT))
 BZCAT := $(call qstrip,$(BR2_BZCAT))
 XZCAT := $(call qstrip,$(BR2_XZCAT))
+LZCAT := $(call qstrip,$(BR2_LZCAT))
 TAR_OPTIONS = $(call qstrip,$(BR2_TAR_OPTIONS)) -xf
 
 # packages compiled for the host go here
package/pkg-generic.mk +1 -5
index 3ca71b03b9df..b45d1109ca4f 100644
@@ -928,11 +928,7 @@ endif # SITE_METHOD
 
 # $(firstword) is used here because the extractor can have arguments, like
 # ZCAT="gzip -d -c", and to check for the dependency we only want 'gzip'.
-# Do not add xzcat to the list of required dependencies, as it gets built
-# automatically if it isn't found.
-ifneq ($$(call suitable-extractor,$$($(2)_SOURCE)),$$(XZCAT))
-DL_TOOLS_DEPENDENCIES += $$(firstword $$(call suitable-extractor,$$($(2)_SOURCE)))
-endif
+DL_TOOLS_DEPENDENCIES += $$(firstword $$(call host-extractor,$$($(2)_SOURCE)))
 
 # Ensure all virtual targets are PHONY. Listed alphabetically.
 .PHONY:        $(1) \
package/pkg-utils.mk +1 -0
index c5d4080c72f4..1c091a43d36c 100644
@@ -36,6 +36,7 @@ pkgname = $(lastword $(subst /, ,$(pkgdir)))
 # Define extractors for different archive suffixes
 INFLATE.bz2  = $(BZCAT)
 INFLATE.gz   = $(ZCAT)
+INFLATE.lz   = $(LZCAT)
 INFLATE.lzma = $(XZCAT)
 INFLATE.tbz  = $(BZCAT)
 INFLATE.tbz2 = $(BZCAT)

package/pkg-utils.mk INFLATE.tar = cat · line 51

Quoted 6 lines
package/pkg-utils.mk +3 -0
@@ -45,6 +46,10 @@ INFLATE.tar  = cat
 # suitable-extractor(filename): returns extractor based on suffix
 suitable-extractor = $(INFLATE$(suffix $(1)))
 
+# host-extractor(filename): same as suitable-extractor, but filter out
+# extractors we build when the host lacks one.
+host-extractor = $(INFLATE$(filter-out .lz .lzma .xz,$(suffix $(1))))

suitable-extractor and host-extractor now render different output for a given file which makes no sense in general for functions named like this. You only need the different output to decide whether extra dependency checks need to be done, not to determine what the extractor is.

Also, the list of extensions is kind of 'magic' and not immediately linked with the changes in support/dependencies (this was already true with the original situation, true). It would be great if the exceptions could be steered from the check-host-foo.mk files directly, which is the conceptually right place: when that file exists it will take over all checking, if it does not then the checking is done in the standard DL_TOOLS_DEPENDENCIES.

I thought hard on a way to attack the problem without needing to know the extensions and more in line with the original change, but it is not easy due to the fact that XZCAT and such can contain spaces which does not fit well with make.
For clarity, I wanted to let check-host-xz.mk append to a new variable, e.g.

    DL_TOOLS_DEPENDENCIES_PRECHECKED_VARS += $(XZCAT)
and in pkg-generic.mk use this list to drive the exceptions:
    ifeq (,$$(filter $$(call
suitable-extractor,$$($(2)_SOURCE)),$$(DL_TOOLS_DEPENDENCIES_PRECHECKED_VARS)))
    DL_TOOLS_DEPENDENCIES += $$(firstword $$(call

suitable-extractor,$$($(2)_SOURCE)))
endif
This could work if we first replace all spaces with another character, both when appending to DL_TOOLS_DEPENDENCIES_PRECHECKED_VARS as in the first argument of the filter function.
But perhaps it makes things too complex.

An alternative is to stick closer to your proposal, but with some changes:
- let check-host-foo.mk define the list of extensions that it handles instead of the magic list, e.g. EXTRACTOR_DEPENDENCY_PRECHECKED_EXTENSIONS += .xz
- rename host-extractor to something else, e.g. extractor-dependency
- perhaps it makes sense to let extractor-dependency handle the $(firstword) call

Given my input, what do you think?

Separate note: it could make sense to split this patch in two: one to refactor the handing of the dependencies, and a second one to cover lz7.

Best regards,
Thomas

A message that is not in the archive (sent off-list or not yet received): CAAXf6LXmYoo0mSdf6-uvMQtLwv0nP2D0WO8t7omkyy4CkeiStg@mail.gmail.com

#10 lore raw

PATCH 1/4 core: add generic support for lz archives

On Feb 11, 2017 22:16, "Thomas De Schampheleire" <[email protected]> wrote:

Hi Baruch,

Quoted 4 lines · Baruch Siach

This commit teaches the generic code how to extract .tar.lz archives. When lzip is not installed on the host, host-lzip gets built automatically.

To avoid the dependency checker complain about missing host lzip (in

addition

Quoted 1 line

to xzcat) add a new host-extractor function that only returns extractors

we

Quoted 1 line

expect to be host installed. Use host-extractor output to feed the

dependency

Makefile KERNEL_ARCH := $(shell echo "$(ARCH)" | sed -e

Quoted 41 lines

checker.

Cc Thomas De Schampheleire <[email protected]>
Cc Peter Seiderer <[email protected]>

---
v2: add host-extractor to fix host lzip check

---
 Config.in                               |  7 +++++++
 Makefile                                |  1 +
 package/pkg-generic.mk                  |  6 +-----
 package/pkg-utils.mk                    |  5 +++++
 support/dependencies/check-host-lzip.mk |  4 ++++
 support/dependencies/check-host-lzip.sh | 14 ++++++++++++++
 6 files changed, 32 insertions(+), 5 deletions(-)
 create mode 100644 support/dependencies/check-host-lzip.mk
 create mode 100755 support/dependencies/check-host-lzip.sh
Config.in +7 -0
index ccd777e8bb00..bd8f0d1a10ee 100644
@@ -158,6 +158,13 @@ config BR2_XZCAT
          Command to be used to extract a xz'ed file to stdout.
          Default is "xzcat"
 
+config BR2_LZCAT
+       string "lzcat command"
+       default "lzip -d -c"
+       help
+         Command to be used to extract a lzip'ed file to stdout.
+         Default is "lzip -d -c"
+
 config BR2_TAR_OPTIONS
        string "Tar options"
        default ""
Makefile +0 -0
index df3b64eb03ec..b4550e098958 100644
@@ -431,6 +431,7 @@ KERNEL_ARCH := $(shell echo "$(ARCH)" | sed -e

"s/-.*//" \

Quoted 12 lines
Makefile +1 -0
 ZCAT := $(call qstrip,$(BR2_ZCAT))
 BZCAT := $(call qstrip,$(BR2_BZCAT))
 XZCAT := $(call qstrip,$(BR2_XZCAT))
+LZCAT := $(call qstrip,$(BR2_LZCAT))
 TAR_OPTIONS = $(call qstrip,$(BR2_TAR_OPTIONS)) -xf
 
 # packages compiled for the host go here
package/pkg-generic.mk +0 -0
index 3ca71b03b9df..b45d1109ca4f 100644

package/pkg-generic.mk endif # SITE_METHOD · line 929

Quoted 2 lines
package/pkg-generic.mk +0 -0
@@ -928,11 +928,7 @@ endif # SITE_METHOD
 
 # $(firstword) is used here because the extractor can have arguments,

like

package/pkg-generic.mk endif # SITE_METHOD

Quoted 5 lines
package/pkg-generic.mk +0 -4
 # ZCAT="gzip -d -c", and to check for the dependency we only want 'gzip'.
-# Do not add xzcat to the list of required dependencies, as it gets built
-# automatically if it isn't found.
-ifneq ($$(call suitable-extractor,$$($(2)_SOURCE)),$$(XZCAT))
-DL_TOOLS_DEPENDENCIES += $$(firstword $$(call suitable-extractor,$$($(2)_

SOURCE)))

package/pkg-generic.mk endif # SITE_METHOD · line 931

Quoted 2 lines
package/pkg-generic.mk +1 -1
-endif
+DL_TOOLS_DEPENDENCIES += $$(firstword $$(call

host-extractor,$$($(2)_SOURCE)))

Quoted 16 lines
package/pkg-generic.mk +0 -0
 # Ensure all virtual targets are PHONY. Listed alphabetically.
 .PHONY:        $(1) \
package/pkg-utils.mk +1 -0
index c5d4080c72f4..1c091a43d36c 100644
@@ -36,6 +36,7 @@ pkgname = $(lastword $(subst /, ,$(pkgdir)))
 # Define extractors for different archive suffixes
 INFLATE.bz2  = $(BZCAT)
 INFLATE.gz   = $(ZCAT)
+INFLATE.lz   = $(LZCAT)
 INFLATE.lzma = $(XZCAT)
 INFLATE.tbz  = $(BZCAT)
 INFLATE.tbz2 = $(BZCAT)

package/pkg-utils.mk INFLATE.tar = cat · line 51

Quoted 6 lines
package/pkg-utils.mk +3 -0
@@ -45,6 +46,10 @@ INFLATE.tar  = cat
 # suitable-extractor(filename): returns extractor based on suffix
 suitable-extractor = $(INFLATE$(suffix $(1)))
 
+# host-extractor(filename): same as suitable-extractor, but filter out
+# extractors we build when the host lacks one.
+host-extractor = $(INFLATE$(filter-out .lz .lzma .xz,$(suffix $(1))))

suitable-extractor and host-extractor now render different output for a given file which makes no sense in general for functions named like this. You only need the different output to decide whether extra dependency checks need to be done, not to determine what the extractor is.

Also, the list of extensions is kind of 'magic' and not immediately linked with the changes in support/dependencies (this was already true with the original situation, true). It would be great if the exceptions could be steered from the check-host-foo.mk files directly, which is the conceptually right place: when that file exists it will take over all checking, if it does not then the checking is done in the standard DL_TOOLS_DEPENDENCIES.

I thought hard on a way to attack the problem without needing to know the extensions and more in line with the original change, but it is not easy due to the fact that XZCAT and such can contain spaces which does not fit well with make.
For clarity, I wanted to let check-host-xz.mk append to a new variable, e.g.

    DL_TOOLS_DEPENDENCIES_PRECHECKED_VARS += $(XZCAT)
and in pkg-generic.mk use this list to drive the exceptions:
    ifeq (,$$(filter $$(call
suitable-extractor,$$($(2)_SOURCE)),$$(DL_TOOLS_
DEPENDENCIES_PRECHECKED_VARS)))
    DL_TOOLS_DEPENDENCIES += $$(firstword $$(call

suitable-extractor,$$($(2)_SOURCE)))
endif
This could work if we first replace all spaces with another character, both when appending to DL_TOOLS_DEPENDENCIES_PRECHECKED_VARS as in the first argument of the filter function.
But perhaps it makes things too complex.

Second thought: by reducing XZCAT to its firstword before appending to some variable, the complexity is gone.
In that case, we can use filter-out to remove these prechecked tools from DL_TOOLS_DEPENDENCIES.
Perhaps then it doesn't need to be done from pkg-generic anymore but could be done from dependencies.mk where DL_TOOLS_DEPENDENCIES is actually used.

An alternative is to stick closer to your proposal, but with some changes:
- let check-host-foo.mk define the list of extensions that it handles instead of the magic list, e.g. EXTRACTOR_DEPENDENCY_PRECHECKED_EXTENSIONS += .xz
- rename host-extractor to something else, e.g. extractor-dependency
- perhaps it makes sense to let extractor-dependency handle the $(firstword) call

Given my input, what do you think?

Separate note: it could make sense to split this patch in two: one to refactor the handing of the dependencies, and a second one to cover lz7.

Best regards,
Thomas
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.busybox.net/pipermail/buildroot/attachments/20170212/f34e1401/attachment.html>

#11 lore raw

PATCH 1/4 core: add generic support for lz archives

In reply to Thomas De Schampheleire

Hi Thomas,

Quoted 29 lines · Thomas De Schampheleire

package/pkg-utils.mk INFLATE.tar = cat · line 51

Quoted 3 lines · Baruch Siach
package/pkg-utils.mk +3 -0
+# host-extractor(filename): same as suitable-extractor, but filter out
+# extractors we build when the host lacks one.
+host-extractor = $(INFLATE$(filter-out .lz .lzma .xz,$(suffix $(1))))

suitable-extractor and host-extractor now render different output for a given file which makes no sense in general for functions named like this. You only need the different output to decide whether extra dependency checks need to be done, not to determine what the extractor is.

Also, the list of extensions is kind of 'magic' and not immediately linked with the changes in support/dependencies (this was already true with the original situation, true). It would be great if the exceptions could be steered from the check-host-foo.mk files directly, which is the conceptually right place: when that file exists it will take over all checking, if it does not then the checking is done in the standard DL_TOOLS_DEPENDENCIES.

I thought hard on a way to attack the problem without needing to know the extensions and more in line with the original change, but it is not easy due to the fact that XZCAT and such can contain spaces which does not fit well with make.
For clarity, I wanted to let check-host-xz.mk append to a new variable, e.g.

    DL_TOOLS_DEPENDENCIES_PRECHECKED_VARS += $(XZCAT)
and in pkg-generic.mk use this list to drive the exceptions:
    ifeq (,$$(filter $$(call
suitable-extractor,$$($(2)_SOURCE)),$$(DL_TOOLS_DEPENDENCIES_PRECHECKED_VARS)))
    DL_TOOLS_DEPENDENCIES += $$(firstword $$(call
Quoted 6 lines

suitable-extractor,$$($(2)_SOURCE)))
endif
This could work if we first replace all spaces with another character, both when appending to DL_TOOLS_DEPENDENCIES_PRECHECKED_VARS as in the first argument of the filter function.
But perhaps it makes things too complex.

Indeed.

Quoted 9 lines

An alternative is to stick closer to your proposal, but with some changes:
- let check-host-foo.mk define the list of extensions that it handles instead of the magic list, e.g. EXTRACTOR_DEPENDENCY_PRECHECKED_EXTENSIONS += .xz
- rename host-extractor to something else, e.g. extractor-dependency
- perhaps it makes sense to let extractor-dependency handle the $(firstword) call

Given my input, what do you think?

I like your second suggestion better. Patches follow.

Quoted 3 lines

Separate note: it could make sense to split this patch in two: one to refactor the handing of the dependencies, and a second one to cover lz7.

I agree.

Thanks for reviewing,
baruch

Signature · 3 lines
     http://baruch.siach.name/blog/                  ~. .~   Tk Open Systems
=}------------------------------------------------ooO--U--Ooo------------{=
   - baruch at tkos.co.il - tel: +972.52.368.4656, http://www.tkos.co.il -