Skip to contentSkip to conversation

xz: make XZ_DEC_BCJ filters non-optional

PATCH In review Kyle McMartin 5 participants 10 messages Feb 28, 2014 – Mar 12, 2014 1 message not in the archive

Subsystems (2)
lore
Share & tools
lore
#0 lore raw

PATCH 1/1 xz: make XZ_DEC_BCJ filters non-optional

From: Kyle McMartin <[email protected]>

Having these optional is more trouble than is justified by the negligible increase in code size to lib/xz/ if they're all compiled in. Their optional status ends up necessitating rebuilds of the kernel in order to be able to decompress XZ-compressed squashfs images which use non-native BCJ filters (ie: you need to enable XZ_DEC_POWERPC on x86_64 in order to loop-mount a ppc64 squashfs that uses it.)

So save ourselves the trouble and build them all into xz_dec_bcj.o to begin with. (I could conceivably see a case where CONFIG_EXPERT made these selectable, but again, even on embedded platforms, the .text increase we're talking about is noise...)

text data bss dec hex filename
1239 0 0 1239 4d7 xz_dec_bcj.o
2263 0 0 2263 8d7 xz_dec_bcj.o.2

regards, Kyle

lib/xz/Kconfig +0 -38
@@ -6,44 +6,6 @@ config XZ_DEC
 	  the .xz file format as the container. For integrity checking,
 	  CRC32 is supported. See Documentation/xz.txt for more information.
 
-if XZ_DEC
-
-config XZ_DEC_X86
-	bool "x86 BCJ filter decoder"
-	default y if X86
-	select XZ_DEC_BCJ
-
-config XZ_DEC_POWERPC
-	bool "PowerPC BCJ filter decoder"
-	default y if PPC
-	select XZ_DEC_BCJ
-
-config XZ_DEC_IA64
-	bool "IA-64 BCJ filter decoder"
-	default y if IA64
-	select XZ_DEC_BCJ
-
-config XZ_DEC_ARM
-	bool "ARM BCJ filter decoder"
-	default y if ARM
-	select XZ_DEC_BCJ
-
-config XZ_DEC_ARMTHUMB
-	bool "ARM-Thumb BCJ filter decoder"
-	default y if (ARM && ARM_THUMB)
-	select XZ_DEC_BCJ
-
-config XZ_DEC_SPARC
-	bool "SPARC BCJ filter decoder"
-	default y if SPARC
-	select XZ_DEC_BCJ
-
-endif
-
-config XZ_DEC_BCJ
-	bool
-	default n
-
 config XZ_DEC_TEST
 	tristate "XZ decompressor tester"
 	default n
lib/xz/Makefile +1 -2
index a7fa769..4f209f7 100644
@@ -1,5 +1,4 @@
 obj-$(CONFIG_XZ_DEC) += xz_dec.o
-xz_dec-y := xz_dec_syms.o xz_dec_stream.o xz_dec_lzma2.o
-xz_dec-$(CONFIG_XZ_DEC_BCJ) += xz_dec_bcj.o
+xz_dec-y := xz_dec_syms.o xz_dec_stream.o xz_dec_lzma2.o xz_dec_bcj.o
 
 obj-$(CONFIG_XZ_DEC_TEST) += xz_dec_test.o
lib/xz/xz_dec_bcj.c +0 -43

135 lines not on this page · raw message

lib/xz/xz_dec_stream.c +0 -16
index ac809b1..a1ce0f7 100644
@@ -130,10 +130,8 @@ struct xz_dec {
 
 	struct xz_dec_lzma2 *lzma2;
 
-#ifdef XZ_DEC_BCJ
 	struct xz_dec_bcj *bcj;
 	bool bcj_active;
-#endif
 };
 
 #ifdef XZ_DEC_ANY_CHECK
@@ -222,11 +220,9 @@ static enum xz_ret dec_block(struct xz_dec *s, struct xz_buf *b)
 	s->in_start = b->in_pos;
 	s->out_start = b->out_pos;
 
-#ifdef XZ_DEC_BCJ
 	if (s->bcj_active)
 		ret = xz_dec_bcj_run(s->bcj, s->lzma2, b);
 	else
-#endif
 		ret = xz_dec_lzma2_run(s->lzma2, b);
 
 	s->block.compressed += b->in_pos - s->in_start;
@@ -465,11 +461,7 @@ static enum xz_ret dec_block_header(struct xz_dec *s)
 	 * Catch unsupported Block Flags. We support only one or two filters
 	 * in the chain, so we catch that with the same test.
 	 */
-#ifdef XZ_DEC_BCJ
 	if (s->temp.buf[1] & 0x3E)
-#else
-	if (s->temp.buf[1] & 0x3F)
-#endif
 		return XZ_OPTIONS_ERROR;
 
 	/* Compressed Size */
@@ -494,7 +486,6 @@ static enum xz_ret dec_block_header(struct xz_dec *s)
 		s->block_header.uncompressed = VLI_UNKNOWN;
 	}
 
-#ifdef XZ_DEC_BCJ
 	/* If there are two filters, the first one must be a BCJ filter. */
 	s->bcj_active = s->temp.buf[1] & 0x01;
 	if (s->bcj_active) {
@@ -512,7 +503,6 @@ static enum xz_ret dec_block_header(struct xz_dec *s)
 		if (s->temp.buf[s->temp.pos++] != 0x00)
 			return XZ_OPTIONS_ERROR;
 	}
-#endif
 
 	/* Valid Filter Flags always take at least two bytes. */
 	if (s->temp.size - s->temp.pos < 2)
@@ -775,11 +765,9 @@ XZ_EXTERN struct xz_dec *xz_dec_init(enum xz_mode mode, uint32_t dict_max)
 
 	s->mode = mode;
 
-#ifdef XZ_DEC_BCJ
 	s->bcj = xz_dec_bcj_create(DEC_IS_SINGLE(mode));
 	if (s->bcj == NULL)
 		goto error_bcj;
-#endif
 
 	s->lzma2 = xz_dec_lzma2_create(mode, dict_max);
 	if (s->lzma2 == NULL)
@@ -789,10 +777,8 @@ XZ_EXTERN struct xz_dec *xz_dec_init(enum xz_mode mode, uint32_t dict_max)
 	return s;
 
 error_lzma2:
-#ifdef XZ_DEC_BCJ
 	xz_dec_bcj_end(s->bcj);
 error_bcj:
-#endif
 	kfree(s);
 	return NULL;
 }
@@ -813,9 +799,7 @@ XZ_EXTERN void xz_dec_end(struct xz_dec *s)
 {
 	if (s != NULL) {
 		xz_dec_lzma2_end(s->lzma2);
-#ifdef XZ_DEC_BCJ
 		xz_dec_bcj_end(s->bcj);
-#endif
 		kfree(s);
 	}
 }
lib/xz/xz_private.h +0 -33
index 482b90f..72334af 100644
@@ -19,24 +19,6 @@
 #		include <linux/slab.h>
 #		include <linux/vmalloc.h>
 #		include <linux/string.h>
-#		ifdef CONFIG_XZ_DEC_X86
-#			define XZ_DEC_X86
-#		endif
-#		ifdef CONFIG_XZ_DEC_POWERPC
-#			define XZ_DEC_POWERPC
-#		endif
-#		ifdef CONFIG_XZ_DEC_IA64
-#			define XZ_DEC_IA64
-#		endif
-#		ifdef CONFIG_XZ_DEC_ARM
-#			define XZ_DEC_ARM
-#		endif
-#		ifdef CONFIG_XZ_DEC_ARMTHUMB
-#			define XZ_DEC_ARMTHUMB
-#		endif
-#		ifdef CONFIG_XZ_DEC_SPARC
-#			define XZ_DEC_SPARC
-#		endif
 #		define memeq(a, b, size) (memcmp(a, b, size) == 0)
 #		define memzero(buf, size) memset(buf, 0, size)
 #	endif
@@ -90,19 +72,6 @@
 #endif
 
 /*
- * If any of the BCJ filter decoders are wanted, define XZ_DEC_BCJ.
- * XZ_DEC_BCJ is used to enable generic support for BCJ decoders.
- */
-#ifndef XZ_DEC_BCJ
-#	if defined(XZ_DEC_X86) || defined(XZ_DEC_POWERPC) \
-			|| defined(XZ_DEC_IA64) || defined(XZ_DEC_ARM) \
-			|| defined(XZ_DEC_ARM) || defined(XZ_DEC_ARMTHUMB) \
-			|| defined(XZ_DEC_SPARC)
-#		define XZ_DEC_BCJ
-#	endif
-#endif
-
-/*
  * Allocate memory for LZMA2 decoder. xz_dec_lzma2_reset() must be used
  * before calling xz_dec_lzma2_run().
  */
@@ -125,7 +94,6 @@ XZ_EXTERN enum xz_ret xz_dec_lzma2_run(struct xz_dec_lzma2 *s,
 /* Free the memory allocated for the LZMA2 decoder. */
 XZ_EXTERN void xz_dec_lzma2_end(struct xz_dec_lzma2 *s);
 
-#ifdef XZ_DEC_BCJ
 /*
  * Allocate memory for BCJ decoders. xz_dec_bcj_reset() must be used before
  * calling xz_dec_bcj_run().
@@ -151,6 +119,5 @@ XZ_EXTERN enum xz_ret xz_dec_bcj_run(struct xz_dec_bcj *s,
 
 /* Free the memory allocated for the BCJ filters. */
 #define xz_dec_bcj_end(s) kfree(s)
-#endif
 
 #endif
maintainer
#1 lore raw
Quoted 6 lines · On 2014-02-28 Kyle McMartin wrote:

Having these optional is more trouble than is justified by the negligible increase in code size to lib/xz/ if they're all compiled in. Their optional status ends up necessitating rebuilds of the kernel in order to be able to decompress XZ-compressed squashfs images which use non-native BCJ filters (ie: you need to enable XZ_DEC_POWERPC on x86_64 in order to loop-mount a ppc64 squashfs that uses it.)

Originally all BCJ filters were enabled by default and CONFIG_EXPERT allowed disabling them one by one. It was changed a year ago to the current state because I wasn't against it when it was suggested:

    http://lkml.org/lkml/2013/1/15/449

It seems that I should have been against it. I suggest things are rolled back like they were: all BCJ filters enabled by default and CONFIG_EXPERT makes them deselectable. This time I have a somewhat strong opinion that this is the best way to go, even if it means that the embedded people must remember to deselect the unnneeded filters.

An alternative could be that with CONFIG_EXPERT only the BCJ filter for the current arch would be selected by default. Without CONFIG_EXPERT all filters would be forced on. I guess this could be confusing since the level of XZ support they get by default when they enable Squashfs' XZ support would depend on CONFIG_EXPERT. So I think my first suggestion is better.

Quoted 8 lines

So save ourselves the trouble and build them all into xz_dec_bcj.o to begin with. (I could conceivably see a case where CONFIG_EXPERT made these selectable, but again, even on embedded platforms, the .text increase we're talking about is noise...)

text data bss dec hex filename
1239 0 0 1239 4d7 xz_dec_bcj.o
2263 0 0 2263 8d7 xz_dec_bcj.o.2

No, let's not unconditionally build them all:

- 1 KiB might be noise, but since on embedded systems that 1 KiB really is completely useless code and it's very easy to omit it, the option to omit such code should be kept.

- If the kernel image is compressed with XZ, a separate copy of the decompressor is built for the pre-boot environment. Conditional compilation of the BCJ filters is also used for pre-boot environments which your patch doesn't touch. The 1 KiB increase would affect the copy in the pre-boot code too.

- This XZ decompressor was written with the Linux kernel in mind, but it's used elsewhere too. It would be nice to minimize the differences between the code in Linux and the out-of-kernel tree. Outside Linux I will keep the BCJ filters optional anyway.

Below are two alternative patches matching my two suggestions. I prefer the first patch and I will submit it in a day or two unless people have better ideas.

lib/xz/Kconfig +12 -12
diff -Nru linux-3.14-rc5.orig/lib/xz/Kconfig linux-3.14-rc5/lib/xz/Kconfig
@@ -9,33 +9,33 @@
 if XZ_DEC
 
 config XZ_DEC_X86
-	bool "x86 BCJ filter decoder"
-	default y if X86
+	bool "x86 BCJ filter decoder" if EXPERT
+	default y
 	select XZ_DEC_BCJ
 
 config XZ_DEC_POWERPC
-	bool "PowerPC BCJ filter decoder"
-	default y if PPC
+	bool "PowerPC BCJ filter decoder" if EXPERT
+	default y
 	select XZ_DEC_BCJ
 
 config XZ_DEC_IA64
-	bool "IA-64 BCJ filter decoder"
-	default y if IA64
+	bool "IA-64 BCJ filter decoder" if EXPERT
+	default y
 	select XZ_DEC_BCJ
 
 config XZ_DEC_ARM
-	bool "ARM BCJ filter decoder"
-	default y if ARM
+	bool "ARM BCJ filter decoder" if EXPERT
+	default y
 	select XZ_DEC_BCJ
 
 config XZ_DEC_ARMTHUMB
-	bool "ARM-Thumb BCJ filter decoder"
-	default y if (ARM && ARM_THUMB)
+	bool "ARM-Thumb BCJ filter decoder" if EXPERT
+	default y
 	select XZ_DEC_BCJ
 
 config XZ_DEC_SPARC
-	bool "SPARC BCJ filter decoder"
-	default y if SPARC
+	bool "SPARC BCJ filter decoder" if EXPERT
+	default y
 	select XZ_DEC_BCJ
 
 endif

The second version enables the BCJ filter only for the current arch by default if CONFIG_EXPERT; without CONFIG_EXPERT all filters are forced on:

lib/xz/Kconfig +12 -12
diff -Nru linux-3.14-rc5.orig/lib/xz/Kconfig linux-3.14-rc5/lib/xz/Kconfig
@@ -9,33 +9,33 @@
 if XZ_DEC
 
 config XZ_DEC_X86
-	bool "x86 BCJ filter decoder"
-	default y if X86
+	bool "x86 BCJ filter decoder" if EXPERT
+	default y if !EXPERT || X86
 	select XZ_DEC_BCJ
 
 config XZ_DEC_POWERPC
-	bool "PowerPC BCJ filter decoder"
-	default y if PPC
+	bool "PowerPC BCJ filter decoder" if EXPERT
+	default y if !EXPERT || PPC
 	select XZ_DEC_BCJ
 
 config XZ_DEC_IA64
-	bool "IA-64 BCJ filter decoder"
-	default y if IA64
+	bool "IA-64 BCJ filter decoder" if EXPERT
+	default y if !EXPERT || IA64
 	select XZ_DEC_BCJ
 
 config XZ_DEC_ARM
-	bool "ARM BCJ filter decoder"
-	default y if ARM
+	bool "ARM BCJ filter decoder" if EXPERT
+	default y if !EXPERT || ARM
 	select XZ_DEC_BCJ
 
 config XZ_DEC_ARMTHUMB
-	bool "ARM-Thumb BCJ filter decoder"
-	default y if (ARM && ARM_THUMB)
+	bool "ARM-Thumb BCJ filter decoder" if EXPERT
+	default y if !EXPERT || (ARM && ARM_THUMB)
 	select XZ_DEC_BCJ
 
 config XZ_DEC_SPARC
-	bool "SPARC BCJ filter decoder"
-	default y if SPARC
+	bool "SPARC BCJ filter decoder" if EXPERT
+	default y if !EXPERT || SPARC
 	select XZ_DEC_BCJ
 
 endif
Signature · 1 line
Lasse Collin  |  IRC: Larhzu @ IRCnet & Freenode
#2 lore raw

2014-03-03 4:51 GMT-08:00 Lasse Collin <[email protected]>:

Quoted 48 lines · On 2014-02-28 Kyle McMartin wrote:
Quoted 6 lines

Having these optional is more trouble than is justified by the negligible increase in code size to lib/xz/ if they're all compiled in. Their optional status ends up necessitating rebuilds of the kernel in order to be able to decompress XZ-compressed squashfs images which use non-native BCJ filters (ie: you need to enable XZ_DEC_POWERPC on x86_64 in order to loop-mount a ppc64 squashfs that uses it.)

Originally all BCJ filters were enabled by default and CONFIG_EXPERT allowed disabling them one by one. It was changed a year ago to the current state because I wasn't against it when it was suggested:

    http://lkml.org/lkml/2013/1/15/449

It seems that I should have been against it. I suggest things are rolled back like they were: all BCJ filters enabled by default and CONFIG_EXPERT makes them deselectable. This time I have a somewhat strong opinion that this is the best way to go, even if it means that the embedded people must remember to deselect the unnneeded filters.

An alternative could be that with CONFIG_EXPERT only the BCJ filter for the current arch would be selected by default. Without CONFIG_EXPERT all filters would be forced on. I guess this could be confusing since the level of XZ support they get by default when they enable Squashfs' XZ support would depend on CONFIG_EXPERT. So I think my first suggestion is better.

Quoted 8 lines

So save ourselves the trouble and build them all into xz_dec_bcj.o to begin with. (I could conceivably see a case where CONFIG_EXPERT made these selectable, but again, even on embedded platforms, the .text increase we're talking about is noise...)

text  data    bss     dec     hex     filename
1239  0       0       1239    4d7     xz_dec_bcj.o
2263  0       0       2263    8d7     xz_dec_bcj.o.2

No, let's not unconditionally build them all:

- 1 KiB might be noise, but since on embedded systems that 1 KiB really is completely useless code and it's very easy to omit it, the option to omit such code should be kept.

- If the kernel image is compressed with XZ, a separate copy of the decompressor is built for the pre-boot environment. Conditional compilation of the BCJ filters is also used for pre-boot environments which your patch doesn't touch. The 1 KiB increase would affect the copy in the pre-boot code too.

Quoted 4 lines

- This XZ decompressor was written with the Linux kernel in mind, but it's used elsewhere too. It would be nice to minimize the differences between the code in Linux and the out-of-kernel tree. Outside Linux I will keep the BCJ filters optional anyway.

Absolutely, this was actually the primary concern for these patches. 1KiB might not sound like a lot, but when you take than into account in the bigger picture of saving kernel size, this ends up making a difference.

Quoted 57 lines

Below are two alternative patches matching my two suggestions. I prefer the first patch and I will submit it in a day or two unless people have better ideas.

diff -Nru linux-3.14-rc5.orig/lib/xz/Kconfig linux-3.14-rc5/lib/xz/Kconfig

lib/xz/Kconfig 2014-03-03 13:26:58.402872951 +0200 renamed from lib/xz/Kconfig 2014-03-03 04:56:16.000000000 +0200 +12 -12
@@ -9,33 +9,33 @@
 if XZ_DEC
 
 config XZ_DEC_X86
-       bool "x86 BCJ filter decoder"
-       default y if X86
+       bool "x86 BCJ filter decoder" if EXPERT
+       default y
        select XZ_DEC_BCJ
 
 config XZ_DEC_POWERPC
-       bool "PowerPC BCJ filter decoder"
-       default y if PPC
+       bool "PowerPC BCJ filter decoder" if EXPERT
+       default y
        select XZ_DEC_BCJ
 
 config XZ_DEC_IA64
-       bool "IA-64 BCJ filter decoder"
-       default y if IA64
+       bool "IA-64 BCJ filter decoder" if EXPERT
+       default y
        select XZ_DEC_BCJ
 
 config XZ_DEC_ARM
-       bool "ARM BCJ filter decoder"
-       default y if ARM
+       bool "ARM BCJ filter decoder" if EXPERT
+       default y
        select XZ_DEC_BCJ
 
 config XZ_DEC_ARMTHUMB
-       bool "ARM-Thumb BCJ filter decoder"
-       default y if (ARM && ARM_THUMB)
+       bool "ARM-Thumb BCJ filter decoder" if EXPERT
+       default y
        select XZ_DEC_BCJ
 
 config XZ_DEC_SPARC
-       bool "SPARC BCJ filter decoder"
-       default y if SPARC
+       bool "SPARC BCJ filter decoder" if EXPERT
+       default y
        select XZ_DEC_BCJ
 
 endif
Quoted 3 lines

The second version enables the BCJ filter only for the current arch by default if CONFIG_EXPERT; without CONFIG_EXPERT all filters are forced on:

I like this version better because it still provides you with accurate defaults (i.e: enable only th X86 BCJ filter decoder where it makes sense by default) and still provides enough flexibility.

Quoted 53 lines

diff -Nru linux-3.14-rc5.orig/lib/xz/Kconfig linux-3.14-rc5/lib/xz/Kconfig

lib/xz/Kconfig 2014-03-03 14:15:42.196209325 +0200 renamed from lib/xz/Kconfig 2014-03-03 04:56:16.000000000 +0200 +12 -13
@@ -9,33 +9,33 @@
 if XZ_DEC
 
 config XZ_DEC_X86
-       bool "x86 BCJ filter decoder"
-       default y if X86
+       bool "x86 BCJ filter decoder" if EXPERT
+       default y if !EXPERT || X86
        select XZ_DEC_BCJ
 
 config XZ_DEC_POWERPC
-       bool "PowerPC BCJ filter decoder"
-       default y if PPC
+       bool "PowerPC BCJ filter decoder" if EXPERT
+       default y if !EXPERT || PPC
        select XZ_DEC_BCJ
 
 config XZ_DEC_IA64
-       bool "IA-64 BCJ filter decoder"
-       default y if IA64
+       bool "IA-64 BCJ filter decoder" if EXPERT
+       default y if !EXPERT || IA64
        select XZ_DEC_BCJ
 
 config XZ_DEC_ARM
-       bool "ARM BCJ filter decoder"
-       default y if ARM
+       bool "ARM BCJ filter decoder" if EXPERT
+       default y if !EXPERT || ARM
        select XZ_DEC_BCJ
 
 config XZ_DEC_ARMTHUMB
-       bool "ARM-Thumb BCJ filter decoder"
-       default y if (ARM && ARM_THUMB)
+       bool "ARM-Thumb BCJ filter decoder" if EXPERT
+       default y if !EXPERT || (ARM && ARM_THUMB)
        select XZ_DEC_BCJ
 
 config XZ_DEC_SPARC
-       bool "SPARC BCJ filter decoder"
-       default y if SPARC
+       bool "SPARC BCJ filter decoder" if EXPERT
+       default y if !EXPERT || SPARC
        select XZ_DEC_BCJ
 
 endif
 
--

Lasse Collin | IRC: Larhzu @ IRCnet & Freenode

Signature · 1 line
Florian
maintainer
#3 lore raw
Quoted 8 lines · On 2014-03-03 Florian Fainelli wrote:

2014-03-03 4:51 GMT-08:00 Lasse Collin <[email protected]>:

Quoted 3 lines

The second version enables the BCJ filter only for the current arch by default if CONFIG_EXPERT; without CONFIG_EXPERT all filters are forced on:

I like this version better because it still provides you with accurate defaults (i.e: enable only th X86 BCJ filter decoder where it makes sense by default) and still provides enough flexibility.

I understand your viewpoint, but different people probably consider different options as "accurate defaults". The current behavior certainly isn't very accurate; otherwise Kyle McMartin wouldn't have had a reason to send a patch. The patch you prefer would give both of you the defaults you want *if* embedded devs always use CONFIG_EXPERT and non-embedded people never do. If a non-embedded dev uses CONFIG_EXPERT, then Kyle's problem can reappear.

I still think it is clearest to enable all by default and allow deselecting filters if CONFIG_EXPERT. That way the default selection doesn't change behind anyone's back when CONFIG_EXPERT is selected. I guess embedded devs go through the config carefully looking for things to deselect anyway, but maybe it was easy to miss these options just like some desktop users have missed them now.

I'd like to make the defaults such that minimum number of people get annoyed in the long term. There will always be people who will need to change the options (otherwise options wouldn't be needed). At the same time this is so very tiny issue that I don't expect many people to care at all.

I'll submit some patch this week but I'll wait for more opinions for a day or two. Thanks.

Signature · 1 line
Lasse Collin  |  IRC: Larhzu @ IRCnet & Freenode

A message that is not in the archive (sent off-list or not yet received): [email protected]

#5 lore raw

2014-03-04 18:51 GMT-08:00 Phillip Lougher <[email protected]>:

Quoted 82 lines · Lasse Collin
Quoted 10 lines · On 2014-03-03 Florian Fainelli wrote:
Quoted 8 lines

2014-03-03 4:51 GMT-08:00 Lasse Collin <[email protected]>:

Quoted 3 lines

The second version enables the BCJ filter only for the current arch by default if CONFIG_EXPERT; without CONFIG_EXPERT all filters are forced on:

I like this version better because it still provides you with accurate defaults (i.e: enable only th X86 BCJ filter decoder where it makes sense by default) and still provides enough flexibility.

Quoted 8 lines

I understand your viewpoint, but different people probably consider different options as "accurate defaults". The current behavior certainly isn't very accurate; otherwise Kyle McMartin wouldn't have had a reason to send a patch. The patch you prefer would give both of you the defaults you want *if* embedded devs always use CONFIG_EXPERT and non-embedded people never do. If a non-embedded dev uses CONFIG_EXPERT, then Kyle's problem can reappear.

CONFIG_EMBEDDED was renamed CONFIG_EXPERT a couple of years ago because it was being used by more than embedded people. As such I'd be wary adding an option which relied on the original meaning.

http://lwn.net/Articles/421304/

Quoted 7 lines

I still think it is clearest to enable all by default and allow deselecting filters if CONFIG_EXPERT. That way the default selection doesn't change behind anyone's back when CONFIG_EXPERT is selected. I guess embedded devs go through the config carefully looking for things to deselect anyway, but maybe it was easy to miss these options just like some desktop users have missed them now.

Quoted 9 lines

I'd like to make the defaults such that minimum number of people get annoyed in the long term. There will always be people who will need to change the options (otherwise options wouldn't be needed). At the same time this is so very tiny issue that I don't expect many people to care at all.

I'll submit some patch this week but I'll wait for more opinions for a day or two. Thanks.

As Kyle should know Redhat hit this problem with Squashfs a while back, and it was I who identified the config changes as the culprit (BTW Kyle you should have CC'd me on the patch as a courtesy). The bug is only viewable by Redhat employees (disclaimer, I'm also a Redhat employee), and so I'll say no more.

But speaking as the Squashfs author, the lack of BCJ support for an architecture creates a subtle failure mode in Squashfs, this is because not all blocks in a Squashfs filesystem get compressed with a BCJ filter. At compression time each block is compressed without any BCJ filter, and then with the BCJ filter(s) selected on the command line, and the best compression for *that* block is chosen. What this means is kernels without a particular BCJ filter can still read the Squashfs metadata (mount, ls etc.) and read many of the files, it is only some files that mysteriously fail with decompression error. As such this will be (and has been) invariably treated as a bug in Squashfs.

Moreover, without expert knowledge of Squashfs, and the config options, most people will not have a clue how to fix the issue.

This is why I prefer the first option, which is to reinstate the enabling of all filters by default, and then to allow people to remove the filters they don't want.

BTW there is a potential additional fix for Squashfs that will make its handling of (lack of) BCJ filters more intelligent at mount time, but this of course only addresses Squashfs, and it relies on an additional call into XZ being added. The BCJ filters specified at filesystem creation are stored in the compression options part of the superblock, and are known at mount time. Squashfs should check that these filters are supported by the kernel and refuse to mount it otherwise. This has not been done because AFAIK there is no way to query XZ to determine which BCJ filters are supported (beyond passing it a test stream which is too messy).

Quoted 2 lines

If this call was added, I would be happy to add a patch to Squashfs to do this.

I am okay with enabling all filters by default as long as there is a knob to allow turning achitecture specific BCJ filters off, as it barely makes sense for embedded systems to have a BCJ filter implementation for anything but the architecture you are running on.

It does make sense for a desktop use-case like what Kyle describes in his initial patch. What you describes sounds like there is room for improvements though instead of silently failing...

Thanks

Signature · 1 line
Florian
#6 lore raw
Quoted 4 lines · Florian Fainelli

I am okay with enabling all filters by default as long as there is a knob to allow turning achitecture specific BCJ filters off, as it barely makes sense for embedded systems to have a BCJ filter implementation for anything but the architecture you are running on.

And for x86? (oh no, I started promoting x86 ;-)

I once tried xz with an initrd on ARM. The kernel complained it couldn't decompress the initrd, oops. I didn't investigate it at that time, but probably I didn't have the x86 BCJ filter enabled, while I compressed the initrd on\
amd64.

Gr{oetje,eeting}s,

Geert

--
Geert Uytterhoeven -- There's lots of Linux beyond ia32 -- [email protected]

In personal conversations with technical people, I call myself a hacker. But when I'm talking to journalists I just say "programmer" or something like that.
-- Linus Torvalds

maintainer
#7 lore raw
Quoted 4 lines · On 2014-03-06 Geert Uytterhoeven wrote:

I once tried xz with an initrd on ARM. The kernel complained it couldn't decompress the initrd, oops. I didn't investigate it at that time, but probably I didn't have the x86 BCJ filter enabled, while I compressed the initrd on\ amd64.

With so little information I can only guess, but it sounds unlikely that you would have enabled the x86 BCJ filter without knowing it unless you used some wrapper script that does it. It's more likely that the ARM kernel didn't support XZ at all, you forgot --check=crc32, or it ran out of RAM due to too big LZMA2 dictionary (if you used -9, the decompressor allocates 64 MiB of memory, but I cannot guess how much RAM the target system had).

In Documentation/xz.txt under "Notes on compression options" there are some tips about compressing files for the in-kernel XZ decompressor.

Signature · 1 line
Lasse Collin  |  IRC: Larhzu @ IRCnet & Freenode
#8 lore raw
Quoted 7 lines · Lasse Collin
Quoted 4 lines · On 2014-03-06 Geert Uytterhoeven wrote:

I once tried xz with an initrd on ARM. The kernel complained it couldn't decompress the initrd, oops. I didn't investigate it at that time, but probably I didn't have the x86 BCJ filter enabled, while I compressed the initrd on\ amd64.

With so little information I can only guess, but it sounds unlikely

I understand. Will see if I find time to retry...

Quoted 3 lines

that you would have enabled the x86 BCJ filter without knowing it unless you used some wrapper script that does it. It's more likely that the ARM kernel didn't support XZ at all, you forgot --check=crc32,

I definitely didn't pass --check=crc32, just plain xz (like I'm used to plain gzip ;-) So that's the most likely culprit.

Quoted 3 lines

or it ran out of RAM due to too big LZMA2 dictionary (if you used -9, the decompressor allocates 64 MiB of memory, but I cannot guess how much RAM the target system had).

2 GiB, so that shouldn't be an issue.

Quoted 2 lines

In Documentation/xz.txt under "Notes on compression options" there are some tips about compressing files for the in-kernel XZ decompressor.

Thanks, the --check=crc32 is indeed mentioned there.

Gr{oetje,eeting}s,

Geert

--
Geert Uytterhoeven -- There's lots of Linux beyond ia32 -- [email protected]

In personal conversations with technical people, I call myself a hacker. But when I'm talking to journalists I just say "programmer" or something like that.
-- Linus Torvalds

maintainer
#9 lore raw
In reply to a message not in the archive
Quoted 1 line · On 2014-03-05 Phillip Lougher wrote:

(BTW Kyle you should have CC'd me on the patch as a courtesy).

I could have done that too but somehow I didn't, sorry.

Quoted 11 lines

But speaking as the Squashfs author, the lack of BCJ support for an architecture creates a subtle failure mode in Squashfs, this is because not all blocks in a Squashfs filesystem get compressed with a BCJ filter. At compression time each block is compressed without any BCJ filter, and then with the BCJ filter(s) selected on the command line, and the best compression for *that* block is chosen. What this means is kernels without a particular BCJ filter can still read the Squashfs metadata (mount, ls etc.) and read many of the files, it is only some files that mysteriously fail with decompression error. As such this will be (and has been) invariably treated as a bug in Squashfs.

There is an easy way to make Squashfs give an error message in the kernel log. xz_dec_run() gives XZ_OPTIONS_ERROR when valid-looking but unsupported input is detected. Currently Squashfs treats all error codes from xz_dec_run() the same way so the reason for the decompression error is silently lost.

Below is an *untested* fix. I'm not sure about the exact wording of the error message, so feel free to improve it.

fs/squashfs/xz_wrapper.c +6 -1
diff -Narup linux-3.14-rc5.orig/fs/squashfs/xz_wrapper.c linux-3.14-rc5/fs/squashfs/xz_wrapper.c
@@ -170,8 +170,13 @@ static int squashfs_xz_uncompress(struct
 
 	squashfs_finish_page(output);
 
-	if (xz_err != XZ_STREAM_END || k < b)
+	if (xz_err != XZ_STREAM_END || k < b) {
+		if (xz_err == XZ_OPTIONS_ERROR)
+			ERROR("Unsupported XZ-compressed data; check the XZ "
+					"options in the kernel config\n");
+
 		goto out;
+	}
 
 	return total + stream->buf.out_pos;
 
Quoted 6 lines

Moreover, without expert knowledge of Squashfs, and the config options, most people will not have a clue how to fix the issue.

This is why I prefer the first option, which is to reinstate the enabling of all filters by default, and then to allow people to remove the filters they don't want.

I will submit the first option. In the other email Florian Fainelli seemed to be OK with that too.

Quoted 11 lines

BTW there is a potential additional fix for Squashfs that will make its handling of (lack of) BCJ filters more intelligent at mount time, but this of course only addresses Squashfs, and it relies on an additional call into XZ being added. The BCJ filters specified at filesystem creation are stored in the compression options part of the superblock, and are known at mount time. Squashfs should check that these filters are supported by the kernel and refuse to mount it otherwise. This has not been done because AFAIK there is no way to query XZ to determine which BCJ filters are supported (beyond passing it a test stream which is too messy).

You can use #ifdef CONFIG_XZ_DEC_X86 and such, although maybe that's not convenient enough.

Adding a function to xz_dec module could be sort of OK. It's important to keep in mind that the ability to disable filters is there to reduce code size *slightly*. If you or we add something extra to figure out what is supported at runtime, the size benefit may get lost.

If all filters are enabled by default, a clear error message on XZ_OPTIONS_ERROR should be enough, I think.

Signature · 1 line
Lasse Collin  |  IRC: Larhzu @ IRCnet & Freenode
#10 lore raw
Quoted 4 lines · Lasse Collin
Quoted 1 line · On 2014-03-05 Phillip Lougher wrote:

(BTW Kyle you should have CC'd me on the patch as a courtesy).

I could have done that too but somehow I didn't, sorry.

np

Quoted 13 lines
Quoted 12 lines

But speaking as the Squashfs author, the lack of BCJ support for an architecture creates a subtle failure mode in Squashfs, this is because not all blocks in a Squashfs filesystem get compressed with a BCJ filter. At compression time each block is compressed without any BCJ filter, and then with the BCJ filter(s) selected on the command line, and the best compression for *that* block is chosen. What this means is kernels without a particular BCJ filter can still read the Squashfs metadata (mount, ls etc.) and read many of the files, it is only some files that mysteriously fail with decompression error. As such this will be (and has been) invariably treated as a bug in Squashfs.

Quoted 5 lines

There is an easy way to make Squashfs give an error message in the kernel log. xz_dec_run() gives XZ_OPTIONS_ERROR when valid-looking but unsupported input is detected. Currently Squashfs treats all error codes from xz_dec_run() the same way so the reason for the decompression error is silently lost.

Yes, that is deliberate. It's to give a generic easy to understand error message for potentially novice users that may be running Linux from LiveCDs.

When I wrote the original zlib support for Squashfs, I put in a lot of debug information in the zlib error messages, e.g.

                       ERROR("zlib_inflate returned unexpected result"
                               " 0x%x, srclength %d, avail_in %d,"
                               " avail_out %d\n", zlib_err, srclength,
                               msblk->stream.avail_in,
                               msblk->stream.avail_out);

But after mainlining Squashfs in 2009, I started to get increasing complaints that the error messages were too technical (full of hex numbers) and confusing to new users - the kind of people who maybe burn a corrupt liveCD and then get screenfulls of these errors full of different numbers coming up on the screen. They would do a websearch and discover that the errors meant "corrupted disk", and then ask why didn't it just say that, and not give all those numbers. Or worse they'd just silently give up and go back to Windows.

So in March 2009 I changed it to the error message

ERROR("zlib_inflate error, data probably corrupt\n")

With *no* numbers ... and I copied the same approach for xz.

In kernel 3.13 (released earlier this year) I went further and pulled out the error message printing from the compression wrappers, and made it a single generic message, because I realised there was no longer any decompressor specific error handling (just the same message in each wrapper).

ERROR("%s decompression failed, data probably corrupt\n",

                        msblk->decompressor->name);

Putting back separate error messages into each decompressor wrapper, and then putting back different error messages based on the error code return seems like a retrograde step because distros don't like them.

Quoted 45 lines

Below is an *untested* fix. I'm not sure about the exact wording of the error message, so feel free to improve it.

diff -Narup linux-3.14-rc5.orig/fs/squashfs/xz_wrapper.c linux-3.14-rc5/fs/squashfs/xz_wrapper.c

fs/squashfs/xz_wrapper.c +6 -1
@@ -170,8 +170,13 @@ static int squashfs_xz_uncompress(struct
 
  	squashfs_finish_page(output);
 
-	if (xz_err != XZ_STREAM_END || k < b)
+	if (xz_err != XZ_STREAM_END || k < b) {
+		if (xz_err == XZ_OPTIONS_ERROR)
+			ERROR("Unsupported XZ-compressed data; check the XZ "
+					"options in the kernel config\n");
+
  		goto out;
+	}
 
  	return total + stream->buf.out_pos;
 
Quoted 7 lines

Moreover, without expert knowledge of Squashfs, and the config options, most people will not have a clue how to fix the issue.

This is why I prefer the first option, which is to reinstate the enabling of all filters by default, and then to allow people to remove the filters they don't want.

I will submit the first option. In the other email Florian Fainelli seemed to be OK with that too.

Quoted 11 lines

BTW there is a potential additional fix for Squashfs that will make its handling of (lack of) BCJ filters more intelligent at mount time, but this of course only addresses Squashfs, and it relies on an additional call into XZ being added. The BCJ filters specified at filesystem creation are stored in the compression options part of the superblock, and are known at mount time. Squashfs should check that these filters are supported by the kernel and refuse to mount it otherwise. This has not been done because AFAIK there is no way to query XZ to determine which BCJ filters are supported (beyond passing it a test stream which is too messy).

Quoted 2 lines

You can use #ifdef CONFIG_XZ_DEC_X86 and such, although maybe that's not convenient enough.

Yes, I will try using the XZ config definitions directly in Squashfs.

Thanks

Phillip