lzop: add LZOP definition for the host variant
PATCH 1/3 lzop: add LZOP definition for the host variant
--- package/lzop/lzop.mk | 3 +++ 1 file changed, 3 insertions(+)
package/lzop/lzop.mk +3 -0
@@ -3,6 +3,7 @@ # lzop # ############################################################# + LZOP_VERSION = 1.03 LZOP_SOURCE = lzop-$(LZOP_VERSION).tar.gz LZOP_SITE = http://www.lzop.org/download/
@@ -12,3 +13,5 @@ LZOP_DEPENDENCIES = lzo $(eval $(autotools-package)) $(eval $(host-autotools-package)) + +LZOP = $(HOST_DIR)/usr/bin/lzop
Signature · 1 line
1.7.12.4
PATCH 2/3 xz: add XZ definition for the host variant
--- package/xz/xz.mk | 2 ++ 1 file changed, 2 insertions(+)
package/xz/xz.mk +2 -0
@@ -14,3 +14,5 @@ XZ_LICENSE_FILES = COPYING.GPLv2 COPYING.GPLv3 COPYING.LGPLv2.1 $(eval $(autotools-package)) $(eval $(host-autotools-package)) + +XZ = $(HOST_DIR)/usr/bin/xz
Signature · 1 line
1.7.12.4
PATCH 3/3 initramfs: add support for LZO and XZ compression methods
In reply to Gustavo Zacarias--- fs/common.mk | 8 +++++++- fs/cpio/Config.in | 18 ++++++++++++++---- 2 files changed, 21 insertions(+), 5 deletions(-)
fs/common.mk +7 -1
@@ -39,7 +39,7 @@ ROOTFS_DEVICE_TABLES = $(call qstrip,$(BR2_ROOTFS_DEVICE_TABLE)) \ define ROOTFS_TARGET_INTERNAL # extra deps -ROOTFS_$(2)_DEPENDENCIES += host-fakeroot host-makedevs $$(if $$(BR2_TARGET_ROOTFS_$(2)_LZMA),host-lzma) +ROOTFS_$(2)_DEPENDENCIES += host-fakeroot host-makedevs $$(if $$(BR2_TARGET_ROOTFS_$(2)_LZMA),host-lzma) $$(if $$(BR2_TARGET_ROOTFS_$(2)_LZO),host-lzop) $$(if $$(BR2_TARGET_ROOTFS_$(2)_XZ),host-xz) $$(BINARIES_DIR)/rootfs.$(1): $$(ROOTFS_$(2)_DEPENDENCIES) @$$(call MESSAGE,"Generating root filesystem image rootfs.$(1)")
@@ -70,6 +70,12 @@ endif ifeq ($$(BR2_TARGET_ROOTFS_$(2)_LZMA),y) $$(LZMA) -9 -c $$@ > [email protected] endif +ifeq ($$(BR2_TARGET_ROOTFS_$(2)_LZO),y) + $$(LZOP) -9 -c $$@ > [email protected] +endif +ifeq ($$(BR2_TARGET_ROOTFS_$(2)_XZ),y) + $$(XZ) -9 -c $$@ > [email protected] +endif rootfs-$(1)-show-depends: @echo $$(ROOTFS_$(2)_DEPENDENCIES)
fs/cpio/Config.in +14 -4
@@ -18,22 +18,32 @@ choice config BR2_TARGET_ROOTFS_CPIO_NONE bool "no compression" help - Do not compress the cpio filesystem. + Do not compress the cpio filesystem. config BR2_TARGET_ROOTFS_CPIO_GZIP bool "gzip" help - Do compress the cpio filesystem with gzip. + Do compress the cpio filesystem with gzip. config BR2_TARGET_ROOTFS_CPIO_BZIP2 bool "bzip2" help - Do compress the cpio filesystem with bzip2. + Do compress the cpio filesystem with bzip2. config BR2_TARGET_ROOTFS_CPIO_LZMA bool "lzma" help - Do compress the cpio filesystem with lzma. + Do compress the cpio filesystem with lzma. + +config BR2_TARGET_ROOTFS_CPIO_LZO + bool "lzo" + help + Do compress the cpio filesystem with lzo. + +config BR2_TARGET_ROOTFS_CPIO_XZ + bool "xz" + help + Do compress the cpio filesystem with xz. endchoice
Signature · 1 line
1.7.12.4
PATCH 3/3 initramfs: add support for LZO and XZ compression methods
Hi Gustavo
22.1.2013 0:55, Gustavo Zacarias kirjoitti:
Quoted 20 lines
Signed-off-by Gustavo Zacarias <[email protected]>--- fs/common.mk | 8 +++++++- fs/cpio/Config.in | 18 ++++++++++++++---- 2 files changed, 21 insertions(+), 5 deletions(-)fs/common.mk +1
-1@@ -39,7 +39,7 @@ ROOTFS_DEVICE_TABLES = $(call qstrip,$(BR2_ROOTFS_DEVICE_TABLE)) \ define ROOTFS_TARGET_INTERNAL # extra deps -ROOTFS_$(2)_DEPENDENCIES += host-fakeroot host-makedevs $$(if $$(BR2_TARGET_ROOTFS_$(2)_LZMA),host-lzma) +ROOTFS_$(2)_DEPENDENCIES += host-fakeroot host-makedevs $$(if $$(BR2_TARGET_ROOTFS_$(2)_LZMA),host-lzma) $$(if $$(BR2_TARGET_ROOTFS_$(2)_LZO),host-lzop) $$(if $$(BR2_TARGET_ROOTFS_$(2)_XZ),host-xz) $$(BINARIES_DIR)/rootfs.$(1): $$(ROOTFS_$(2)_DEPENDENCIES) @$$(call MESSAGE,"Generating root filesystem image rootfs.$(1)")
fs/common.mk endif · line 78
Quoted 9 lines
fs/common.mk +6
-0@@ -70,6 +70,12 @@ endif ifeq ($$(BR2_TARGET_ROOTFS_$(2)_LZMA),y) $$(LZMA) -9 -c $$@ > [email protected] endif +ifeq ($$(BR2_TARGET_ROOTFS_$(2)_LZO),y) + $$(LZOP) -9 -c $$@ > [email protected] +endif +ifeq ($$(BR2_TARGET_ROOTFS_$(2)_XZ),y) + $$(XZ) -9 -c $$@ > [email protected] +endif
You will probably need --check=crc32 with that xz-compression. I have had difficulties booting linux 3.3.1 kernel (don't know about later versions)
with xz-compressed initramfs without that option.
Regards
Stefan
PATCH 3/3 initramfs: add support for LZO and XZ compression methods
Quoted 4 lines · Stefan Fr?berg
You will probably need --check=crc32 with that xz-compression. I have had difficulties booting linux 3.3.1 kernel (don't know about later versions)
with xz-compressed initramfs without that option.
You are 100% correct.
It didn't fail me during testing because buildroot is never using any compressed cpio archive, thus the previous options (gzip, bzip2) were never used.
I noticed when checking the bzImage size in a tweaked qemu_x86_defconfig test run that it never changed much.
New patchset will follow that fixes this and the previous issue. Regards.
PATCH 3/3 initramfs: add support for LZO and XZ compression methods
22.1.2013 1:44, Gustavo Zacarias kirjoitti:
Quoted 10 lines · Stefan Fr?berg
Quoted 5 lines
You will probably need --check=crc32 with that xz-compression. I have had difficulties booting linux 3.3.1 kernel (don't know about later versions)
with xz-compressed initramfs without that option.You are 100% correct.
It didn't fail me during testing because buildroot is never using any compressed cpio archive, thus the previous options (gzip, bzip2) were never used.
Yeah.
It really doesn't make sense to compress if the initramfs file in question is embedded directly into
kernel (which at least version 2012.05 and earlier buildroot did) because usually the kernel
itself would be also compressed. So in that specific case doing double compression
(once for the initramfs file embedded into kernel and then again for the whole kernel+initramfs thing)
would be waste of time.
But for initramfs files *outside* kernel these two new compression methods are very usefull.
lzo being just barely better in compression than gzip but fastest decompression of all,
while xz being the best compression.
(So lzo replaces gzip and xz replaces bzip2 and lzma)
I use xz for kernel itself and also for outside initramfs
Quoted 4 lines
I noticed when checking the bzImage size in a tweaked
qemu_x86_defconfigtest run that it never changed much.
New patchset will follow that fixes this and the previous issue. Regards.
Stefan
PATCH 3/3 initramfs: add support for LZO and XZ compression methods
In reply to Gustavo ZacariasDear Gustavo Zacarias,
fs/cpio/Config.in choice · line 46
Quoted 9 lines · Gustavo Zacarias
fs/cpio/Config.in +9
-0+config BR2_TARGET_ROOTFS_CPIO_LZO + bool "lzo" + help + Do compress the cpio filesystem with lzo. + +config BR2_TARGET_ROOTFS_CPIO_XZ + bool "xz" + help + Do compress the cpio filesystem with xz.
I don't remember exactly, but isn't the kernel build process itself capable of compressing the initramfs already? I remember we had some discussion about this a while ago, but I don't remember the outcome of these. Arnout was leading this discussion, IIRC.
Best regards,
Thomas
Signature · 4 lines
Thomas Petazzoni, Free Electrons Kernel, drivers, real-time and embedded Linux development, consulting, training and support. http://free-electrons.com
PATCH 3/3 initramfs: add support for LZO and XZ compression methods
22.1.2013 19:05, Thomas Petazzoni kirjoitti:
Quoted 4 lines
Dear Gustavo Zacarias,
On Mon, 21 Jan 2013 19:55:43 -0300, Gustavo Zacarias wrote:
Quoted 13 lines
fs/cpio/Config.in choice · line 46
Quoted 9 lines
fs/cpio/Config.in +9
-0+config BR2_TARGET_ROOTFS_CPIO_LZO + bool "lzo" + help + Do compress the cpio filesystem with lzo. + +config BR2_TARGET_ROOTFS_CPIO_XZ + bool "xz" + help + Do compress the cpio filesystem with xz.I don't remember exactly, but isn't the kernel build process itself capable of compressing the initramfs already? I remember we had some discussion about this a while ago, but I don't remember the outcome of these. Arnout was leading this discussion, IIRC.
If it's embedded initramfs then yes, kernel can handle it all itself just nicely.
without needing to compress inside initramfs.
(images kernel.png and kernel2.png)
But if outside cpio-archive (aka initramfs) then no. It has to be done by hand.
And like I said to Gustavo it's bad to do double compression.
Here's are the different combinations:
1 ) compressed initramfs inside compressed kernel = bad (waste of time decompressing twice, and also waste of package file header size overhead) 2 ) compressed initramfs inside uncompressed kernel = better (no time waste but still space waste)
3 ) uncompressed initramfs inside compressed kernel = best solution for embedded initramfs (no time waste & time space waste)
4 ) uncompressed outside initramfs and uncompressed kernel (waste of space and even according to some benchmarks, waste of time) 5 ) compressed outside initramfs and compressed kernel (no wasted time and no wasted space)
So option 3 is best for embedded initramfs and option 5 for otherwise for those who want more flexibility with their enivronment. For example I could have this:
-------------------------------------------------------------------------------------------------------------------------------------------------
| embedded, uncompressed initramfs insize xz-compressed kernel, with
just busybox and minimal stuff |
--------------------------------------------------------------------------------------------------------------------------------------------------
|
---------------------------------------------------------------------------------------------------------------------------------
| compressed outside initramfs with ncurses & directfb application,
like links web-browser |
---------------------------------------------------------------------------------------------------------------------------------
|
----------------------------------------------------------------------------------------------------------------------------------
| still another compressed initramfs with Xorg, firefox and all the
heavy stuff |
----------------------------------------------------------------------------------------------------------------------------------
All stackable one top over another (at least grub legacy allows giving several initramfs).
And in Grub conf I could have boot entries like:
- Minimal system (just busybox and it's apps)
- Light system (ncurses, directfb and other non-Xorg "graphical" apps)
- Full system (full Xorg stuff and apps)
Without outside initramfs option I would have to stick kernel (and it's modules) to each of those initramfs files which would be a waste of space. Now I have only one kernel (with or without initramfs) and 2 outside initramfs which I can mix and match like lego blocks.
Stefan
Quoted 3 lines
Best regards,
Thomas
-------------- next part --------------
A non-text attachment was scrubbed...
Name: kernel.png
Type: image/png
Size: 24654 bytes
Desc: not available
URL: <http://lists.busybox.net/pipermail/buildroot/attachments/20130122/3fb45fea/attachment-0002.png>
-------------- next part --------------
A non-text attachment was scrubbed...
Name: kernel2.png
Type: image/png
Size: 24921 bytes
Desc: not available
URL: <http://lists.busybox.net/pipermail/buildroot/attachments/20130122/3fb45fea/attachment-0003.png>
PATCH 3/3 initramfs: add support for LZO and XZ compression methods
On 01/22/13 19:17, Stefan Fr?berg wrote:
[snip]
Quoted 1 line
Here's are the different combinations:
Excellent explanation! How about adding this to the documentation?
Regards,
Arnout
Signature · 6 lines
Arnout Vandecappelle arnout at mind be Senior Embedded Software Architect +32-16-286500 Essensium/Mind http://www.mind.be G.Geenslaan 9, 3001 Leuven, Belgium BE 872 984 063 RPR Leuven LinkedIn profile: http://www.linkedin.com/in/arnoutvandecappelle GPG fingerprint: 7CB5 E4CC 6C2E EFD4 6E3D A754 F963 ECAB 2450 2F1F
PATCH 3/3 initramfs: add support for LZO and XZ compression methods
24.1.2013 9:11, Arnout Vandecappelle kirjoitti:
Quoted 8 lines · Stefan Fr?berg
[snip]
Quoted 1 line
Here's are the different combinations:
Excellent explanation! How about adding this to the documentation?
Regards,
Arnout
Well, uh...ok.
But before that I have to finish that <pkg>_CONFIG_FIXUP thing first. I think ill just make one final, v3 version, of it with fixed doc and your sed changes
as one single combined patch.
And then Im going to test it one final time before resubmit.
Regards
Stefan
PATCH 3/3 initramfs: add support for LZO and XZ compression methods
In reply to Stefan FröbergQuoted 5 lines · Stefan Fr?berg
Quoted 4 lines
I don't remember exactly, but isn't the kernel build process itself capable of compressing the initramfs already? I remember we had some discussion about this a while ago, but I don't remember the outcome of these. Arnout was leading this discussion, IIRC.
Quoted 8 lines
If it's embedded initramfs then yes, kernel can handle it all itself just nicely.
without needing to compress inside initramfs.
(images kernel.png and kernel2.png)But if outside cpio-archive (aka initramfs) then no. It has to be done by hand.
And like I said to Gustavo it's bad to do double compression.
Well, it's not always bad to do double compression.
On some systems where touching the bootloader isn't an option with say, uboot, if uboot doesn't understand the newer compression method you're most likely stuck with gzip or none for the uImage.
So doing xz for the initramfs in that case would be a space saver - even if you're doing double (de)compression.
Granted, you'll pay it dearly in cpu time.
I started looking at this because someone in #elinux started asking about making the tinyest possible kernel+initramfs for x86 for pxeboot with some packages and i saw we were lacking options to go for the best. Also noticed we weren't doing anything about the compressed cpio archive so the resulting combined kernel was roughly always the same size. For some unknown reason he wanted everything to fit in a 1.8 MB file - maybe some PXE limitation, i don't know.
Question is, should we care about those uses cases or not? Regards.
PATCH 3/3 initramfs: add support for LZO and XZ compression methods
24.1.2013 12:48, Gustavo Zacarias kirjoitti:
Quoted 13 lines · Stefan Fr?berg
Quoted 4 lines
Quoted 4 lines
I don't remember exactly, but isn't the kernel build process itself capable of compressing the initramfs already? I remember we had some discussion about this a while ago, but I don't remember the outcome of these. Arnout was leading this discussion, IIRC.
Quoted 8 lines
If it's embedded initramfs then yes, kernel can handle it all itself just nicely.
without needing to compress inside initramfs.
(images kernel.png and kernel2.png)But if outside cpio-archive (aka initramfs) then no. It has to be done by hand.
And like I said to Gustavo it's bad to do double compression.
Quoted 16 lines
Well, it's not always bad to do double compression.
On some systems where touching the bootloader isn't an option with say, uboot, if uboot doesn't understand the newer compression method you're most likely stuck with gzip or none for the uImage.
So doing xz for the initramfs in that case would be a space saver - even if you're doing double (de)compression.
Granted, you'll pay it dearly in cpu time.
I started looking at this because someone in #elinux started asking about making the tinyest possible kernel+initramfs for x86 for pxeboot with some packages and i saw we were lacking options to go for the best. Also noticed we weren't doing anything about the compressed cpio archive so the resulting combined kernel was roughly always the same size. For some unknown reason he wanted everything to fit in a 1.8 MB file - maybe some PXE limitation, i don't know.
Question is, should we care about those uses cases or not? Regards.
I think users should be allowed to do every possible combination they want to.
But also give them big fat warning in cases when their choices might drop performance
so that they are perfectly aware of consequences of the choices they made in config.
So yeah, double (de)compression is option too but with big fat warning added that might
affect your system speed.
User is the King.
Stefan
PATCH 3/3 initramfs: add support for LZO and XZ compression methods
Quoted 11 lines · Stefan Fr?berg
I think users should be allowed to do every possible combination they want to.
But also give them big fat warning in cases when their choices might drop performance
so that they are perfectly aware of consequences of the choices they made in config.
So yeah, double (de)compression is option too but with big fat warning added that might
affect your system speed.User is the King.
At this point my patchset (v2) does nothing to address the issue, so it's safe to commit.
I was leaving that functionality for a separate one afterwards if the consensus is "go for it".
Regards.
PATCH 3/3 initramfs: add support for LZO and XZ compression methods
In reply to Gustavo ZacariasQuoted 11 lines · Gustavo Zacarias
Signed-off-by Gustavo Zacarias<[email protected]>--- fs/common.mk | 8 +++++++- fs/cpio/Config.in | 18 ++++++++++++++---- 2 files changed, 21 insertions(+), 5 deletions(-)fs/common.mk +0
-0
fs/common.mk ROOTFS_DEVICE_TABLES = $(call qstrip,$(BR2_ROOTFS_DEVICE_TABLE)) \ · line 42
Quoted 5 lines
fs/common.mk +1
-1@@ -39,7 +39,7 @@ ROOTFS_DEVICE_TABLES = $(call qstrip,$(BR2_ROOTFS_DEVICE_TABLE)) \ define ROOTFS_TARGET_INTERNAL # extra deps -ROOTFS_$(2)_DEPENDENCIES += host-fakeroot host-makedevs $$(if $$(BR2_TARGET_ROOTFS_$(2)_LZMA),host-lzma) +ROOTFS_$(2)_DEPENDENCIES += host-fakeroot host-makedevs $$(if $$(BR2_TARGET_ROOTFS_$(2)_LZMA),host-lzma) $$(if $$(BR2_TARGET_ROOTFS_$(2)_LZO),host-lzop) $$(if $$(BR2_TARGET_ROOTFS_$(2)_XZ),host-xz)
Gustavo,
If you were to submit a v4, could you split this very long line?
Regards,
Arnout
Signature · 6 lines
Arnout Vandecappelle arnout at mind be Senior Embedded Software Architect +32-16-286500 Essensium/Mind http://www.mind.be G.Geenslaan 9, 3001 Leuven, Belgium BE 872 984 063 RPR Leuven LinkedIn profile: http://www.linkedin.com/in/arnoutvandecappelle GPG fingerprint: 7CB5 E4CC 6C2E EFD4 6E3D A754 F963 ECAB 2450 2F1F
PATCH 3/3 initramfs: add support for LZO and XZ compression methods
Quoted 6 lines · Arnout Vandecappelle
Gustavo,
If you were to submit a v4, could you split this very long line?
Regards,
Arnout
v4? There's not even a v3 yet! ;-)
There are many fixable lines inside fs/ that are easily wrappable. For example in ubifs/ubifs.mk, cloop/cloop.mk, initramfs/initramfs.mk (if we're going for the strict 80 column standard) and others. But yes, on a new patchset if necessary i could.
Regards.