Skip to contentSkip to conversation

pv-grub: Support bzip2, xz and lzo compressed kernels

PATCH In review Ian Campbell 7 participants 19 messages Jan 4, 2013 – Sep 10, 2013

lore
Share & tools
lore
#0 lore raw

PATCH 1/1 pv-grub: Support bzip2, xz and lzo compressed kernels

xz compressed kernels seem to be what we are starting to see in the wild, but we may as well support everything that libxc does in pv-grub.

Also update the zlib library to the latest version

---
This is only build tested as I don't have a convenient pv-grub setup at the moment.

As well as committing the committer should fetch the following URLs to xenbits.xen.org:/var/xenbits-www/html/xen-extfiles
- http://www.zlib.net/zlib-1.2.7.tar.gz
- http://www.bzip.org/1.0.6/bzip2-1.0.6.tar.gz
md5sum 00b516f4704d4a7cb50a1d97e6e8e15b
- http://tukaani.org/xz/xz-5.0.4.tar.bz2
http://tukaani.org/xz/xz-5.0.4.tar.bz2.sig
- http://www.oberhumer.com/opensource/lzo/download/lzo-2.06.tar.gz

---
 .gitignore              |    3 ++
 .hgignore               |    3 ++
 extras/mini-os/Makefile |    3 ++
 stubdom/Makefile        |   84 +++++++++++++++++++++++++++++++++++++++++++++--
 stubdom/bzip2.patch     |   11 ++++++
 stubdom/grub/kexec.c    |    6 +++
 6 files changed, 107 insertions(+), 3 deletions(-)
 create mode 100644 stubdom/bzip2.patch
.gitignore +3 -0
index f71cff8..a95183a 100644
@@ -107,6 +107,9 @@ stubdom/newlib-1.*
 stubdom/newlib-x86*
 stubdom/pciutils-*
 stubdom/zlib-*
+stubdom/bzip2-*
+stubdom/xz-*
+stubdom/lzo-*
 stubdom/grub-*
 stubdom/ocaml-*
 stubdom/lwip/
.hgignore +3 -0
index 344792a..044a88a 100644
@@ -102,6 +102,9 @@
 ^stubdom/newlib-.*$
 ^stubdom/pciutils-.*$
 ^stubdom/zlib-.*$
+^stubdom/bzip2-.*$
+^stubdom/xz-.*$
+^stubdom/lzo-.*$
 ^stubdom/grub-.*$
 ^stubdom/ocaml-.*$
 ^stubdom/lwip/
extras/mini-os/Makefile +3 -0
index 2302a23..717b209 100644
@@ -159,6 +159,9 @@ APP_LDLIBS += -L$(XEN_ROOT)/stubdom/libxc-$(XEN_TARGET_ARCH) -whole-archive -lxe
 endif
 APP_LDLIBS += -lpci
 APP_LDLIBS += -lz
+APP_LDLIBS += -lbz2
+APP_LDLIBS += -llzma
+APP_LDLIBS += -llzo2
 APP_LDLIBS += -lm
 LDLIBS += -lc
 endif
stubdom/Makefile +81 -3

104 lines not on this page · raw message

stubdom/bzip2.patch new file +11 -0
new file mode 100644
index 0000000..ee0df30
@@ -0,0 +1,11 @@
+--- bzip2-x86_64.orig/Makefile	2013-01-04 10:56:57.000000000 +0000
++++ bzip2-x86_64/Makefile	2013-01-04 10:57:06.000000000 +0000
+@@ -21,7 +21,7 @@
+ LDFLAGS=
+ 
+ BIGFILES=-D_FILE_OFFSET_BITS=64
+-CFLAGS=-Wall -Winline -O2 -g $(BIGFILES)
++CFLAGS=-Wall -Winline -O2 -g $(BIGFILES) -DBZ_NO_STDIO
+ 
+ # Where you want it installed when you do 'make install'
+ PREFIX=/usr/local
stubdom/grub/kexec.c +6 -0
index b21c91a..9d10d32 100644
@@ -54,6 +54,12 @@ static unsigned long allocated;
 int pin_table(xc_interface *xc_handle, unsigned int type, unsigned long mfn,
               domid_t dom);
 
+/* Required by libbz2 */
+void bz_internal_error(int errcode)
+{
+    printk("BZIP2: Internal Error %d", __FUNCTION__);
+}
+
 /* We need mfn to appear as target_pfn, so exchange with the MFN there */
 static void do_exchange(struct xc_dom_image *dom, xen_pfn_t target_pfn, xen_pfn_t source_mfn)
 {
Signature · 1 line
1.7.2.5
#1 lore raw

stubdom/Makefile $(ZLIB_STAMPFILE): zlib-$(XEN_TARGET_ARCH) $(NEWLIB_STAMPFILE) · line 169

Quoted 9 lines · Ian Campbell, le Fri 04 Jan 2013 11:56:33 +0000, a écrit :
stubdom/Makefile +9 -0
+BZIP2_STAMPFILE=$(CROSS_ROOT)/$(GNU_TARGET_ARCH)-xen-elf/lib/libbz2.a
+.PHONY: cross-bzip2
+cross-bzip2: $(BZIP2_STAMPFILE)
+$(BZIP2_STAMPFILE): bzip2-$(XEN_TARGET_ARCH) $(NEWLIB_STAMPFILE)
+	( cd $< && \
+	  $(MAKE) DESTDIR= libbz2.a && \
+	  $(INSTALL_DATA) libbz2.a $(CROSS_PREFIX)/$(GNU_TARGET_ARCH)-xen-elf/lib/ && \
+	  $(INSTALL_DATA) bzlib.h $(CROSS_PREFIX)/$(GNU_TARGET_ARCH)-xen-elf/include/ \
+	)

This is missing the cross-compilation flags, isn't it?

Samuel

#2 lore raw
Quoted 12 lines · Samuel Thibault

Review

Quoted 9 lines · Ian Campbell, le Fri 04 Jan 2013 11:56:33 +0000, a écrit :
(quoted diff) +9 -0
+BZIP2_STAMPFILE=$(CROSS_ROOT)/$(GNU_TARGET_ARCH)-xen-elf/lib/libbz2.a
+.PHONY: cross-bzip2
+cross-bzip2: $(BZIP2_STAMPFILE)
+$(BZIP2_STAMPFILE): bzip2-$(XEN_TARGET_ARCH) $(NEWLIB_STAMPFILE)
+	( cd $< && \
+	  $(MAKE) DESTDIR= libbz2.a && \
+	  $(INSTALL_DATA) libbz2.a $(CROSS_PREFIX)/$(GNU_TARGET_ARCH)-xen-elf/lib/ && \
+	  $(INSTALL_DATA) bzlib.h $(CROSS_PREFIX)/$(GNU_TARGET_ARCH)-xen-elf/include/ \
+	)

This is missing the cross-compilation flags, isn't it?

Oops, yes, I cut them out with the unneeded configure by mistake!

8<----------------------------------------

From d7116c4f19c54613f28a3bd44c84fef1b3340a1e Mon Sep 17 00:00:00 2001
From: Ian Campbell <[email protected]>
Date: Fri, 4 Jan 2013 11:37:31 +0000
Subject: [PATCH] pv-grub: Support bzip2, xz and lzo compressed kernels

xz compressed kernels seem to be what we are starting to see in the wild, but we may as well support everything that libxc does in pv-grub.

Also update the zlib library to the latest version

---
v2: Pass cross-compile flags to bzip2 build.

This is only build tested as I don't have a convenient pv-grub setup at the moment.

As well as committing the committer should fetch the following URLs to xenbits.xen.org:/var/xenbits-www/html/xen-extfiles
- http://www.zlib.net/zlib-1.2.7.tar.gz
- http://www.bzip.org/1.0.6/bzip2-1.0.6.tar.gz
md5sum 00b516f4704d4a7cb50a1d97e6e8e15b
- http://tukaani.org/xz/xz-5.0.4.tar.bz2
http://tukaani.org/xz/xz-5.0.4.tar.bz2.sig
- http://www.oberhumer.com/opensource/lzo/download/lzo-2.06.tar.gz

---
 .gitignore              |    3 ++
 .hgignore               |    3 ++
 extras/mini-os/Makefile |    3 ++
 stubdom/Makefile        |   84 +++++++++++++++++++++++++++++++++++++++++++++--
 stubdom/bzip2.patch     |   11 ++++++
 stubdom/grub/kexec.c    |    6 +++
 6 files changed, 107 insertions(+), 3 deletions(-)
 create mode 100644 stubdom/bzip2.patch
.gitignore +3 -0
index f71cff8..a95183a 100644
@@ -107,6 +107,9 @@ stubdom/newlib-1.*
 stubdom/newlib-x86*
 stubdom/pciutils-*
 stubdom/zlib-*
+stubdom/bzip2-*
+stubdom/xz-*
+stubdom/lzo-*
 stubdom/grub-*
 stubdom/ocaml-*
 stubdom/lwip/
.hgignore +3 -0
index 344792a..044a88a 100644
@@ -102,6 +102,9 @@
 ^stubdom/newlib-.*$
 ^stubdom/pciutils-.*$
 ^stubdom/zlib-.*$
+^stubdom/bzip2-.*$
+^stubdom/xz-.*$
+^stubdom/lzo-.*$
 ^stubdom/grub-.*$
 ^stubdom/ocaml-.*$
 ^stubdom/lwip/
extras/mini-os/Makefile +3 -0
index 2302a23..717b209 100644
@@ -159,6 +159,9 @@ APP_LDLIBS += -L$(XEN_ROOT)/stubdom/libxc-$(XEN_TARGET_ARCH) -whole-archive -lxe
 endif
 APP_LDLIBS += -lpci
 APP_LDLIBS += -lz
+APP_LDLIBS += -lbz2
+APP_LDLIBS += -llzma
+APP_LDLIBS += -llzo2
 APP_LDLIBS += -lm
 LDLIBS += -lc
 endif
stubdom/Makefile +81 -3

104 lines not on this page · raw message

stubdom/bzip2.patch new file +11 -0
new file mode 100644
index 0000000..2eb0416
@@ -0,0 +1,11 @@
+--- bzip2-x86_64.orig/Makefile	2013-01-04 10:56:57.000000000 +0000
++++ bzip2-x86_64/Makefile	2013-01-04 10:57:06.000000000 +0000
+@@ -21,7 +21,7 @@
+ LDFLAGS=
+ 
+ BIGFILES=-D_FILE_OFFSET_BITS=64
+-CFLAGS=-Wall -Winline -O2 -g $(BIGFILES)
++CFLAGS=$(CROSS_CFLAGS) -Wall -Winline -O2 -g $(BIGFILES)
+ 
+ # Where you want it installed when you do 'make install'
+ PREFIX=/usr/local
stubdom/grub/kexec.c +6 -0
index b21c91a..9d10d32 100644
@@ -54,6 +54,12 @@ static unsigned long allocated;
 int pin_table(xc_interface *xc_handle, unsigned int type, unsigned long mfn,
               domid_t dom);
 
+/* Required by libbz2 */
+void bz_internal_error(int errcode)
+{
+    printk("BZIP2: Internal Error %d", __FUNCTION__);
+}
+
 /* We need mfn to appear as target_pfn, so exchange with the MFN there */
 static void do_exchange(struct xc_dom_image *dom, xen_pfn_t target_pfn, xen_pfn_t source_mfn)
 {
Signature · 9 lines
1.7.2.5




_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xen.org/xen-devel
Reviewed-by Tested-by Ben Cressey
#4 lore raw
In reply to Ian Campbell
Quoted 23 lines · Ian Campbell
Quoted 12 lines · Samuel Thibault

stubdom/Makefile $(ZLIB_STAMPFILE): zlib-$(XEN_TARGET_ARCH) $(NEWLIB_STAMPFILE) · line 169

Quoted 9 lines · Ian Campbell, le Fri 04 Jan 2013 11:56:33 +0000, a écrit :
stubdom/Makefile +9 -0
+BZIP2_STAMPFILE=$(CROSS_ROOT)/$(GNU_TARGET_ARCH)-xen-elf/lib/libbz2.a
+.PHONY: cross-bzip2
+cross-bzip2: $(BZIP2_STAMPFILE)
+$(BZIP2_STAMPFILE): bzip2-$(XEN_TARGET_ARCH) $(NEWLIB_STAMPFILE)
+	( cd $< && \
+	  $(MAKE) DESTDIR= libbz2.a && \
+	  $(INSTALL_DATA) libbz2.a $(CROSS_PREFIX)/$(GNU_TARGET_ARCH)-xen-elf/lib/ && \
+	  $(INSTALL_DATA) bzlib.h $(CROSS_PREFIX)/$(GNU_TARGET_ARCH)-xen-elf/include/ \
+	)

This is missing the cross-compilation flags, isn't it?

Oops, yes, I cut them out with the unneeded configure by mistake!

8<----------------------------------------

From d7116c4f19c54613f28a3bd44c84fef1b3340a1e Mon Sep 17 00:00:00 2001
From: Ian Campbell <[email protected]>
Date: Fri, 4 Jan 2013 11:37:31 +0000
Subject: [PATCH] pv-grub: Support bzip2, xz and lzo compressed kernels

Quoted 7 lines

xz compressed kernels seem to be what we are starting to see in the wild, but we may as well support everything that libxc does in pv-grub.

Also update the zlib library to the latest version

Ian, this doesn't seem to have been committed. This looks good, and Ben has tested booting lzo, xz and bz2 compressed kernels using PV-GRUB with this patch applied.

Quoted 212 lines

---
v2: Pass cross-compile flags to bzip2 build.

This is only build tested as I don't have a convenient pv-grub setup at the moment.

As well as committing the committer should fetch the following URLs to xenbits.xen.org:/var/xenbits-www/html/xen-extfiles
- http://www.zlib.net/zlib-1.2.7.tar.gz
- http://www.bzip.org/1.0.6/bzip2-1.0.6.tar.gz
md5sum 00b516f4704d4a7cb50a1d97e6e8e15b
- http://tukaani.org/xz/xz-5.0.4.tar.bz2
http://tukaani.org/xz/xz-5.0.4.tar.bz2.sig
- http://www.oberhumer.com/opensource/lzo/download/lzo-2.06.tar.gz

---
 .gitignore              |    3 ++
 .hgignore               |    3 ++
 extras/mini-os/Makefile |    3 ++
 stubdom/Makefile        |   84 +++++++++++++++++++++++++++++++++++++++++++++--
 stubdom/bzip2.patch     |   11 ++++++
 stubdom/grub/kexec.c    |    6 +++
 6 files changed, 107 insertions(+), 3 deletions(-)
 create mode 100644 stubdom/bzip2.patch
.gitignore +3 -0
index f71cff8..a95183a 100644
@@ -107,6 +107,9 @@ stubdom/newlib-1.*
 stubdom/newlib-x86*
 stubdom/pciutils-*
 stubdom/zlib-*
+stubdom/bzip2-*
+stubdom/xz-*
+stubdom/lzo-*
 stubdom/grub-*
 stubdom/ocaml-*
 stubdom/lwip/
.hgignore +3 -0
index 344792a..044a88a 100644
@@ -102,6 +102,9 @@
 ^stubdom/newlib-.*$
 ^stubdom/pciutils-.*$
 ^stubdom/zlib-.*$
+^stubdom/bzip2-.*$
+^stubdom/xz-.*$
+^stubdom/lzo-.*$
 ^stubdom/grub-.*$
 ^stubdom/ocaml-.*$
 ^stubdom/lwip/
extras/mini-os/Makefile +3 -0
index 2302a23..717b209 100644
@@ -159,6 +159,9 @@ APP_LDLIBS += -L$(XEN_ROOT)/stubdom/libxc-$(XEN_TARGET_ARCH) -whole-archive -lxe
 endif
 APP_LDLIBS += -lpci
 APP_LDLIBS += -lz
+APP_LDLIBS += -lbz2
+APP_LDLIBS += -llzma
+APP_LDLIBS += -llzo2
 APP_LDLIBS += -lm
 LDLIBS += -lc
 endif
stubdom/Makefile +81 -3
index 50ba360..c85dea2 100644
@@ -9,7 +9,19 @@ include $(XEN_ROOT)/Config.mk
 
 #ZLIB_URL?=http://www.zlib.net
 ZLIB_URL=$(XEN_EXTFILES_URL)
-ZLIB_VERSION=1.2.3
+ZLIB_VERSION=1.2.7
+
+#BZIP2_URL?=http://www.bzip.org/$(BZIP2_VERSION)
+BZIP2_URL=$(XEN_EXTFILES_URL)
+BZIP2_VERSION=1.0.6
+
+#XZ_URL?=http://tukaani.org/xz
+XZ_URL=$(XEN_EXTFILES_URL)
+XZ_VERSION=5.0.4
+
+#LZO_URL?=http://www.oberhumer.com/opensource/lzo/download
+LZO_URL=$(XEN_EXTFILES_URL)
+LZO_VERSION=2.06
 
 #LIBPCI_URL?=http://www.kernel.org/pub/software/utils/pciutils
 LIBPCI_URL?=$(XEN_EXTFILES_URL)
@@ -134,6 +146,69 @@ $(ZLIB_STAMPFILE): zlib-$(XEN_TARGET_ARCH) $(NEWLIB_STAMPFILE)
 	  $(MAKE) DESTDIR= libz.a && \
 	  $(MAKE) DESTDIR= install )
 
+############
+# Cross-bzip2
+############
+
+bzip2-$(BZIP2_VERSION).tar.gz:
+	$(WGET) $(BZIP2_URL)/$@
+
+bzip2-$(XEN_TARGET_ARCH): bzip2-$(BZIP2_VERSION).tar.gz
+	tar xzf $<
+	mv bzip2-$(BZIP2_VERSION) $@
+	patch -d $@ -p1 < bzip2.patch
+
+BZIP2_STAMPFILE=$(CROSS_ROOT)/$(GNU_TARGET_ARCH)-xen-elf/lib/libbz2.a
+.PHONY: cross-bzip2
+cross-bzip2: $(BZIP2_STAMPFILE)
+$(BZIP2_STAMPFILE): bzip2-$(XEN_TARGET_ARCH) $(NEWLIB_STAMPFILE)
+	( cd $< && \
+	  $(MAKE) CROSS_CFLAGS="$(TARGET_CPPFLAGS) $(TARGET_CFLAGS) -DBZ_NO_STDIO" CC=$(CC) DESTDIR= libbz2.a && \
+	  $(INSTALL_DATA) libbz2.a $(CROSS_PREFIX)/$(GNU_TARGET_ARCH)-xen-elf/lib/ && \
+	  $(INSTALL_DATA) bzlib.h $(CROSS_PREFIX)/$(GNU_TARGET_ARCH)-xen-elf/include/ \
+	)
+
+##############
+# Cross-xz
+##############
+
+xz-$(XZ_VERSION).tar.bz2:
+	$(WGET) $(XZ_URL)/$@
+
+xz-$(XEN_TARGET_ARCH): xz-$(XZ_VERSION).tar.bz2
+	tar xjf $<
+	mv xz-$(XZ_VERSION) $@
+
+XZ_STAMPFILE=$(CROSS_ROOT)/$(GNU_TARGET_ARCH)-xen-elf/lib/liblzma.a
+.PHONY: cross-xz
+cross-xz: $(XZ_STAMPFILE)
+$(XZ_STAMPFILE): xz-$(XEN_TARGET_ARCH) $(NEWLIB_STAMPFILE)
+	( cd $< && \
+	  CFLAGS="$(TARGET_CPPFLAGS) $(TARGET_CFLAGS)" CC=$(CC) ./configure --prefix=$(CROSS_PREFIX)/$(GNU_TARGET_ARCH)-xen-elf && \
+	  cd src/liblzma && \
+	  $(MAKE) DESTDIR= liblzma.la && \
+	  $(MAKE) DESTDIR= install )
+
+##############
+# Cross-lzo
+##############
+
+lzo-$(LZO_VERSION).tar.gz:
+	$(WGET) $(LZO_URL)/$@
+
+lzo-$(XEN_TARGET_ARCH): lzo-$(LZO_VERSION).tar.gz
+	tar xzf $<
+	mv lzo-$(LZO_VERSION) $@
+
+LZO_STAMPFILE=$(CROSS_ROOT)/$(GNU_TARGET_ARCH)-xen-elf/lib/liblzo2.a
+.PHONY: cross-lzo
+cross-lzo: $(LZO_STAMPFILE)
+$(LZO_STAMPFILE): lzo-$(XEN_TARGET_ARCH) $(NEWLIB_STAMPFILE)
+	( cd $< && \
+	  CFLAGS="$(TARGET_CPPFLAGS) $(TARGET_CFLAGS)" CC=$(CC) ./configure --prefix=$(CROSS_PREFIX)/$(GNU_TARGET_ARCH)-xen-elf && \
+	  $(MAKE) DESTDIR= src/liblzo2.la && \
+	  $(MAKE) DESTDIR= install-libLTLIBRARIES install-pkgincludeHEADERS )
+
 ##############
 # Cross-libpci
 ##############
@@ -281,10 +356,13 @@ $(TARGETS_MINIOS): mini-os-%:
 
 .PHONY: libxc
 libxc: libxc-$(XEN_TARGET_ARCH)/libxenctrl.a libxc-$(XEN_TARGET_ARCH)/libxenguest.a
-libxc-$(XEN_TARGET_ARCH)/libxenctrl.a: cross-zlib
+libxc-$(XEN_TARGET_ARCH)/libxenctrl.a: TARGET_CFLAGS += -DHAVE_BZLIB
+libxc-$(XEN_TARGET_ARCH)/libxenctrl.a: TARGET_CFLAGS += -DHAVE_LZMA
+libxc-$(XEN_TARGET_ARCH)/libxenctrl.a: TARGET_CFLAGS += -DHAVE_LZO1X
+libxc-$(XEN_TARGET_ARCH)/libxenctrl.a: cross-zlib cross-bzip2 cross-xz cross-lzo
 	CPPFLAGS="$(TARGET_CPPFLAGS)" CFLAGS="$(TARGET_CFLAGS)" $(MAKE) DESTDIR= -C libxc-$(XEN_TARGET_ARCH)
 
- libxc-$(XEN_TARGET_ARCH)/libxenguest.a: libxc-$(XEN_TARGET_ARCH)/libxenctrl.a
+libxc-$(XEN_TARGET_ARCH)/libxenguest.a: libxc-$(XEN_TARGET_ARCH)/libxenctrl.a
 
 #######
 # ioemu
stubdom/bzip2.patch new file +11 -0
new file mode 100644
index 0000000..2eb0416
@@ -0,0 +1,11 @@
+--- bzip2-x86_64.orig/Makefile	2013-01-04 10:56:57.000000000 +0000
++++ bzip2-x86_64/Makefile	2013-01-04 10:57:06.000000000 +0000
+@@ -21,7 +21,7 @@
+ LDFLAGS=
+ 
+ BIGFILES=-D_FILE_OFFSET_BITS=64
+-CFLAGS=-Wall -Winline -O2 -g $(BIGFILES)
++CFLAGS=$(CROSS_CFLAGS) -Wall -Winline -O2 -g $(BIGFILES)
+ 
+ # Where you want it installed when you do 'make install'
+ PREFIX=/usr/local
stubdom/grub/kexec.c +6 -0
index b21c91a..9d10d32 100644
@@ -54,6 +54,12 @@ static unsigned long allocated;
 int pin_table(xc_interface *xc_handle, unsigned int type, unsigned long mfn,
               domid_t dom);
 
+/* Required by libbz2 */
+void bz_internal_error(int errcode)
+{
+    printk("BZIP2: Internal Error %d", __FUNCTION__);
+}
+
 /* We need mfn to appear as target_pfn, so exchange with the MFN there */
 static void do_exchange(struct xc_dom_image *dom, xen_pfn_t target_pfn, xen_pfn_t source_mfn)
 {
#5 lore raw
Quoted 3 lines · Matt Wilson

Ian, this doesn't seem to have been committed. This looks good, and Ben has tested booting lzo, xz and bz2 compressed kernels using PV-GRUB with this patch applied.

Because this was superseeded by a completely different patch.

What reminds me: zlib may go to.

Bastian

Signature · 2 lines
Those who hate and fight must stop themselves -- otherwise it is not stopped.
		-- Spock, "Day of the Dove", stardate unknown
#6 lore raw
Quoted 6 lines · Bastian Blank
Quoted 3 lines · Matt Wilson

Ian, this doesn't seem to have been committed. This looks good, and Ben has tested booting lzo, xz and bz2 compressed kernels using PV-GRUB with this patch applied.

Because this was superseeded by a completely different patch.

Aah, this one:
http://xenbits.xen.org/gitweb/?p=xen.git;a=commitdiff;h=592d3ef9

Makes sense.

--msw

#7 lore raw
In reply to Ian Campbell
Quoted 4 lines · Ian Campbell

xz compressed kernels seem to be what we are starting to see in the wild, but we may as well support everything that libxc does in pv-grub.
Also update the zlib library to the latest version

Is there a reason not to use the implementations used by the hypervisor?

Bastian

Signature · 2 lines
Murder is contrary to the laws of man and God.
		-- M-5 Computer, "The Ultimate Computer", stardate 4731.3
#8 lore raw
Quoted 7 lines · Bastian Blank
Quoted 4 lines · Ian Campbell

xz compressed kernels seem to be what we are starting to see in the wild, but we may as well support everything that libxc does in pv-grub.
Also update the zlib library to the latest version

Is there a reason not to use the implementations used by the hypervisor?

Decompressing a potentially untrusted compressed file in the context of the hypervisor would open you up to all sorts of security issues. One of the points of pv-grub is that the untrusted code runs only with guest privileges.

Ian.

#9 lore raw
Quoted 6 lines · Ian Campbell
Quoted 1 line · Bastian Blank

Is there a reason not to use the implementations used by the hypervisor?

Decompressing a potentially untrusted compressed file in the context of the hypervisor would open you up to all sorts of security issues. One of the points of pv-grub is that the untrusted code runs only with guest privileges.

This was not what I meant, but you realized that already.

Anyway, I have some not really nice patches that integrate the hypervisor bzip2, lzma, xz and lzo code into pv-grub. gzip is not handled via this code path.

Bastian

Signature · 2 lines
Insufficient facts always invite danger.
		-- Spock, "Space Seed", stardate 3141.9
#10 lore raw

RFC 1/1 libxc: Move compression support into own file

Move the existing decompression methods into its own file.

tools/libxc/Makefile +3 -2
diff -r 66f563be41d9 tools/libxc/Makefile
@@ -58,6 +58,7 @@
 GUEST_SRCS-y                 += xc_dom_core.c xc_dom_boot.c
 GUEST_SRCS-y                 += xc_dom_elfloader.c
 GUEST_SRCS-$(CONFIG_X86)     += xc_dom_bzimageloader.c
+GUEST_SRCS-$(CONFIG_X86)     += xc_dom_decompress.c
 GUEST_SRCS-$(CONFIG_ARM)     += xc_dom_armzimageloader.c
 GUEST_SRCS-y                 += xc_dom_binloader.c
 GUEST_SRCS-y                 += xc_dom_compat_linux.c
@@ -182,8 +183,8 @@
 zlib-options = $(ZLIB)
 endif
 
-xc_dom_bzimageloader.o: CFLAGS += $(call zlib-options,D)
-xc_dom_bzimageloader.opic: CFLAGS += $(call zlib-options,D)
+xc_dom_decompress.o: CFLAGS += $(call zlib-options,D)
+xc_dom_decompress.opic: CFLAGS += $(call zlib-options,D)
 
 libxenguest.so.$(MAJOR).$(MINOR): COMPRESSION_LIBS = $(call zlib-options,l)
 libxenguest.so.$(MAJOR).$(MINOR): $(GUEST_PIC_OBJS) libxenctrl.so
tools/libxc/xc_dom.h +11 -0
diff -r 66f563be41d9 tools/libxc/xc_dom.h
@@ -293,6 +293,17 @@
 void xc_dom_unmap_one(struct xc_dom_image *dom, xen_pfn_t pfn);
 void xc_dom_unmap_all(struct xc_dom_image *dom);
 
+int xc_try_bzip2_decode(struct xc_dom_image *dom, void **blob, size_t *size)
+    __attribute__((visibility("hidden")));
+int xc_try_gzip_decode(struct xc_dom_image *dom, void **blob, size_t *size)
+    __attribute__((visibility("hidden")));
+int xc_try_lzma_decode(struct xc_dom_image *dom, void **blob, size_t *size)
+    __attribute__((visibility("hidden")));
+int xc_try_lzo1x_decode(struct xc_dom_image *dom, void **blob, size_t *size)
+    __attribute__((visibility("hidden")));
+int xc_try_xz_decode(struct xc_dom_image *dom, void **blob, size_t *size)
+    __attribute__((visibility("hidden")));
+
 static inline void *xc_dom_seg_to_ptr(struct xc_dom_image *dom,
                                       struct xc_dom_seg *seg)
 {
tools/libxc/xc_dom_bzimageloader.c +0 -527

533 lines not on this page · raw message

tools/libxc/xc_dom_decompress.c new file +567 -0

567 lines not on this page · raw message

#11 lore raw

RFC 1/1 libxc: Add trusted decompressors

In reply to Bastian Blank

Add trusted decompressors based on hypervisor code. This are used in mini-os by pv-grub.

stubdom/Makefile +1 -1
diff -r efdc4bbfdd20 stubdom/Makefile
@@ -328,7 +328,7 @@
 .PHONY: libxc
 libxc: libxc-$(XEN_TARGET_ARCH)/libxenctrl.a libxc-$(XEN_TARGET_ARCH)/libxenguest.a
 libxc-$(XEN_TARGET_ARCH)/libxenctrl.a: cross-zlib
-	CPPFLAGS="$(TARGET_CPPFLAGS)" CFLAGS="$(TARGET_CFLAGS)" $(MAKE) DESTDIR= -C libxc-$(XEN_TARGET_ARCH)
+	CPPFLAGS="$(TARGET_CPPFLAGS)" CFLAGS="$(TARGET_CFLAGS)" $(MAKE) DESTDIR= CONFIG_LIBXC_MINIOS=y -C libxc-$(XEN_TARGET_ARCH)
 
  libxc-$(XEN_TARGET_ARCH)/libxenguest.a: libxc-$(XEN_TARGET_ARCH)/libxenctrl.a
 
tools/libxc/Makefile +10 -1
diff -r efdc4bbfdd20 tools/libxc/Makefile
@@ -58,7 +58,6 @@
 GUEST_SRCS-y                 += xc_dom_core.c xc_dom_boot.c
 GUEST_SRCS-y                 += xc_dom_elfloader.c
 GUEST_SRCS-$(CONFIG_X86)     += xc_dom_bzimageloader.c
-GUEST_SRCS-$(CONFIG_X86)     += xc_dom_decompress.c
 GUEST_SRCS-$(CONFIG_ARM)     += xc_dom_armzimageloader.c
 GUEST_SRCS-y                 += xc_dom_binloader.c
 GUEST_SRCS-y                 += xc_dom_compat_linux.c
@@ -69,6 +68,16 @@
 GUEST_SRCS-$(CONFIG_ARM)     += xc_dom_arm.c
 GUEST_SRCS-$(CONFIG_ARM)     += xc_hvm_build_arm.c
 
+ifeq ($(CONFIG_LIBXC_MINIOS),y)
+GUEST_SRCS-y                 += xc_dom_decompress_trusted.c
+GUEST_SRCS-y                 += xc_dom_decompress_trusted_bzip2.c
+GUEST_SRCS-y                 += xc_dom_decompress_trusted_lzma.c
+GUEST_SRCS-y                 += xc_dom_decompress_trusted_lzo1x.c
+GUEST_SRCS-y                 += xc_dom_decompress_trusted_xz.c
+else
+GUEST_SRCS-y                 += xc_dom_decompress.c
+endif
+
 OSDEP_SRCS-y                 += xenctrl_osdep_ENOSYS.c
 
 -include $(XEN_TARGET_ARCH)/Makefile
tools/libxc/xc_dom_decompress_trusted.c new file +49 -0

49 lines not on this page · raw message

tools/libxc/xc_dom_decompress_trusted.h new file +12 -0
diff -r efdc4bbfdd20 tools/libxc/xc_dom_decompress_trusted.h
@@ -0,0 +1,12 @@
+#include "xc_dom.h"
+
+typedef int decompress_fn(unsigned char *inbuf, unsigned int len,
+                          int (*fill)(void*, unsigned int),
+                          int (*flush)(void*, unsigned int),
+                          unsigned char *outbuf, unsigned int *posp,
+                          void (*error)(const char *x));
+
+int xc_dom_decompress_trusted_decompress(
+    decompress_fn fn, struct xc_dom_image *dom, void **blob, size_t *size)
+    __attribute__((visibility("hidden")));
+
tools/libxc/xc_dom_decompress_trusted_bzip2.c new file +14 -0
diff -r efdc4bbfdd20 tools/libxc/xc_dom_decompress_trusted_bzip2.c
@@ -0,0 +1,14 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <inttypes.h>
+
+#include "xg_private.h"
+#include "xc_dom_decompress_trusted.h"
+
+#include "../../xen/common/bunzip2.c"
+
+int xc_try_bzip2_decode(
+    struct xc_dom_image *dom, void **blob, size_t *size)
+{
+    return xc_dom_decompress_trusted_decompress(bunzip2, dom, blob, size);
+}
tools/libxc/xc_dom_decompress_trusted_lzma.c new file +14 -0
diff -r efdc4bbfdd20 tools/libxc/xc_dom_decompress_trusted_lzma.c
@@ -0,0 +1,14 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <inttypes.h>
+
+#include "xg_private.h"
+#include "xc_dom_decompress_trusted.h"
+
+#include "../../xen/common/unlzma.c"
+
+int xc_try_lzma_decode(
+    struct xc_dom_image *dom, void **blob, size_t *size)
+{
+    return xc_dom_decompress_trusted_decompress(unlzma, dom, blob, size);
+}
tools/libxc/xc_dom_decompress_trusted_lzo1x.c new file +15 -0
diff -r efdc4bbfdd20 tools/libxc/xc_dom_decompress_trusted_lzo1x.c
@@ -0,0 +1,15 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <inttypes.h>
+
+#include "xg_private.h"
+#include "xc_dom_decompress_trusted.h"
+
+#include "../../xen/common/lzo.c"
+#include "../../xen/common/unlzo.c"
+
+int xc_try_lzo1x_decode(
+    struct xc_dom_image *dom, void **blob, size_t *size)
+{
+    return xc_dom_decompress_trusted_decompress(unlzo, dom, blob, size);
+}
tools/libxc/xc_dom_decompress_trusted_xz.c new file +17 -0
diff -r efdc4bbfdd20 tools/libxc/xc_dom_decompress_trusted_xz.c
@@ -0,0 +1,17 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <inttypes.h>
+
+#include "xg_private.h"
+#include "xc_dom_decompress_trusted.h"
+
+// TODO
+#define XZ_DEC_X86
+
+#include "../../xen/common/unxz.c"
+
+int xc_try_xz_decode(
+    struct xc_dom_image *dom, void **blob, size_t *size)
+{
+    return xc_dom_decompress_trusted_decompress(unxz, dom, blob, size);
+}
xen/common/decompress.h +13 -0
diff -r efdc4bbfdd20 xen/common/decompress.h
@@ -1,3 +1,5 @@
+#ifdef __XEN__
+
 #include <xen/config.h>
 #include <xen/cache.h>
 #include <xen/decompress.h>
@@ -15,3 +17,14 @@
 
 #define large_malloc xmalloc_bytes
 #define large_free xfree
+
+#else
+
+#define STATIC static
+#define INIT
+#define INITDATA
+
+#define large_malloc malloc
+#define large_free free
+
+#endif
xen/common/lzo.c +12 -0
diff -r efdc4bbfdd20 xen/common/lzo.c
@@ -68,7 +68,19 @@
  *  Richard Purdie <[email protected]>
  */
 
+#ifndef __MINIOS__
 #include <xen/types.h>
+#else
+#include <stdint.h>
+
+typedef uint32_t u32;
+typedef uint16_t u16;
+
+#define likely(a) a
+#define noinline
+#define unlikely(a) a
+#endif
+
 #include <xen/lzo.h>
 #define get_unaligned(_p) (*(_p))
 #define put_unaligned(_val,_p) (*(_p)=_val)
xen/common/unlzma.c +2 -0
diff -r efdc4bbfdd20 xen/common/unlzma.c
@@ -54,7 +54,9 @@
  * Copyright (c) 1999-2005  Igor Pavlov
  */
 
+#ifndef __MINIOS__
 #include <xen/compiler.h>
+#endif
 
 #define LZMA_IOBUF_SIZE	0x10000
 
xen/common/unlzo.c +36 -0
diff -r efdc4bbfdd20 xen/common/unlzo.c
@@ -32,7 +32,43 @@
 
 #include "decompress.h"
 #include <xen/lzo.h>
+
+#ifndef __MINIOS__
 #include <asm/byteorder.h>
+#else
+#include <endian.h>
+#include <stdint.h>
+
+typedef uint8_t u8;
+typedef uint16_t u16;
+typedef uint32_t u32;
+
+static inline u16 be16_to_cpup(const u16 *p)
+{
+	u16 v = *p;
+#if BYTE_ORDER == LITTLE_ENDIAN
+	return (((v & 0x00ffU) << 8) |
+                ((v & 0xff00U) >> 8));
+#else
+	return v;
+#endif
+}
+
+static inline u32 be32_to_cpup(const u32 *p)
+{
+	u32 v = *p;
+#if BYTE_ORDER == LITTLE_ENDIAN
+	return (((v & 0x000000ffUL) << 24) |
+                ((v & 0x0000ff00UL) <<  8) |
+                ((v & 0x00ff0000UL) >>  8) |
+                ((v & 0xff000000UL) >> 24));
+#else
+	return v;
+#endif
+}
+
+#define unlikely(a) a
+#endif
 
 #if 1 /* ndef CONFIG_??? */
 static inline u16 INIT get_unaligned_be16(void *p)
xen/common/xz/private.h +45 -0

53 lines not on this page · raw message

#12 lore raw

Re: [RFC] libxc: Add trusted decompressors

Quoted 2 lines · Bastian Blank

Add trusted decompressors based on hypervisor code. This are used in mini-os by pv-grub.

I think this is a reasonably pragmatic way to arrange the build.

I'm not sure "trusted" is quite the right term though, these aren't really any more trustworthy than the library supplied ones -- they are just more suitable for a mini-os environment.

I can't think of a better term though...

Quoted 352 lines

diff -r efdc4bbfdd20 stubdom/Makefile

stubdom/Makefile +1 -1
@@ -328,7 +328,7 @@
 .PHONY: libxc
 libxc: libxc-$(XEN_TARGET_ARCH)/libxenctrl.a libxc-$(XEN_TARGET_ARCH)/libxenguest.a
 libxc-$(XEN_TARGET_ARCH)/libxenctrl.a: cross-zlib
-	CPPFLAGS="$(TARGET_CPPFLAGS)" CFLAGS="$(TARGET_CFLAGS)" $(MAKE) DESTDIR= -C libxc-$(XEN_TARGET_ARCH)
+	CPPFLAGS="$(TARGET_CPPFLAGS)" CFLAGS="$(TARGET_CFLAGS)" $(MAKE) DESTDIR= CONFIG_LIBXC_MINIOS=y -C libxc-$(XEN_TARGET_ARCH)
 
  libxc-$(XEN_TARGET_ARCH)/libxenguest.a: libxc-$(XEN_TARGET_ARCH)/libxenctrl.a
 

diff -r efdc4bbfdd20 tools/libxc/Makefile

tools/libxc/Makefile +10 -1
@@ -58,7 +58,6 @@
 GUEST_SRCS-y                 += xc_dom_core.c xc_dom_boot.c
 GUEST_SRCS-y                 += xc_dom_elfloader.c
 GUEST_SRCS-$(CONFIG_X86)     += xc_dom_bzimageloader.c
-GUEST_SRCS-$(CONFIG_X86)     += xc_dom_decompress.c
 GUEST_SRCS-$(CONFIG_ARM)     += xc_dom_armzimageloader.c
 GUEST_SRCS-y                 += xc_dom_binloader.c
 GUEST_SRCS-y                 += xc_dom_compat_linux.c
@@ -69,6 +68,16 @@
 GUEST_SRCS-$(CONFIG_ARM)     += xc_dom_arm.c
 GUEST_SRCS-$(CONFIG_ARM)     += xc_hvm_build_arm.c
 
+ifeq ($(CONFIG_LIBXC_MINIOS),y)
+GUEST_SRCS-y                 += xc_dom_decompress_trusted.c
+GUEST_SRCS-y                 += xc_dom_decompress_trusted_bzip2.c
+GUEST_SRCS-y                 += xc_dom_decompress_trusted_lzma.c
+GUEST_SRCS-y                 += xc_dom_decompress_trusted_lzo1x.c
+GUEST_SRCS-y                 += xc_dom_decompress_trusted_xz.c
+else
+GUEST_SRCS-y                 += xc_dom_decompress.c
+endif
+
 OSDEP_SRCS-y                 += xenctrl_osdep_ENOSYS.c
 
 -include $(XEN_TARGET_ARCH)/Makefile

diff -r efdc4bbfdd20 tools/libxc/xc_dom_decompress_trusted.c

tools/libxc/xc_dom_decompress_trusted.c new file +49 -0
@@ -0,0 +1,49 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <inttypes.h>
+
+#include "xg_private.h"
+#include "xc_dom_decompress_trusted.h"
+
+struct xc_dom_image *trusted_dom;
+static unsigned char *output_blob;
+static unsigned int output_size;
+
+static void trusted_error(const char *msg)
+{
+    xc_dom_panic(trusted_dom->xch, XC_INVALID_KERNEL, "%s", msg);
+}
+
+static int trusted_flush(void *src, unsigned int size)
+{
+    void *n = realloc(output_blob, output_size + size);
+    if (!n)
+        return -1;
+    output_blob = n;
+
+    memcpy(&output_blob[output_size], src, size);
+    output_size += size;
+    return size;
+}
+
+int xc_dom_decompress_trusted_decompress(
+    decompress_fn fn, struct xc_dom_image *dom, void **blob, size_t *size)
+{
+    int ret;
+
+    trusted_dom = dom;
+    output_blob = NULL;
+    output_size = 0;
+
+    ret = fn(dom->kernel_blob, dom->kernel_size, NULL, trusted_flush, NULL, NULL, trusted_error);
+
+    if (ret)
+        free(output_blob);
+    else {
+        *blob = output_blob;
+        *size = output_size;
+    }
+
+    return ret;
+}
+

diff -r efdc4bbfdd20 tools/libxc/xc_dom_decompress_trusted.h

tools/libxc/xc_dom_decompress_trusted.h new file +12 -0
@@ -0,0 +1,12 @@
+#include "xc_dom.h"
+
+typedef int decompress_fn(unsigned char *inbuf, unsigned int len,
+                          int (*fill)(void*, unsigned int),
+                          int (*flush)(void*, unsigned int),
+                          unsigned char *outbuf, unsigned int *posp,
+                          void (*error)(const char *x));
+
+int xc_dom_decompress_trusted_decompress(
+    decompress_fn fn, struct xc_dom_image *dom, void **blob, size_t *size)
+    __attribute__((visibility("hidden")));
+

diff -r efdc4bbfdd20 tools/libxc/xc_dom_decompress_trusted_bzip2.c

tools/libxc/xc_dom_decompress_trusted_bzip2.c new file +14 -0
@@ -0,0 +1,14 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <inttypes.h>
+
+#include "xg_private.h"
+#include "xc_dom_decompress_trusted.h"
+
+#include "../../xen/common/bunzip2.c"
+
+int xc_try_bzip2_decode(
+    struct xc_dom_image *dom, void **blob, size_t *size)
+{
+    return xc_dom_decompress_trusted_decompress(bunzip2, dom, blob, size);
+}

diff -r efdc4bbfdd20 tools/libxc/xc_dom_decompress_trusted_lzma.c

tools/libxc/xc_dom_decompress_trusted_lzma.c new file +14 -0
@@ -0,0 +1,14 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <inttypes.h>
+
+#include "xg_private.h"
+#include "xc_dom_decompress_trusted.h"
+
+#include "../../xen/common/unlzma.c"
+
+int xc_try_lzma_decode(
+    struct xc_dom_image *dom, void **blob, size_t *size)
+{
+    return xc_dom_decompress_trusted_decompress(unlzma, dom, blob, size);
+}

diff -r efdc4bbfdd20 tools/libxc/xc_dom_decompress_trusted_lzo1x.c

tools/libxc/xc_dom_decompress_trusted_lzo1x.c new file +15 -0
@@ -0,0 +1,15 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <inttypes.h>
+
+#include "xg_private.h"
+#include "xc_dom_decompress_trusted.h"
+
+#include "../../xen/common/lzo.c"
+#include "../../xen/common/unlzo.c"
+
+int xc_try_lzo1x_decode(
+    struct xc_dom_image *dom, void **blob, size_t *size)
+{
+    return xc_dom_decompress_trusted_decompress(unlzo, dom, blob, size);
+}

diff -r efdc4bbfdd20 tools/libxc/xc_dom_decompress_trusted_xz.c

tools/libxc/xc_dom_decompress_trusted_xz.c new file +17 -0
@@ -0,0 +1,17 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <inttypes.h>
+
+#include "xg_private.h"
+#include "xc_dom_decompress_trusted.h"
+
+// TODO
+#define XZ_DEC_X86
+
+#include "../../xen/common/unxz.c"
+
+int xc_try_xz_decode(
+    struct xc_dom_image *dom, void **blob, size_t *size)
+{
+    return xc_dom_decompress_trusted_decompress(unxz, dom, blob, size);
+}

diff -r efdc4bbfdd20 xen/common/decompress.h

xen/common/decompress.h +13 -0
@@ -1,3 +1,5 @@
+#ifdef __XEN__
+
 #include <xen/config.h>
 #include <xen/cache.h>
 #include <xen/decompress.h>
@@ -15,3 +17,14 @@
 
 #define large_malloc xmalloc_bytes
 #define large_free xfree
+
+#else
+
+#define STATIC static
+#define INIT
+#define INITDATA
+
+#define large_malloc malloc
+#define large_free free
+
+#endif

diff -r efdc4bbfdd20 xen/common/lzo.c

xen/common/lzo.c +12 -0
@@ -68,7 +68,19 @@
  *  Richard Purdie <[email protected]>
  */
 
+#ifndef __MINIOS__
 #include <xen/types.h>
+#else
+#include <stdint.h>
+
+typedef uint32_t u32;
+typedef uint16_t u16;
+
+#define likely(a) a
+#define noinline
+#define unlikely(a) a
+#endif
+
 #include <xen/lzo.h>
 #define get_unaligned(_p) (*(_p))
 #define put_unaligned(_val,_p) (*(_p)=_val)

diff -r efdc4bbfdd20 xen/common/unlzma.c

xen/common/unlzma.c +2 -0
@@ -54,7 +54,9 @@
  * Copyright (c) 1999-2005  Igor Pavlov
  */
 
+#ifndef __MINIOS__
 #include <xen/compiler.h>
+#endif
 
 #define LZMA_IOBUF_SIZE	0x10000
 

diff -r efdc4bbfdd20 xen/common/unlzo.c

xen/common/unlzo.c +36 -0
@@ -32,7 +32,43 @@
 
 #include "decompress.h"
 #include <xen/lzo.h>
+
+#ifndef __MINIOS__
 #include <asm/byteorder.h>
+#else
+#include <endian.h>
+#include <stdint.h>
+
+typedef uint8_t u8;
+typedef uint16_t u16;
+typedef uint32_t u32;
+
+static inline u16 be16_to_cpup(const u16 *p)
+{
+	u16 v = *p;
+#if BYTE_ORDER == LITTLE_ENDIAN
+	return (((v & 0x00ffU) << 8) |
+                ((v & 0xff00U) >> 8));
+#else
+	return v;
+#endif
+}
+
+static inline u32 be32_to_cpup(const u32 *p)
+{
+	u32 v = *p;
+#if BYTE_ORDER == LITTLE_ENDIAN
+	return (((v & 0x000000ffUL) << 24) |
+                ((v & 0x0000ff00UL) <<  8) |
+                ((v & 0x00ff0000UL) >>  8) |
+                ((v & 0xff000000UL) >> 24));
+#else
+	return v;
+#endif
+}
+
+#define unlikely(a) a
+#endif
 
 #if 1 /* ndef CONFIG_??? */
 static inline u16 INIT get_unaligned_be16(void *p)

diff -r efdc4bbfdd20 xen/common/xz/private.h

xen/common/xz/private.h +45 -0
@@ -10,8 +10,53 @@
 #ifndef XZ_PRIVATE_H
 #define XZ_PRIVATE_H
 
+#ifndef __MINIOS__
 #include <xen/kernel.h>
 #include <asm/byteorder.h>
+#else
+#include <endian.h>
+#include <stddef.h>
+#include <stdint.h>
+
+typedef char bool_t;
+typedef uint8_t u8;
+typedef uint16_t u16;
+typedef uint32_t u32;
+typedef uint32_t __le32;
+
+static inline u32 cpu_to_le32(const u32 v)
+{
+#if BYTE_ORDER == BIG_ENDIAN
+	return (((v & 0x000000ffUL) << 24) |
+	        ((v & 0x0000ff00UL) <<  8) |
+	        ((v & 0x00ff0000UL) >>  8) |
+	        ((v & 0xff000000UL) >> 24));
+#else  
+	return v;
+#endif
+}
+
+static inline u32 le32_to_cpup(const u32 *p)
+{
+	return cpu_to_le32(*p);
+}
+
+#define min(x,y) ({ \
+        const typeof(x) _x = (x);       \
+        const typeof(y) _y = (y);       \
+        (void) (&_x == &_y);            \
+        _x < _y ? _x : _y; })
+
+#define min_t(type,x,y) \
+        ({ type __x = (x); type __y = (y); __x < __y ? __x: __y; })
+#define max_t(type,x,y) \
+        ({ type __x = (x); type __y = (y); __x > __y ? __x: __y; })
+
+#define __force
+#define always_inline
+
+#endif
+
 #define get_le32(p) le32_to_cpup((const uint32_t *)(p))
 
 #if 1 /* ndef CONFIG_??? */
#13 lore raw

Re: [RFC] libxc: Add trusted decompressors

Quoted 4 lines · Ian Campbell
Quoted 2 lines · Bastian Blank

Add trusted decompressors based on hypervisor code. This are used in mini-os by pv-grub.

I think this is a reasonably pragmatic way to arrange the build.

What should be fixed later:
- Use standard types in the decompressors (this both extends compiler support and portability)
- Remove stuff like always_inline (the compiler almost always knows better these days)

What should be done now:
- Drop bzip2 and lzma decoders. I have never seen bzip2 used anywhere because it is slow and lzma is replaced entirely by xz.
- Fix the arch detection for the xz bcj decoders.

Quoted 3 lines

I'm not sure "trusted" is quite the right term though, these aren't really any more trustworthy than the library supplied ones -- they are just more suitable for a mini-os environment.

I used the term "trusted" because it should not be fed with untrusted input. So it should not be used in the normal libxenguest. In the case of pv-grub, all input is trusted as it runs in the same security domain.

Bastian

Signature · 2 lines
I'm frequently appalled by the low regard you Earthmen have for life.
		-- Spock, "The Galileo Seven", stardate 2822.3
#14 lore raw

Re: [RFC] libxc: Add trusted decompressors

Quoted 7 lines · Bastian Blank
Quoted 3 lines

I'm not sure "trusted" is quite the right term though, these aren't really any more trustworthy than the library supplied ones -- they are just more suitable for a mini-os environment.

I used the term "trusted" because it should not be fed with untrusted input. So it should not be used in the normal libxenguest. In the case of pv-grub, all input is trusted as it runs in the same security domain.

That makes sense.

Ian.

#15 lore raw

Re: [RFC] libxc: Add trusted decompressors

In reply to Bastian Blank
Quoted 7 lines · Bastian Blank
Quoted 3 lines

I'm not sure "trusted" is quite the right term though, these aren't really any more trustworthy than the library supplied ones -- they are just more suitable for a mini-os environment.

I used the term "trusted" because it should not be fed with untrusted input. So it should not be used in the normal libxenguest. In the case of pv-grub, all input is trusted as it runs in the same security domain.

So it's not the decompressors you trust, but the data that you trust?

"Trusted decompressors" definitely means that the decompressors themselves are more trustworthy somehow; "trusting decompressors" would be a more accurate description. ;-)

What I'm afraid of is that if people see "trusted decompressors", they will interpret it in exactly the opposite way -- i.e.,, they will think that the decompressors can be trusted to deal with untrusted data, perhaps because they do input sanitation or are designed to be robust against malformed data.

Maybe change the comment to say "Add decompressors for trusted data
based on hypervisor code", and a brief comment to each file describing what "trusted" means?

-George

#16 lore raw

Re: [RFC] libxc: Add trusted decompressors

Quoted 9 lines · George Dunlap
Quoted 7 lines · Bastian Blank
Quoted 3 lines

I'm not sure "trusted" is quite the right term though, these aren't really any more trustworthy than the library supplied ones -- they are just more suitable for a mini-os environment.

I used the term "trusted" because it should not be fed with untrusted input. So it should not be used in the normal libxenguest. In the case of pv-grub, all input is trusted as it runs in the same security domain.

Quoted 5 lines

So it's not the decompressors you trust, but the data that you trust?

"Trusted decompressors" definitely means that the decompressors themselves are more trustworthy somehow; "trusting decompressors" would be a more accurate description. ;-)

Yeah, I don't think "trusted" is quite the right word here. I would be inclined to use "unsafe" instead, to make it clear that the caller had better sanitize the input, or not care about unwanted side-effects.

(Generally, I think using "trusted" to mean "relied upon to preserve invariants" is a more useful way of thinking than the common meaning of something like "bug-free". But this isn't quite that either.)

Cheers,

Tim.

#17 lore raw

Re: [RFC] libxc: Add trusted decompressors

Quoted 12 lines · Tim Deegan
Quoted 11 lines · George Dunlap
Quoted 7 lines · Bastian Blank
Quoted 3 lines

I'm not sure "trusted" is quite the right term though, these aren't really any more trustworthy than the library supplied ones -- they are just more suitable for a mini-os environment.

I used the term "trusted" because it should not be fed with untrusted input. So it should not be used in the normal libxenguest. In the case of pv-grub, all input is trusted as it runs in the same security domain.

So it's not the decompressors you trust, but the data that you trust?

Quoted 7 lines
Quoted 3 lines

"Trusted decompressors" definitely means that the decompressors themselves are more trustworthy somehow; "trusting decompressors" would be a more accurate description. ;-)

Yeah, I don't think "trusted" is quite the right word here. I would be inclined to use "unsafe" instead, to make it clear that the caller had better sanitize the input, or not care about unwanted side-effects.

yes, "unsafe" is probably a better word. This is probably almost as simple as running sed on the patch to implement.

Ian.

#18 lore raw

PATCH 1/1 libxc: Add trusted decompressors

In reply to Bastian Blank

Add trusted decompressors based on hypervisor code. This are used in mini-os by pv-grub.

stubdom/Makefile +1 -1
diff -r 66f563be41d9 stubdom/Makefile
@@ -328,7 +328,7 @@
 .PHONY: libxc
 libxc: libxc-$(XEN_TARGET_ARCH)/libxenctrl.a libxc-$(XEN_TARGET_ARCH)/libxenguest.a
 libxc-$(XEN_TARGET_ARCH)/libxenctrl.a: cross-zlib
-	CPPFLAGS="$(TARGET_CPPFLAGS)" CFLAGS="$(TARGET_CFLAGS)" $(MAKE) DESTDIR= -C libxc-$(XEN_TARGET_ARCH)
+	CPPFLAGS="$(TARGET_CPPFLAGS)" CFLAGS="$(TARGET_CFLAGS)" $(MAKE) DESTDIR= CONFIG_LIBXC_MINIOS=y -C libxc-$(XEN_TARGET_ARCH)
 
  libxc-$(XEN_TARGET_ARCH)/libxenguest.a: libxc-$(XEN_TARGET_ARCH)/libxenctrl.a
 
tools/libxc/Makefile +8 -0
diff -r 66f563be41d9 tools/libxc/Makefile
@@ -68,6 +68,14 @@
 GUEST_SRCS-$(CONFIG_ARM)     += xc_dom_arm.c
 GUEST_SRCS-$(CONFIG_ARM)     += xc_hvm_build_arm.c
 
+ifeq ($(CONFIG_LIBXC_MINIOS),y)
+GUEST_SRCS-y                 += xc_dom_decompress_trusted.c
+GUEST_SRCS-y                 += xc_dom_decompress_trusted_bzip2.c
+GUEST_SRCS-y                 += xc_dom_decompress_trusted_lzma.c
+GUEST_SRCS-y                 += xc_dom_decompress_trusted_lzo1x.c
+GUEST_SRCS-y                 += xc_dom_decompress_trusted_xz.c
+endif
+
 OSDEP_SRCS-y                 += xenctrl_osdep_ENOSYS.c
 
 -include $(XEN_TARGET_ARCH)/Makefile
tools/libxc/xc_dom_bzimageloader.c +11 -0
diff -r 66f563be41d9 tools/libxc/xc_dom_bzimageloader.c
@@ -35,6 +35,8 @@
 #include "xg_private.h"
 #include "xc_dom.h"
 
+#ifndef __MINIOS__
+
 #if defined(HAVE_BZLIB)
 
 #include <bzlib.h>
@@ -562,6 +564,15 @@
 
 #endif
 
+#else /* __MINIOS__ */
+
+int xc_try_bzip2_decode(struct xc_dom_image *dom, void **blob, size_t *size);
+int xc_try_lzma_decode(struct xc_dom_image *dom, void **blob, size_t *size);
+int xc_try_lzo1x_decode(struct xc_dom_image *dom, void **blob, size_t *size);
+int xc_try_xz_decode(struct xc_dom_image *dom, void **blob, size_t *size);
+
+#endif /* !__MINIOS__ */
+
 struct setup_header {
     uint8_t  _pad0[0x1f1];  /* skip uninteresting stuff */
     uint8_t  setup_sects;
tools/libxc/xc_dom_decompress_trusted.c new file +49 -0

49 lines not on this page · raw message

tools/libxc/xc_dom_decompress_trusted.h new file +21 -0
diff -r 66f563be41d9 tools/libxc/xc_dom_decompress_trusted.h
@@ -0,0 +1,21 @@
+#include "xc_dom.h"
+
+typedef int decompress_fn(unsigned char *inbuf, unsigned int len,
+                          int (*fill)(void*, unsigned int),
+                          int (*flush)(void*, unsigned int),
+                          unsigned char *outbuf, unsigned int *posp,
+                          void (*error)(const char *x));
+
+int xc_dom_decompress_trusted(
+    decompress_fn fn, struct xc_dom_image *dom, void **blob, size_t *size)
+    __attribute__((visibility("internal")));
+
+int xc_try_bzip2_decode(struct xc_dom_image *dom, void **blob, size_t *size)
+    __attribute__((visibility("internal")));
+int xc_try_lzma_decode(struct xc_dom_image *dom, void **blob, size_t *size)
+    __attribute__((visibility("internal")));
+int xc_try_lzo1x_decode(struct xc_dom_image *dom, void **blob, size_t *size)
+    __attribute__((visibility("internal")));
+int xc_try_xz_decode(struct xc_dom_image *dom, void **blob, size_t *size)
+    __attribute__((visibility("internal")));
+
tools/libxc/xc_dom_decompress_trusted_bzip2.c new file +14 -0
diff -r 66f563be41d9 tools/libxc/xc_dom_decompress_trusted_bzip2.c
@@ -0,0 +1,14 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <inttypes.h>
+
+#include "xg_private.h"
+#include "xc_dom_decompress_trusted.h"
+
+#include "../../xen/common/bunzip2.c"
+
+int xc_try_bzip2_decode(
+    struct xc_dom_image *dom, void **blob, size_t *size)
+{
+    return xc_dom_decompress_trusted(bunzip2, dom, blob, size);
+}
tools/libxc/xc_dom_decompress_trusted_lzma.c new file +14 -0
diff -r 66f563be41d9 tools/libxc/xc_dom_decompress_trusted_lzma.c
@@ -0,0 +1,14 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <inttypes.h>
+
+#include "xg_private.h"
+#include "xc_dom_decompress_trusted.h"
+
+#include "../../xen/common/unlzma.c"
+
+int xc_try_lzma_decode(
+    struct xc_dom_image *dom, void **blob, size_t *size)
+{
+    return xc_dom_decompress_trusted(unlzma, dom, blob, size);
+}
tools/libxc/xc_dom_decompress_trusted_lzo1x.c new file +15 -0
diff -r 66f563be41d9 tools/libxc/xc_dom_decompress_trusted_lzo1x.c
@@ -0,0 +1,15 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <inttypes.h>
+
+#include "xg_private.h"
+#include "xc_dom_decompress_trusted.h"
+
+#include "../../xen/common/lzo.c"
+#include "../../xen/common/unlzo.c"
+
+int xc_try_lzo1x_decode(
+    struct xc_dom_image *dom, void **blob, size_t *size)
+{
+    return xc_dom_decompress_trusted(unlzo, dom, blob, size);
+}
tools/libxc/xc_dom_decompress_trusted_xz.c new file +17 -0
diff -r 66f563be41d9 tools/libxc/xc_dom_decompress_trusted_xz.c
@@ -0,0 +1,17 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <inttypes.h>
+
+#include "xg_private.h"
+#include "xc_dom_decompress_trusted.h"
+
+// TODO
+#define XZ_DEC_X86
+
+#include "../../xen/common/unxz.c"
+
+int xc_try_xz_decode(
+    struct xc_dom_image *dom, void **blob, size_t *size)
+{
+    return xc_dom_decompress_trusted(unxz, dom, blob, size);
+}
xen/common/decompress.h +13 -0
diff -r 66f563be41d9 xen/common/decompress.h
@@ -1,3 +1,5 @@
+#ifdef __XEN__
+
 #include <xen/config.h>
 #include <xen/cache.h>
 #include <xen/decompress.h>
@@ -15,3 +17,14 @@
 
 #define large_malloc xmalloc_bytes
 #define large_free xfree
+
+#else
+
+#define STATIC static
+#define INIT
+#define INITDATA
+
+#define large_malloc malloc
+#define large_free free
+
+#endif
xen/common/lzo.c +12 -0
diff -r 66f563be41d9 xen/common/lzo.c
@@ -68,7 +68,19 @@
  *  Richard Purdie <[email protected]>
  */
 
+#ifdef __XEN__
 #include <xen/types.h>
+#else
+#include <stdint.h>
+
+typedef uint32_t u32;
+typedef uint16_t u16;
+
+#define likely(a) a
+#define noinline
+#define unlikely(a) a
+#endif
+
 #include <xen/lzo.h>
 #define get_unaligned(_p) (*(_p))
 #define put_unaligned(_val,_p) (*(_p)=_val)
xen/common/unlzma.c +2 -0
diff -r 66f563be41d9 xen/common/unlzma.c
@@ -54,7 +54,9 @@
  * Copyright (c) 1999-2005  Igor Pavlov
  */
 
+#ifdef __XEN__
 #include <xen/compiler.h>
+#endif
 
 #define LZMA_IOBUF_SIZE	0x10000
 
xen/common/unlzo.c +36 -0
diff -r 66f563be41d9 xen/common/unlzo.c
@@ -32,7 +32,43 @@
 
 #include "decompress.h"
 #include <xen/lzo.h>
+
+#ifdef __XEN__
 #include <asm/byteorder.h>
+#else
+#include <endian.h>
+#include <stdint.h>
+
+typedef uint8_t u8;
+typedef uint16_t u16;
+typedef uint32_t u32;
+
+static inline u16 be16_to_cpup(const u16 *p)
+{
+	u16 v = *p;
+#if BYTE_ORDER == LITTLE_ENDIAN
+	return (((v & 0x00ffU) << 8) |
+                ((v & 0xff00U) >> 8));
+#else
+	return v;
+#endif
+}
+
+static inline u32 be32_to_cpup(const u32 *p)
+{
+	u32 v = *p;
+#if BYTE_ORDER == LITTLE_ENDIAN
+	return (((v & 0x000000ffUL) << 24) |
+                ((v & 0x0000ff00UL) <<  8) |
+                ((v & 0x00ff0000UL) >>  8) |
+                ((v & 0xff000000UL) >> 24));
+#else
+	return v;
+#endif
+}
+
+#define unlikely(a) a
+#endif
 
 #if 1 /* ndef CONFIG_??? */
 static inline u16 INIT get_unaligned_be16(void *p)
xen/common/xz/private.h +45 -0

53 lines not on this page · raw message