: Implement bzip2 and LZMA loaders

All,
Recent upstream kernels can be compressed using either gzip, bzip2, or
LZMA. However, the PV kernel loader in Xen currently only understands gzip, and will fail on the other two types. The attached patch implements kernel decompression for gzip, bzip2, and LZMA so that kernels compressed with any of these methods can be launched. Note that the patch is still a little bit rough, but I thought I would post it to get feedback about what I'm doing wrong.
I developed this against the RHEL-5 version of the xen tools, but a quick
look through xen-unstable shows that this should apply pretty easily to the current code.

Signed-off-by Chris Lalancette <[email protected]>

diff -urp xen-3.1.0-src/tools/libxc/Makefile xen-3.1.0-src.working/tools/libxc/Makefile

tools/libxc/Makefile +1 -1

@@ -161,7 +161,7 @@ libxenguest.so.$(MAJOR): libxenguest.so.
 	ln -sf $< $@
 
 libxenguest.so.$(MAJOR).$(MINOR): $(GUEST_PIC_OBJS) libxenctrl.so
-	$(CC) $(CFLAGS) $(LDFLAGS) -Wl,$(SONAME_LDFLAG) -Wl,libxenguest.so.$(MAJOR) $(SHLIB_CFLAGS) -o $@ $(GUEST_PIC_OBJS) -lz -lxenctrl -lpthread
+	$(CC) $(CFLAGS) $(LDFLAGS) -Wl,$(SONAME_LDFLAG) -Wl,libxenguest.so.$(MAJOR) $(SHLIB_CFLAGS) -o $@ $(GUEST_PIC_OBJS) -lz -llzma -lbz2 -lxenctrl -lpthread
 
 -include $(DEPS)
 

diff -urp xen-3.1.0-src/tools/libxc/xc_dom_bzimageloader.c xen-3.1.0-src.working/tools/libxc/xc_dom_bzimageloader.c

tools/libxc/xc_dom_bzimageloader.c +230 -22

@@ -11,15 +11,208 @@
  * written 2006 by Gerd Hoffmann <[email protected]>.
  * written 2007 by Jeremy Fitzhardinge <[email protected]>
  * written 2008 by Ian Campbell <[email protected]>
+ * written 2009 by Chris Lalancette <[email protected]>
  *
  */
 #include <stdio.h>
 #include <stdlib.h>
 #include <inttypes.h>
+#include <bzlib.h>
+#include <lzma.h>
 
 #include "xg_private.h"
 #include "xc_dom.h"
 
+static inline uint64_t physmem(void)
+{
+	uint64_t ret = 0;
+
+	const long pagesize = sysconf(_SC_PAGESIZE);
+	const long pages = sysconf(_SC_PHYS_PAGES);
+	if (pagesize != -1 || pages != -1)
+		// According to docs, pagesize * pages can overflow.
+		// Simple case is 32-bit box with 4 GiB or more RAM,
+		// which may report exactly 4 GiB of RAM, and "long"
+		// being 32-bit will overflow. Casting to uint64_t
+		// hopefully avoids overflows in the near future.
+		ret = (uint64_t)(pagesize) * (uint64_t)(pages);
+
+    return ret;
+}
+
+static int xc_try_bzip2_decode(struct xc_dom_image *dom, void **blob, size_t *size)
+{
+    bz_stream stream;
+    int ret;
+    char *out_buf;
+    int retval = -1;
+    int outsize;
+    uint64_t total;
+
+    stream.bzalloc = NULL;
+    stream.bzfree = NULL;
+    stream.opaque = NULL;
+
+    ret = BZ2_bzDecompressInit(&stream, 0, 0);
+    if (ret != BZ_OK) {
+        xc_dom_printf("Error initting bz2 stream\n");
+        return -1;
+    }
+
+    /* sigh.  We don't know up-front how much memory we are going to need
+     * for the output buffer.  Allocate the output buffer to be equal
+     * the input buffer to start, and we'll realloc as needed.
+     */
+    outsize = dom->kernel_size;
+    out_buf = malloc(outsize);
+    if (out_buf == NULL) {
+        xc_dom_printf("Failed to alloc memory\n");
+        goto bzip2_cleanup;
+    }
+
+    stream.next_in = dom->kernel_blob;
+    stream.avail_in = dom->kernel_size;
+
+    stream.next_out = out_buf;
+    stream.avail_out = dom->kernel_size;
+
+    while (1) {
+        ret = BZ2_bzDecompress(&stream);
+        if (stream.avail_out == 0 || ret != BZ_OK) {
+            out_buf = realloc(out_buf, outsize * 2);
+            if (out_buf == NULL) {
+                xc_dom_printf("Failed to realloc memory\n");
+                break;
+            }
+
+            stream.next_out = out_buf + outsize;
+            stream.avail_out = (outsize * 2) - outsize;
+            outsize *= 2;
+        }
+
+        if (ret != BZ_OK) {
+            if (ret == BZ_STREAM_END) {
+                xc_dom_printf("Saw data stream end\n");
+                retval = 0;
+                break;
+            }
+            xc_dom_printf("BZIP error\n");
+        }
+    }
+
+    total = (stream.total_out_hi32 << 31) | stream.total_out_lo32;
+
+    xc_dom_printf("%s: BZIP2 decompress OK, 0x%zx -> 0x%lx\n", __FUNCTION__, *size, total);
+
+    *blob = out_buf;
+    *size = total;
+
+bzip2_cleanup:
+    BZ2_bzDecompressEnd(&stream);
+
+    return retval;
+}
+
+static int xc_try_lzma_decode(struct xc_dom_image *dom, void **blob, size_t *size)
+{
+    lzma_stream stream = LZMA_STREAM_INIT;
+    lzma_ret ret;
+    lzma_action action = LZMA_RUN;
+    unsigned char *out_buf;
+    int retval = -1;
+    int outsize;
+    const char *msg;
+
+    ret = lzma_alone_decoder(&stream, physmem() / 3);
+    if (ret != LZMA_OK) {
+        xc_dom_printf("Failed to init lzma stream decoder\n");
+        return -1;
+    }
+
+    /* sigh.  We don't know up-front how much memory we are going to need
+     * for the output buffer.  Allocate the output buffer to be equal
+     * the input buffer to start, and we'll realloc as needed.
+     */
+    outsize = dom->kernel_size;
+    out_buf = malloc(outsize);
+    if (out_buf == NULL) {
+        xc_dom_printf("Failed to alloc memory\n");
+        goto lzma_cleanup;
+    }
+
+    stream.next_in = dom->kernel_blob;
+    stream.avail_in = dom->kernel_size;
+
+    stream.next_out = out_buf;
+    stream.avail_out = dom->kernel_size;
+
+    while (1) {
+        ret = lzma_code(&stream, action);
+        if (stream.avail_out == 0 || ret != LZMA_OK) {
+            out_buf = realloc(out_buf, outsize * 2);
+            if (out_buf == NULL) {
+                xc_dom_printf("Failed to realloc memory\n");
+                break;
+            }
+
+            stream.next_out = out_buf + outsize;
+            stream.avail_out = (outsize * 2) - outsize;
+            outsize *= 2;
+        }
+
+        if (ret != LZMA_OK) {
+            if (ret == LZMA_STREAM_END) {
+                xc_dom_printf("Saw data stream end\n");
+                retval = 0;
+                break;
+            }
+
+            switch (ret) {
+            case LZMA_MEM_ERROR:
+                msg = strerror(ENOMEM);
+                break;
+
+            case LZMA_MEMLIMIT_ERROR:
+                msg = "Memory usage limit reached";
+                break;
+
+            case LZMA_FORMAT_ERROR:
+                msg = "File format not recognized";
+                break;
+
+            case LZMA_OPTIONS_ERROR:
+                // FIXME: Better message?
+                msg = "Unsupported compression options";
+                break;
+
+            case LZMA_DATA_ERROR:
+                msg = "File is corrupt";
+                break;
+
+            case LZMA_BUF_ERROR:
+                msg = "Unexpected end of input";
+                break;
+
+            default:
+                msg = "Internal program error (bug)";
+                break;
+            }
+            xc_dom_printf("%s: LZMA decompression error %s\n", __FUNCTION__, msg);
+            break;
+        }
+    }
+
+    xc_dom_printf("%s: LZMA decompress OK, 0x%zx -> 0x%zx\n", __FUNCTION__, *size, stream.total_out);
+
+    *blob = out_buf;
+    *size = stream.total_out;
+
+ lzma_cleanup:
+    lzma_end(&stream);
+
+    return retval;
+}
+
 struct setup_header {
 	uint8_t		_pad0[0x1f1];		/* skip uninteresting stuff */
 	uint8_t		setup_sects;

@@ -70,22 +263,22 @@ static unsigned int payload_offset(struc
     return off;
 }
 
-static int check_bzimage_kernel(struct xc_dom_image *dom, int verbose)
+static int xc_dom_probe_bzimage_kernel(struct xc_dom_image *dom)
 {
     struct setup_header *hdr;
+    int ret;
 
     if ( dom->kernel_blob == NULL )
     {
-        if ( verbose )
-            xc_dom_panic(XC_INTERNAL_ERROR, "%s: no kernel image loaded\n",
-                         __FUNCTION__);
+        xc_dom_panic(XC_INTERNAL_ERROR, "%s: no kernel image loaded\n",
+                     __FUNCTION__);
         return -EINVAL;
     }
+
     if ( dom->kernel_size < sizeof(struct setup_header) )
     {
-        if ( verbose )
-            xc_dom_panic(XC_INTERNAL_ERROR, "%s: kernel image too small\n",
-                         __FUNCTION__);
+        xc_dom_panic(XC_INTERNAL_ERROR, "%s: kernel image too small\n",
+                     __FUNCTION__);
         return -EINVAL;
     }
 

@@ -93,39 +286,54 @@ static int check_bzimage_kernel(struct x
 
     if ( memcmp(&hdr->header, HDR_MAGIC, HDR_MAGIC_SZ) != 0 )
     {
-        if ( verbose )
-            xc_dom_panic(XC_INVALID_KERNEL, "%s: kernel is not a bzImage\n",
-                         __FUNCTION__);
+        xc_dom_panic(XC_INVALID_KERNEL, "%s: kernel is not a bzImage\n",
+                     __FUNCTION__);
         return -EINVAL;
     }
 
     if ( hdr->version < VERSION(2,8) )
     {
-        if ( verbose )
-            xc_dom_panic(XC_INVALID_KERNEL, "%s: boot protocol too old (%04x)\n",
-                         __FUNCTION__, hdr->version);
+        xc_dom_panic(XC_INVALID_KERNEL, "%s: boot protocol too old (%04x)\n",
+                     __FUNCTION__, hdr->version);
         return -EINVAL;
     }
 
     dom->kernel_blob = dom->kernel_blob + payload_offset(hdr);
     dom->kernel_size = hdr->payload_length;
 
-    if ( xc_dom_try_gunzip(dom, &dom->kernel_blob, &dom->kernel_size) == -1 )
-    {
-        if ( verbose )
-            xc_dom_panic(XC_INVALID_KERNEL, "%s: unable to decompress kernel\n",
+    if (memcmp(dom->kernel_blob, "\037\213", 2) == 0) {
+        ret = xc_dom_try_gunzip(dom, &dom->kernel_blob, &dom->kernel_size);
+        if (ret == -1) {
+            xc_dom_panic(XC_INVALID_KERNEL, "%s: unable to gzip decompress kernel\n",
+                         __FUNCTION__);
+            return -EINVAL;
+        }
+    }
+    else if (memcmp(dom->kernel_blob, "\102\132\150", 3) == 0) {
+        ret = xc_try_bzip2_decode(dom, &dom->kernel_blob, &dom->kernel_size);
+        if (ret < 0) {
+            xc_dom_panic(XC_INVALID_KERNEL, "%s unable to BZIP2 decompress kernel",
+                         __FUNCTION__);
+            return -EINVAL;
+        }
+    }
+    else if (memcmp(dom->kernel_blob, "\135\000", 2) == 0) {
+        ret = xc_try_lzma_decode(dom, &dom->kernel_blob, &dom->kernel_size);
+        if (ret < 0) {
+            xc_dom_panic(XC_INVALID_KERNEL, "%s unable to LZMA decompress kernel\n",
                          __FUNCTION__);
+            return -EINVAL;
+        }
+    }
+    else {
+        xc_dom_panic(XC_INVALID_KERNEL, "%s: unknown compression format\n",
+                     __FUNCTION__);
         return -EINVAL;
     }
 
     return elf_loader.probe(dom);
 }
 
-static int xc_dom_probe_bzimage_kernel(struct xc_dom_image *dom)
-{
-    return check_bzimage_kernel(dom, 0);
-}
-
 static int xc_dom_parse_bzimage_kernel(struct xc_dom_image *dom)
 {
     return elf_loader.parser(dom);

_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xensource.com/xen-devel

Quoted 11 lines · Chris Lalancette

All,
Recent upstream kernels can be compressed using either gzip, bzip2, or
LZMA. However, the PV kernel loader in Xen currently only understands gzip, and will fail on the other two types. The attached patch implements kernel decompression for gzip, bzip2, and LZMA so that kernels compressed with any of these methods can be launched. Note that the patch is still a little bit rough, but I thought I would post it to get feedback about what I'm doing wrong.
I developed this against the RHEL-5 version of the xen tools, but a quick
look through xen-unstable shows that this should apply pretty easily to the current code.

Attached are two patches:

- Patch by Chris modified to apply to Xen 3.4.1 (only a small Makefile change)
- Patch by me to fix compilation with gcc 4.4.0.

Quoted 6 lines

Attached are two patches:

- Patch by Chris modified to apply to Xen 3.4.1 (only a small Makefile change)
- Patch by me to fix compilation with gcc 4.4.0.

Hmm, actually this patch seems to break stubdom build/linking:

Quoted 1 line

Hmm, actually this patch seems to break stubdom build/linking:

I checked in a variant as c/s 20103, which at least for non-stubdom build of libxenguest should do the right thing if liblzma-devel or libbz2-devel packages (or whatever they are called) are not installed. See how that goes with stubdom. We may just have to massage the Makefile a little more to kill off the bz2/lzma stuff for stubdom. I'm not sure why libxenguest is getting built for stubdom in the first place though. It's not likely to be very useful!

Quoted 9 lines · Keir Fraser

Quoted 1 line

Hmm, actually this patch seems to break stubdom build/linking:

I checked in a variant as c/s 20103, which at least for non-stubdom build of libxenguest should do the right thing if liblzma-devel or libbz2-devel packages (or whatever they are called) are not installed. See how that goes with stubdom. We may just have to massage the Makefile a little more to kill off the bz2/lzma stuff for stubdom. I'm not sure why libxenguest is getting built for stubdom in the first place though. It's not likely to be very useful!

Okay, I tested and fixed the stubdom case, so hopefully c/s 20104 will actually work for you.

Quoted 3 lines

Okay, I tested and fixed the stubdom case, so hopefully c/s 20104 will actually work for you.

xen-unstable "make tools && make stubdom" compiles for me with and without libbz2-devel/liblzma-devel installed.

But even when I have liblzma-devel installed it doesn't seem to use it. ie. I only see -DHAVE_BZLIB in the build log and libxenguest.so.3.4.0 gets only linked against -lbz2.

Investigating more..

Quoted 10 lines · Pasi Kärkkäinen

Quoted 3 lines

Okay, I tested and fixed the stubdom case, so hopefully c/s 20104 will actually work for you.

xen-unstable "make tools && make stubdom" compiles for me with and without libbz2-devel/liblzma-devel installed.

But even when I have liblzma-devel installed it doesn't seem to use it. ie. I only see -DHAVE_BZLIB in the build log and libxenguest.so.3.4.0 gets only linked against -lbz2.

Quoted 1 line

Investigating more..

With that package installed, do you have a header /usr/include/lzma.h, or otherwise have lzma.h on the standard include path? If not then we're doing the right thing, since Chris's patch wants to #include <lzma.h>. If it is, then the has_header lzma.h check in the libxc Makefile is broken somehow.

Any ideas how to fix that? Do we need to include bzip2 and lzma libs in stubdom/ ?

Quoted 7 lines

xen-unstable "make tools && make stubdom" compiles for me with and without libbz2-devel/liblzma-devel installed.

But even when I have liblzma-devel installed it doesn't seem to use it. ie. I only see -DHAVE_BZLIB in the build log and libxenguest.so.3.4.0 gets only linked against -lbz2.

Quoted 1 line

Investigating more..

With that package installed, do you have a header /usr/include/lzma.h, or otherwise have lzma.h on the standard include path? If not then we're doing the right thing, since Chris's patch wants to #include <lzma.h>. If it is, then the has_header lzma.h check in the libxc Makefile is broken somehow.

Quoted 12 lines · Keir Fraser

Quoted 9 lines · Keir Fraser

Quoted 1 line

Hmm, actually this patch seems to break stubdom build/linking:

I checked in a variant as c/s 20103, which at least for non-stubdom build of libxenguest should do the right thing if liblzma-devel or libbz2-devel packages (or whatever they are called) are not installed. See how that goes with stubdom. We may just have to massage the Makefile a little more to kill off the bz2/lzma stuff for stubdom. I'm not sure why libxenguest is getting built for stubdom in the first place though. It's not likely to be very useful!

Quoted 3 lines

Okay, I tested and fixed the stubdom case, so hopefully c/s 20104 will actually work for you.

xen-unstable "make tools && make stubdom" compiles for me with and without libbz2-devel/liblzma-devel installed.

But even when I have liblzma-devel installed it doesn't seem to use it. ie. I only see -DHAVE_BZLIB in the build log and libxenguest.so.3.4.0 gets only linked against -lbz2.

Investigating more..

Quoted 12 lines · Keir Fraser

Quoted 10 lines · Pasi Kärkkäinen

Quoted 3 lines

Okay, I tested and fixed the stubdom case, so hopefully c/s 20104 will actually work for you.

xen-unstable "make tools && make stubdom" compiles for me with and without libbz2-devel/liblzma-devel installed.

But even when I have liblzma-devel installed it doesn't seem to use it. ie. I only see -DHAVE_BZLIB in the build log and libxenguest.so.3.4.0 gets only linked against -lbz2.

Quoted 6 lines

Quoted 1 line

Investigating more..

With that package installed, do you have a header /usr/include/lzma.h, or otherwise have lzma.h on the standard include path? If not then we're doing the right thing, since Chris's patch wants to #include <lzma.h>. If it is, then the has_header lzma.h check in the libxc Makefile is broken somehow.

Hm, there's also a somewhat confusing situation with LZMA. I'm not sure I completely understand it, but from what I can tell the lzma.h header file should be coming from the new "xz" project, not from the lzma project. That is, on my Fedora 10 box, in order to get LZMA support compiled, I needed to install the xz-devel package; lzma-devel was not sufficient.

Quoted 4 lines · Pasi Kärkkäinen

Any ideas how to fix that? Do we need to include bzip2 and lzma libs in stubdom/ ?

Yes. Otherwise bzip2+lzma support will not work in pvgrub.

Quoted 5 lines

Quoted 4 lines

Any ideas how to fix that? Do we need to include bzip2 and lzma libs in stubdom/ ?

Quoted 2 lines

Yes. Otherwise bzip2+lzma support will not work in pvgrub.

Good point. pvgrub should support these aswell.

Keir: Would you like to add those libs to stubdom?

(Note the lzma.h is coming from that "xz" package..)

Quoted 5 lines

Good point. pvgrub should support these aswell.

Keir: Would you like to add those libs to stubdom?

(Note the lzma.h is coming from that "xz" package..)

No. :-) But feel free to make a patch! You'll just need to modify libxc/Makefile too to force those libs on instead of off, as well as download and build the libs of course.

Quoted 6 lines · Keir Fraser

Quoted 5 lines

Good point. pvgrub should support these aswell.

Keir: Would you like to add those libs to stubdom?

(Note the lzma.h is coming from that "xz" package..)

Quoted 3 lines

No. :-) But feel free to make a patch! You'll just need to modify libxc/Makefile too to force those libs on instead of off, as well as download and build the libs of course.

Actually I just checked in c/s 20105 to get rid of the minios-specific hack in that Makefile. So all you need to do is patch to download and build the libs. If you make a patch, I will arrange for the lib tarballs to be hosted on xenbits and adjust the patch URLs appropriately.

Quoted 5 lines

Actually I just checked in c/s 20105 to get rid of the minios-specific hack in that Makefile. So all you need to do is patch to download and build the libs. If you make a patch, I will arrange for the lib tarballs to be hosted on xenbits and adjust the patch URLs appropriately.

Ok :)

c/s 20105 stubdom fails to build now, but I guess that's expected until the patch for adding the libs is done.

I'll take a look at it today.

Quoted 6 lines · Pasi Kärkkäinen

Quoted 5 lines

Actually I just checked in c/s 20105 to get rid of the minios-specific hack in that Makefile. So all you need to do is patch to download and build the libs. If you make a patch, I will arrange for the lib tarballs to be hosted on xenbits and adjust the patch URLs appropriately.

Quoted 4 lines

Ok :)

c/s 20105 stubdom fails to build now, but I guess that's expected until the patch for adding the libs is done.

No, that isn't expected. It builds okay for me.

Quoted 13 lines · Keir Fraser

Quoted 10 lines · Pasi Kärkkäinen

Quoted 5 lines

Actually I just checked in c/s 20105 to get rid of the minios-specific hack in that Makefile. So all you need to do is patch to download and build the libs. If you make a patch, I will arrange for the lib tarballs to be hosted on xenbits and adjust the patch URLs appropriately.

Ok :)

c/s 20105 stubdom fails to build now, but I guess that's expected until the patch for adding the libs is done.

Quoted 2 lines

No, that isn't expected. It builds okay for me.

For me it builds _without_ libbz2-devel/liblzma-devel.

With those installed stubdom build fails with errors about not finding the headers for those libs.

Quoted 14 lines · Pasi Kärkkäinen

Quoted 13 lines · Keir Fraser

Quoted 10 lines · Pasi Kärkkäinen

Quoted 5 lines

Actually I just checked in c/s 20105 to get rid of the minios-specific hack in that Makefile. So all you need to do is patch to download and build the libs. If you make a patch, I will arrange for the lib tarballs to be hosted on xenbits and adjust the patch URLs appropriately.

Ok :)

c/s 20105 stubdom fails to build now, but I guess that's expected until the patch for adding the libs is done.

Quoted 8 lines

Quoted 2 lines

No, that isn't expected. It builds okay for me.

For me it builds _without_ libbz2-devel/liblzma-devel.

With those installed stubdom build fails with errors about not finding the headers for those libs.

c/s 20104:
- "make tools && make stubdom" without libbz2-devel/liblzma-devel succeeds
- "make tools && make stubdom" with libbz2-devel/liblzma-devel succeeds

c/s 20105:
- "make tools && make stubdom" without libbz2-devel/liblzma-devel succeeds
- "make tools && make stubdom" with libbz2-devel/liblzma-devel: tools succeeds, stubdom fails

xc_dom_bzimageloader.c:26:19: error: bzlib.h: No such file or directory xc_dom_bzimageloader.c: In function 'xc_try_bzip2_decode':

xc_dom_bzimageloader.c:124:18: error: lzma.h: No such file or directory xc_dom_bzimageloader.c: In function 'xc_try_lzma_decode':

Quoted 5 lines

Quoted 2 lines

c/s 20105 stubdom fails to build now, but I guess that's expected until the patch for adding the libs is done.

No, that isn't expected. It builds okay for me.

For me it builds _without_ libbz2-devel/liblzma-devel.

With those installed stubdom build fails with errors about not finding the headers for those libs.

Quoted 6 lines · Keir Fraser

Quoted 5 lines

Good point. pvgrub should support these aswell.

Keir: Would you like to add those libs to stubdom?

(Note the lzma.h is coming from that "xz" package..)

Quoted 3 lines

No. :-) But feel free to make a patch! You'll just need to modify libxc/Makefile too to force those libs on instead of off, as well as download and build the libs of course.

Actually I just checked in c/s 20105 to get rid of the minios-specific hack in that Makefile. So all you need to do is patch to download and build the libs. If you make a patch, I will arrange for the lib tarballs to be hosted on xenbits and adjust the patch URLs appropriately.

Quoted 8 lines · Keir Fraser

Quoted 6 lines · Keir Fraser

Quoted 5 lines

Good point. pvgrub should support these aswell.

Keir: Would you like to add those libs to stubdom?

(Note the lzma.h is coming from that "xz" package..)

Quoted 9 lines

Quoted 3 lines

No. :-) But feel free to make a patch! You'll just need to modify libxc/Makefile too to force those libs on instead of off, as well as download and build the libs of course.

Actually I just checked in c/s 20105 to get rid of the minios-specific hack in that Makefile. So all you need to do is patch to download and build the libs. If you make a patch, I will arrange for the lib tarballs to be hosted on xenbits and adjust the patch URLs appropriately.

I can now see why you didn't want to do that.. :) I managed to get the libs to compile, but now I'm having other problems..

"make stubdom" gets to the point where it compiles ioemu-stubdom, but it fails to link because xen-unstable.hg/stubdom/mini-os-x86_32-ioemu/mini-os.o has undefined references in xc_try_bzip2_decode and xc_try_lzma_decode functions..

I guess only pvgrub would need to have this stuff in? I've been trying to figure out how the makefile magic works, but haven't really understood it yet.

Any tips how to make ioemu-stubdom compile/link without including bzip2/lzma stuff into it?

Quoted 4 lines

I guess only pvgrub would need to have this stuff in? I've been trying to figure out how the makefile magic works, but haven't really understood it yet.

Quoted 3 lines

Any tips how to make ioemu-stubdom compile/link without including bzip2/lzma stuff into it?

Actually I guess I could add the missing libs to extras/mini-os/Makefile to APP_LDLIBS to get them included into ioemu-stubdom?

Is that the right way to do it?

Quoted 29 lines · Pasi Kärkkäinen

Quoted 8 lines · Keir Fraser

Quoted 6 lines · Keir Fraser

Quoted 5 lines

Good point. pvgrub should support these aswell.

Keir: Would you like to add those libs to stubdom?

(Note the lzma.h is coming from that "xz" package..)

Quoted 9 lines

Quoted 3 lines

No. :-) But feel free to make a patch! You'll just need to modify libxc/Makefile too to force those libs on instead of off, as well as download and build the libs of course.

Actually I just checked in c/s 20105 to get rid of the minios-specific hack in that Makefile. So all you need to do is patch to download and build the libs. If you make a patch, I will arrange for the lib tarballs to be hosted on xenbits and adjust the patch URLs appropriately.

I can now see why you didn't want to do that.. :) I managed to get the libs to compile, but now I'm having other problems..

"make stubdom" gets to the point where it compiles ioemu-stubdom, but it fails to link because xen-unstable.hg/stubdom/mini-os-x86_32-ioemu/mini-os.o has undefined references in xc_try_bzip2_decode and xc_try_lzma_decode functions..

I guess only pvgrub would need to have this stuff in? I've been trying to figure out how the makefile magic works, but haven't really understood it yet.

Quoted 2 lines

Any tips how to make ioemu-stubdom compile/link without including bzip2/lzma stuff into it?

Actually I guess I could add the missing libs to extras/mini-os/Makefile to APP_LDLIBS to get them included into ioemu-stubdom?

Is that the right way to do it?

Quoted 36 lines · Pasi Kärkkäinen

Quoted 22 lines · Pasi Kärkkäinen

Quoted 8 lines · Keir Fraser

Quoted 6 lines · Keir Fraser

Quoted 5 lines

Good point. pvgrub should support these aswell.

Keir: Would you like to add those libs to stubdom?

(Note the lzma.h is coming from that "xz" package..)

Quoted 9 lines

Quoted 3 lines

No. :-) But feel free to make a patch! You'll just need to modify libxc/Makefile too to force those libs on instead of off, as well as download and build the libs of course.

Actually I just checked in c/s 20105 to get rid of the minios-specific hack in that Makefile. So all you need to do is patch to download and build the libs. If you make a patch, I will arrange for the lib tarballs to be hosted on xenbits and adjust the patch URLs appropriately.

I can now see why you didn't want to do that.. :) I managed to get the libs to compile, but now I'm having other problems..

Quoted 9 lines

"make stubdom" gets to the point where it compiles ioemu-stubdom, but it fails to link because xen-unstable.hg/stubdom/mini-os-x86_32-ioemu/mini-os.o has undefined references in xc_try_bzip2_decode and xc_try_lzma_decode functions..

I guess only pvgrub would need to have this stuff in? I've been trying to figure out how the makefile magic works, but haven't really understood it yet.

Any tips how to make ioemu-stubdom compile/link without including bzip2/lzma stuff into it?

Actually I guess I could add the missing libs to extras/mini-os/Makefile to APP_LDLIBS to get them included into ioemu-stubdom?

Quoted 2 lines

Is that the right way to do it?

bzlib and lzma are libxc dependencies now, so firstly you have to add the two libraries to the stubdom build system, take a look at pciutils in stubdom/Makefile, that is a good example of how a new library is added.

Then you also need to add them both at least to the libxc target, take a look at zlib (and cross-zlib), that should be close to what you need.

Finally you need to tweak the libxc Makefile for the stubdom case so that the two libraries are correctly added to zlib-options.

Quoted 37 lines · Stefano Stabellini

Quoted 36 lines · Pasi Kärkkäinen

Quoted 22 lines · Pasi Kärkkäinen

Quoted 8 lines · Keir Fraser

Quoted 6 lines · Keir Fraser

Quoted 5 lines

Good point. pvgrub should support these aswell.

Keir: Would you like to add those libs to stubdom?

(Note the lzma.h is coming from that "xz" package..)

Quoted 9 lines

Quoted 3 lines

No. :-) But feel free to make a patch! You'll just need to modify libxc/Makefile too to force those libs on instead of off, as well as download and build the libs of course.

Actually I just checked in c/s 20105 to get rid of the minios-specific hack in that Makefile. So all you need to do is patch to download and build the libs. If you make a patch, I will arrange for the lib tarballs to be hosted on xenbits and adjust the patch URLs appropriately.

I can now see why you didn't want to do that.. :) I managed to get the libs to compile, but now I'm having other problems..

Quoted 9 lines

"make stubdom" gets to the point where it compiles ioemu-stubdom, but it fails to link because xen-unstable.hg/stubdom/mini-os-x86_32-ioemu/mini-os.o has undefined references in xc_try_bzip2_decode and xc_try_lzma_decode functions..

I guess only pvgrub would need to have this stuff in? I've been trying to figure out how the makefile magic works, but haven't really understood it yet.

Any tips how to make ioemu-stubdom compile/link without including bzip2/lzma stuff into it?

Actually I guess I could add the missing libs to extras/mini-os/Makefile to APP_LDLIBS to get them included into ioemu-stubdom?

Quoted 7 lines

Quoted 2 lines

Is that the right way to do it?

bzlib and lzma are libxc dependencies now, so firstly you have to add the two libraries to the stubdom build system, take a look at pciutils in stubdom/Makefile, that is a good example of how a new library is added.

This is done, and works. libbz2.a and liblzma.a end up in stubdom/cross-root-i686/i686-xen-elf/lib/ directory on my 32bit host.

Quoted 2 lines

Then you also need to add them both at least to the libxc target, take a look at zlib (and cross-zlib), that should be close to what you need.

Done.

Quoted 3 lines

Finally you need to tweak the libxc Makefile for the stubdom case so that the two libraries are correctly added to zlib-options.

So you mean xen-unstable/tools/libxc/Makefile here? This part?

ifeq ($(CONFIG_MiniOS),y)
zlib-options =

The problem I'm seeing now is "ioemu" target from xen-unstable/stubdom/Makefile fails to link:

ld -nostdlib -L/root/tem/xen-unstable.hg/stubdom/cross-root-i686/i686-xen-elf/lib -m elf_i386 -T arch/x86/minios-x86_32.lds /root/tem/xen-unstable.hg/stubdom/mini-os-x86_32-ioemu/mini-os.o -o /root/tem/xen-unstable.hg/stubdom/mini-os-x86_32-ioemu/mini-os
ld: warning: section .bss' type changed to PROGBITS /root/tem/xen-unstable.hg/stubdom/mini-os-x86_32-ioemu/mini-os.o: In function xc_try_bzip2_decode':
/root/tem/xen-unstable.hg/stubdom/libxc-x86_32/xc_dom_bzimageloader.c:42: undefined reference to BZ2_bzDecompressInit' /root/tem/xen-unstable.hg/stubdom/libxc-x86_32/xc_dom_bzimageloader.c:105: undefined reference to BZ2_bzDecompressEnd' /root/tem/xen-unstable.hg/stubdom/libxc-x86_32/xc_dom_bzimageloader.c:69: undefined reference to BZ2_bzDecompress' /root/tem/xen-unstable.hg/stubdom/libxc-x86_32/xc_dom_bzimageloader.c:105: undefined reference to BZ2_bzDecompressEnd' /root/tem/xen-unstable.hg/stubdom/mini-os-x86_32-ioemu/mini-os.o: In function `xc_try_lzma_decode':
/root/tem/xen-unstable.hg/stubdom/libxc-x86_32/xc_dom_bzimageloader.c:158: undefined reference to lzma_alone_decoder' /root/tem/xen-unstable.hg/stubdom/libxc-x86_32/xc_dom_bzimageloader.c:253: undefined reference to lzma_end' /root/tem/xen-unstable.hg/stubdom/libxc-x86_32/xc_dom_bzimageloader.c:185: undefined reference to lzma_code' /root/tem/xen-unstable.hg/stubdom/libxc-x86_32/xc_dom_bzimageloader.c:253: undefined reference to lzma_end' make[1]: ***
[/root/tem/xen-unstable.hg/stubdom/mini-os-x86_32-ioemu/mini-os] Error 1
make[1]: Leaving directory `/root/tem/xen-unstable.hg/extras/mini-os'
make: *** [ioemu-stubdom] Error 2

So basicly I should add libbz2.a and liblzma.a to the list of files to link, but I'm a bit lost where this is happening. Should I edit xen-unstable/extras/mini-os/Makefile to add those libs or somewhere else?

I tried adding those libs to xen-unstable/stubdom/Makefile to "ioemu" TARGET_LDFLAGS, but that didn't seem to help.

All tips welcome :)

Attached are my current work-in-progress patches. It still doesn't link properly..

Stefano: Would you like to take a look? The patches are against current xen-unstable.

I'm getting weird linking error about stdin/stdout/stderr missing.. libc (newlib) is definitely linked in, so I'm not sure what's happening..

ld -nostdlib -L/root/xen-unstable.hg/stubdom/cross-root-i686/i686-xen-elf/lib -m elf_i386 -T arch/x86/minios-x86_32.lds /root/xen-unstable.hg/stubdom/mini-os-x86_32-ioemu/mini-os.o -o /root/xen-unstable.hg/stubdom/mini-os-x86_32-ioemu/mini-os

xen-unstable.hg/stubdom/mini-os-x86_32-ioemu/mini-os.o: In function `bzopen_or_bzdopen':
xen-unstable.hg/stubdom/bzip2-x86_32/bzlib.c:1411: undefined reference to __ctype_b_loc' xen-unstable.hg/stubdom/bzip2-x86_32/bzlib.c:1447: undefined reference to stdin' xen-unstable.hg/stubdom/bzip2-x86_32/bzlib.c:1447: undefined reference to `stdout'
...

In the earlier step mini-os.o was definitely linked against -lc ..

"strings /path/mini-os.o | grep -i stdin" doesn't give anything though.. I wonder why the stuff doesn't get included there.

Hello again,

Stefano: Have you ever seen this kind of linking failures with stubdoms/ioemu?

Any ideas?

-- Pasi

Quoted 13 lines · Pasi Kärkkäinen

Attached are my current work-in-progress patches. It still doesn't link properly..

Stefano: Would you like to take a look? The patches are against current xen-unstable.

I'm getting weird linking error about stdin/stdout/stderr missing.. libc (newlib) is definitely linked in, so I'm not sure what's happening..

ld -nostdlib -L/root/xen-unstable.hg/stubdom/cross-root-i686/i686-xen-elf/lib -m elf_i386 -T arch/x86/minios-x86_32.lds /root/xen-unstable.hg/stubdom/mini-os-x86_32-ioemu/mini-os.o -o /root/xen-unstable.hg/stubdom/mini-os-x86_32-ioemu/mini-os

xen-unstable.hg/stubdom/mini-os-x86_32-ioemu/mini-os.o: In function `bzopen_or_bzdopen':
xen-unstable.hg/stubdom/bzip2-x86_32/bzlib.c:1411: undefined reference to `__ctype_b_loc'

Quoted 3 lines

xen-unstable.hg/stubdom/bzip2-x86_32/bzlib.c:1447: undefined reference to stdin' xen-unstable.hg/stubdom/bzip2-x86_32/bzlib.c:1447: undefined reference to stdout'
...