Add HTTP protocol using curl v7
Subsystems (2)
PATCH 1/1 Add HTTP protocol using curl v7
Currently Qemu can read from posix I/O and NBD. This patch adds a third protocol to the game: HTTP.
In certain situations it can be useful to access HTTP data directly, for example if you want to try out an http provided OS image, but don't know if you want to download it yet.
Using this patch you can now try it on on the fly. Just use it like:
qemu -cdrom http://host/path/my.iso
In order to not reinvent the wheel, this patch uses libcurl.
v2 changes:
- fix the segfault (yay!)
- implement AIO
v3 changes:
- remove synchronous API
- implement caching
v4 changes:
- enable other protocols (HTTPS, FTP, FTPS, TFTP, SFTP, SCP) (I only tested FTP so far, but they _should_ work)
v5 changes:
- address Anthony's comments
- deactivate ssh protocols again
v6 changes:
- s/^I/ /g
- s/http/curl/g
v7 changes:
- add Signed-off-by again
--- Makefile | 6 + Makefile.target | 2 +- block-curl.c | 540 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ block.c | 12 ++ block.h | 7 + configure | 25 +++ 6 files changed, 591 insertions(+), 1 deletions(-) create mode 100644 block-curl.c
Makefile +6 -0
@@ -77,6 +77,10 @@ endif BLOCK_OBJS += block-raw-posix.o endif +ifdef CONFIG_CURL +BLOCK_OBJS += block-curl.o +endif + ###################################################################### # libqemu_common.a: Target independent part of system emulation. The # long term path is to suppress *all* target specific code in case of
@@ -185,6 +189,8 @@ endif LIBS+=$(VDE_LIBS) +LIBS+=$(CURL_LIBS) + cocoa.o: cocoa.m keymaps.o: keymaps.c keymaps.h
Makefile.target +1 -1
@@ -735,7 +735,7 @@ endif vl.o: qemu-options.h -$(QEMU_PROG): LIBS += $(SDL_LIBS) $(COCOA_LIBS) $(CURSES_LIBS) $(BRLAPI_LIBS) $(VDE_LIBS) +$(QEMU_PROG): LIBS += $(SDL_LIBS) $(COCOA_LIBS) $(CURSES_LIBS) $(BRLAPI_LIBS) $(VDE_LIBS) $(CURL_LIBS) $(QEMU_PROG): $(OBJS) ../libqemu_common.a libqemu.a $(LINK)
block-curl.c new file +540 -0
540 lines not on this page · raw message
block.c +12 -0
@@ -1504,6 +1504,18 @@ void bdrv_init(void)bdrv_register (&bdrv_qcow2);bdrv_register (&bdrv_parallels);bdrv_register (&bdrv_nbd); +#ifdef CONFIG_CURL +bdrv_register (&bdrv_http); +bdrv_register (&bdrv_https); +bdrv_register (&bdrv_ftp); +bdrv_register (&bdrv_ftps); +bdrv_register (&bdrv_tftp); +#if 0 +/* SSH over curl can't detect file sizes :-( */ +bdrv_register (&bdrv_sftp); +bdrv_register (&bdrv_scp); +#endif +#endif } void aio_pool_init(AIOPool *pool, int aiocb_size,
block.h +7 -0
@@ -20,6 +20,13 @@ extern BlockDriver bdrv_vvfat;extern BlockDriver bdrv_qcow2;extern BlockDriver bdrv_parallels;extern BlockDriver bdrv_nbd; +extern BlockDriver bdrv_http; +extern BlockDriver bdrv_https; +extern BlockDriver bdrv_ftp; +extern BlockDriver bdrv_ftps; +extern BlockDriver bdrv_sftp; +extern BlockDriver bdrv_scp; +extern BlockDriver bdrv_tftp; typedef struct BlockDriverInfo { /* in bytes, 0 if irrelevant */
configure +25 -0
@@ -181,6 +181,7 @@ bsd_user="no" build_docs="no" uname_release="" curses="yes" +curl="yes" pthread="yes" aio="yes" io_thread="no"
@@ -477,6 +478,8 @@ for opt do ;; --disable-curses) curses="no" ;; + --disable-curl) curl="no" + ;; --disable-nptl) nptl="no" ;; --enable-mixemu) mixemu="yes"
@@ -600,6 +603,7 @@ echo " --disable-brlapi disable BrlAPI" echo " --disable-vnc-tls disable TLS encryption for VNC server" echo " --disable-vnc-sasl disable SASL encryption for VNC server" echo " --disable-curses disable curses output" +echo " --disable-curl disable curl connectivity" echo " --disable-bluez disable bluez stack connectivity" echo " --disable-kvm disable KVM acceleration support" echo " --disable-nptl disable usermode NPTL support"
@@ -1062,6 +1066,21 @@ EOF fi # test "$curses" ########################################## +# curl probe + +if test "$curl" = "yes" ; then + curl=no + cat > $TMPC << EOF +#include <curl/curl.h> +int main(void) { return curl_easy_init(); } +EOF + curl_libs=`curl-config --libs` + if $cc $ARCH_CFLAGS $curl_libs -o $TMPE $TMPC > /dev/null 2> /dev/null ; then + curl=yes + fi +fi # test "$curl" + +########################################## # bluez support probe if test "$bluez" = "yes" ; then `pkg-config bluez 2> /dev/null` || bluez="no"
@@ -1311,6 +1330,7 @@ if test "$sdl" != "no" ; then echo "SDL static link $sdl_static" fi echo "curses support $curses" +echo "curl support $curl" echo "mingw32 support $mingw32" echo "Audio drivers $audio_drv_list" echo "Extra audio cards $audio_card_list"
@@ -1637,6 +1657,11 @@ fi if test "$inotify" = "yes" ; then echo "#define CONFIG_INOTIFY 1" >> $config_h fi +if test "$curl" = "yes" ; then + echo "CONFIG_CURL=yes" >> $config_mak + echo "CURL_LIBS=$curl_libs" >> $config_mak + echo "#define CONFIG_CURL 1" >> $config_h +fi if test "$brlapi" = "yes" ; then echo "CONFIG_BRLAPI=yes" >> $config_mak echo "#define CONFIG_BRLAPI 1" >> $config_h
Signature · 1 line
1.6.0.2
Quoted 7 lines · Alexander Graf
Currently Qemu can read from posix I/O and NBD. This patch adds a third protocol to the game: HTTP.
In certain situations it can be useful to access HTTP data directly, for example if you want to try out an http provided OS image, but don't know if you want to download it yet.
Quoted 7 lines
Using this patch you can now try it on on the fly. Just use it like:
qemu -cdrom http://host/path/my.iso
In order to not reinvent the wheel, this patch uses libcurl.
Signed-off-by Alexander Graf <[email protected]>
Just a comment on the form: changelog should appear after the "---" with diffstat to not be commited with the patch comment.
(they work like that on the linux kernel mailing list...)
Quoted 43 lines
v2 changes:
- fix the segfault (yay!)
- implement AIOv3 changes:
- remove synchronous API
- implement cachingv4 changes:
- enable other protocols (HTTPS, FTP, FTPS, TFTP, SFTP, SCP) (I only tested FTP so far, but they _should_ work)
v5 changes:
- address Anthony's comments
- deactivate ssh protocols againv6 changes:
- s/^I/ /g
- s/http/curl/gv7 changes:
- add Signed-off-by again
--- Makefile | 6 + Makefile.target | 2 +- block-curl.c | 540 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ block.c | 12 ++ block.h | 7 + configure | 25 +++ 6 files changed, 591 insertions(+), 1 deletions(-) create mode 100644 block-curl.cMakefile +0
-0
[...]
I tried to test out this patch, but unfortunately it doesn't apply to the latest git repo - have you got an updated patch?
Rich.
Signature · 4 lines
Richard Jones, Emerging Technologies, Red Hat http://et.redhat.com/~rjones New in Fedora 11: Fedora Windows cross-compiler. Compile Windows programs, test, and build Windows installers. Over 70 libraries supprt'd http://fedoraproject.org/wiki/MinGW http://www.annexia.org/fedora_mingw
Quoted 2 lines · Richard W.M. Jones
I tried to test out this patch, but unfortunately it doesn't apply to the latest git repo - have you got an updated patch?
Last time I checked it was in qemu.git.
Alex
Quoted 7 lines · Alexander Graf
Quoted 2 lines · Richard W.M. Jones
I tried to test out this patch, but unfortunately it doesn't apply to the latest git repo - have you got an updated patch?
Last time I checked it was in qemu.git.
So it is, sorry for the noise.
Great feature BTW ...
Rich.
Signature · 4 lines
Richard Jones, Emerging Technologies, Red Hat http://et.redhat.com/~rjones virt-p2v converts physical machines to virtual machines. Boot with a live CD or over the network (PXE) and turn machines into Xen guests. http://et.redhat.com/~rjones/virt-p2v
Quoted 3 lines · Richard W.M. Jones
I tried to test out this patch, but unfortunately it doesn't apply to the latest git repo - have you got an updated patch?
It's already in git.
Regards,
Anthony Liguori
Quoted 3 lines
Rich.
Quoted 1 line · Anthony Liguori
It's already in git.
Does it work on Windows for anybody? In my case the git version of configure does not even recognize the presence of libcurl. The hack below makes it to compile in, but running for example "qemu -cdrom http://aleksoft.net/fdbasecd.iso" hangs the emulator with the following log. Any hints?
Alex
curl --version
curl 7.19.5 (i686-pc-mingw32) libcurl/7.19.5 OpenSSL/0.9.8g zlib/1.2.3 libssh2/0.18
Protocols: tftp ftp telnet dict ldap http file https ftps scp sftp
Features: Largefile NTLM SSL libz
<log>
C:\qemu-dist>qemu -cdrom http://aleksoft.net/fdbasecd.iso
CURL: Opening http://aleksoft.net/fdbasecd.iso
* About to connect() to aleksoft.net port 80 (#0)
* Trying 74.220.214.207... * connected
* Connected to aleksoft.net (74.220.214.207) port 80 (#0)
Quoted 1 line
HEAD /fdbasecd.iso HTTP/1.1
Host: aleksoft.net
Accept: */*
< HTTP/1.1 200 OK
< Date: Wed, 27 May 2009 16:28:38 GMT
< Server: Apache/2.2.11 (Unix) mod_ssl/2.2.11 OpenSSL/0.9.8i DAV/2 mod_auth_pass through/2.1 mod_bwlimited/1.4 FrontPage/5.0.2.2635
< Last-Modified: Wed, 27 May 2009 16:27:53 GMT
< ETag: "40719c4-7f2800-46ae75337d440"
< Accept-Ranges: bytes
< Content-Length: 8333312
< Content-Type: application/octet-stream
<
* Connection #0 to host aleksoft.net left intact
CURL: Size = 8333312
* Closing connection #0
CURL (AIO): Reading 2048 at 0 (0-0)
CURL (AIO): Sock action 2 on fd 1828
CURL (AIO): Sock action 4 on fd 1828
* About to connect() to aleksoft.net port 80 (#0)
* Trying 74.220.214.207... CURL (AIO): Sock action 2 on fd 1784
* Connected to aleksoft.net (74.220.214.207) port 80 (#0)
Quoted 1 line
GET /fdbasecd.iso HTTP/1.1
Range: bytes=0-0
Host: aleksoft.net
Accept: */*
CURL (AIO): Sock action 1 on fd 1784
< HTTP/1.1 206 Partial Content
< Date: Wed, 27 May 2009 16:28:38 GMT
< Server: Apache/2.2.11 (Unix) mod_ssl/2.2.11 OpenSSL/0.9.8i DAV/2 mod_auth_pass through/2.1 mod_bwlimited/1.4 FrontPage/5.0.2.2635
< Last-Modified: Wed, 27 May 2009 16:27:53 GMT
< ETag: "40719c4-7f2800-46ae75337d440"
< Accept-Ranges: bytes
< Content-Length: 1
< Content-Range: bytes 0-0/8333312
< Content-Type: application/octet-stream
<
CURL: Just reading 1 bytes
read 1
CURL (AIO): Sock action 4 on fd 1784
* Connection #0 to host aleksoft.net left intact
</log>
block/curl.c +7 -2
@@ -25,8 +25,8 @@#include "block_int.h" #include <curl/curl.h> -// #define DEBUG -// #define DEBUG_VERBOSE +#define DEBUG_CURL +#define DEBUG_VERBOSE #ifdef DEBUG_CURL #define dprintf (fmt, ...) do { printf(fmt, ## __VA_ARGS__); } while (0)
@@ -141,6 +141,7 @@ static size_t curl_read_cb(void *ptr, size_t size, size_t nmemb, void *opaque) } read_end: +printf ("read %x\n" ,realsize);return realsize; }
@@ -255,7 +256,11 @@ static CURLState *curl_init_state(BDRVCURLState *s)break ; }if (!state) { +#ifndef _WIN32usleep (100 ); +#else +Sleep (0 ); +#endif curl_multi_do (s); } }while (!state);
configure +2 -2
@@ -640,7 +640,7 @@ if test "$mingw32" = "yes" ; then oss="no" linux_user="no" bsd_user="no" - OS_CFLAGS="$OS_CFLAGS -DWIN32_LEAN_AND_MEAN -DWINVER=0x501" + OS_CFLAGS="$OS_CFLAGS -DWIN32_LEAN_AND_MEAN -DWINVER=0x501 -DCURL_STATICLIB" fi if test ! -x "$(which cgcc 2>/dev/null)"; then
@@ -1103,7 +1103,7 @@ if test "$curl" = "yes" ; then int main(void) { return curl_easy_init(); } EOF curl_libs=`curl-config --libs 2>/dev/null` - if $cc $ARCH_CFLAGS $curl_libs -o $TMPE $TMPC > /dev/null 2> /dev/null ; then + if $cc $ARCH_CFLAGS -DCURL_STATICLIB -o $TMPE $TMPC $curl_libs > /dev/null 2> /dev/null ; then curl=yes fi fi # test "$curl"
Quoted 5 lines · Consul
Quoted 1 line · Anthony Liguori
It's already in git.
Does it work on Windows for anybody? In my case the git version of configure does not even recognize the presence of libcurl. \
Is curl-config in your path? Does the following help?
Quoted 3 lines
The hack below makes it to compile in, but running for example "qemu -cdrom http://aleksoft.net/fdbasecd.iso" hangs the emulator with the following log. Any hints?
It probably doesn't. AIO on Windows is really busted. It needs some serious work.
Regards,
Anthony Liguori
commit 2712d0cbaa816ab6b0976f3d65b7cc9f222dd63f
Author: Anthony Liguori <[email protected]> Date: Thu May 28 03:44:23 2009 -0500
Make sure to respect curl-config CFLAGS
Makefile.target +6 -1
@@ -548,6 +548,11 @@ ifdef CONFIG_BLUEZ LIBS += $(CONFIG_BLUEZ_LIBS) endif +ifdef CONFIG_CURL +CPPFLAGS += $(CURL_CFLAGS) +LIBS += $(CURL_LIBS) +endif + # xen backend driver support XEN_OBJS := xen_machine_pv.o xen_domainbuild.o ifeq ($(CONFIG_XEN), yes)
@@ -732,7 +737,7 @@ endif vl.o: qemu-options.h -$(QEMU_PROG): LIBS += $(SDL_LIBS) $(COCOA_LIBS) $(CURSES_LIBS) $(BRLAPI_LIBS) $(VDE_LIBS) $(CURL_LIBS) +$(QEMU_PROG): LIBS += $(SDL_LIBS) $(COCOA_LIBS) $(CURSES_LIBS) $(BRLAPI_LIBS) $(VDE_LIBS) $(QEMU_PROG): ARLIBS=../libqemu_common.a libqemu.a $(HWLIB) $(QEMU_PROG): $(OBJS) ../libqemu_common.a libqemu.a $(HWLIB) $(call LINK,$(OBJS))
configure +3 -1
@@ -1079,7 +1079,8 @@ if test "$curl" = "yes" ; then int main(void) { return curl_easy_init(); } EOF curl_libs=`curl-config --libs 2>/dev/null` - if $cc $ARCH_CFLAGS $curl_libs -o $TMPE $TMPC > /dev/null 2> /dev/null ; then + curl_cflags=`curl-config --cflags 2>/dev/null` + if $cc $ARCH_CFLAGS $curl_cflags $curl_libs -o $TMPE $TMPC > /dev/null 2> /dev/null ; then curl=yes fi fi # test "$curl"
@@ -1663,6 +1664,7 @@ fi if test "$curl" = "yes" ; then echo "CONFIG_CURL=yes" >> $config_mak echo "CURL_LIBS=$curl_libs" >> $config_mak + echo "CURL_CFLAGS=$curl_cflags" >> $config_mak echo "#define CONFIG_CURL 1" >> $config_h fi if test "$brlapi" = "yes" ; then
Quoted 3 lines · Anthony Liguori
Is curl-config in your path? Does the following help?
Yes.
How about this patch on top of yours? Still does not run well on windows, but at least compiles on both linux and windows. Without re-arranging the args order linker can't resolve the symbols.
Makefile +2 -0
@@ -202,6 +202,8 @@ endif LIBS+=$(CURL_LIBS) +block/curl.o: CFLAGS += $(CURL_CFLAGS) + cocoa.o: cocoa.m keymaps.o: keymaps.c keymaps.h
block/curl.c +4 -0
@@ -255,7 +255,11 @@ static CURLState *curl_init_state(BDRVCURLState *s)break ; }if (!state) { +#ifndef _WIN32usleep (100 ); +#else +Sleep (0 ); +#endif curl_multi_do (s); } }while (!state);
configure +1 -1
@@ -1080,7 +1080,7 @@ int main(void) { return curl_easy_init(); } EOF curl_libs=`curl-config --libs 2>/dev/null` curl_cflags=`curl-config --cflags 2>/dev/null` - if $cc $ARCH_CFLAGS $curl_cflags $curl_libs -o $TMPE $TMPC > /dev/null 2> /dev/null ; then + if $cc $ARCH_CFLAGS $curl_cflags -o $TMPE $TMPC $curl_libs > /dev/null 2> /dev/null ; then curl=yes fi fi # test "$curl"
Makefile +2 -0
@@ -202,6 +202,8 @@ endif LIBS+=$(CURL_LIBS) +block/curl.o: CFLAGS += $(CURL_CFLAGS) + cocoa.o: cocoa.m keymaps.o: keymaps.c keymaps.h
block/curl.c +4 -0
@@ -255,7 +255,11 @@ static CURLState *curl_init_state(BDRVCURLState *s)break ; }if (!state) { +#ifndef _WIN32usleep (100 ); +#else +Sleep (0 ); +#endif curl_multi_do (s); } }while (!state);
configure +1 -1
@@ -1080,7 +1080,7 @@ int main(void) { return curl_easy_init(); } EOF curl_libs=`curl-config --libs 2>/dev/null` curl_cflags=`curl-config --cflags 2>/dev/null` - if $cc $ARCH_CFLAGS $curl_cflags $curl_libs -o $TMPE $TMPC > /dev/null 2> /dev/null ; then + if $cc $ARCH_CFLAGS $curl_cflags -o $TMPE $TMPC $curl_libs > /dev/null 2> /dev/null ; then curl=yes fi fi # test "$curl"
Quoted 3 lines · Anthony Liguori
Is curl-config in your path? Does the following help?
Yes.
How about this patch on top of yours? Still does not run well on windows, but at least compiles on both linux and windows. Without re-arranging the args order linker can't resolve the symbols.
Makefile +2 -0
@@ -202,6 +202,8 @@ endif LIBS+=$(CURL_LIBS) +block/curl.o: CFLAGS += $(CURL_CFLAGS) + cocoa.o: cocoa.m keymaps.o: keymaps.c keymaps.h
block/curl.c +4 -0
@@ -255,7 +255,11 @@ static CURLState *curl_init_state(BDRVCURLState *s)break ; }if (!state) { +#ifndef _WIN32usleep (100 ); +#else +Sleep (0 ); +#endif curl_multi_do (s); } }while (!state);
configure +1 -1
@@ -1080,7 +1080,7 @@ int main(void) { return curl_easy_init(); } EOF curl_libs=`curl-config --libs 2>/dev/null` curl_cflags=`curl-config --cflags 2>/dev/null` - if $cc $ARCH_CFLAGS $curl_cflags $curl_libs -o $TMPE $TMPC > /dev/null 2> /dev/null ; then + if $cc $ARCH_CFLAGS $curl_cflags -o $TMPE $TMPC $curl_libs > /dev/null 2> /dev/null ; then curl=yes fi fi # test "$curl"
Makefile +2 -0
@@ -202,6 +202,8 @@ endif LIBS+=$(CURL_LIBS) +block/curl.o: CFLAGS += $(CURL_CFLAGS) + cocoa.o: cocoa.m keymaps.o: keymaps.c keymaps.h
block/curl.c +4 -0
@@ -255,7 +255,11 @@ static CURLState *curl_init_state(BDRVCURLState *s)break ; }if (!state) { +#ifndef _WIN32usleep (100 ); +#else +Sleep (0 ); +#endif curl_multi_do (s); } }while (!state);
configure +1 -1
@@ -1080,7 +1080,7 @@ int main(void) { return curl_easy_init(); } EOF curl_libs=`curl-config --libs 2>/dev/null` curl_cflags=`curl-config --cflags 2>/dev/null` - if $cc $ARCH_CFLAGS $curl_cflags $curl_libs -o $TMPE $TMPC > /dev/null 2> /dev/null ; then + if $cc $ARCH_CFLAGS $curl_cflags -o $TMPE $TMPC $curl_libs > /dev/null 2> /dev/null ; then curl=yes fi fi # test "$curl"