Skip to contentSkip to conversation

Add HTTP protocol using curl v7

PATCH In review Alexander Graf 5 participants 10 messages May 22, 2009 – May 28, 2009

Subsystems (2)
lore
Share & tools
lore
#0 lore raw

PATCH 1/1 Add HTTP protocol using curl v7

Currently Qemu can read from posix I/O and NBD. This patch adds a third protocol to the game: HTTP.

In certain situations it can be useful to access HTTP data directly, for example if you want to try out an http provided OS image, but don't know if you want to download it yet.

Using this patch you can now try it on on the fly. Just use it like:

qemu -cdrom http://host/path/my.iso

In order to not reinvent the wheel, this patch uses libcurl.

v2 changes:

- fix the segfault (yay!)
- implement AIO

v3 changes:

- remove synchronous API
- implement caching

v4 changes:

- enable other protocols (HTTPS, FTP, FTPS, TFTP, SFTP, SCP) (I only tested FTP so far, but they _should_ work)

v5 changes:

- address Anthony's comments
- deactivate ssh protocols again

v6 changes:

- s/^I/ /g
- s/http/curl/g

v7 changes:

- add Signed-off-by again

---
 Makefile        |    6 +
 Makefile.target |    2 +-
 block-curl.c    |  540 +++++++++++++++++++++++++++++++++++++++++++++++++++++++
 block.c         |   12 ++
 block.h         |    7 +
 configure       |   25 +++
 6 files changed, 591 insertions(+), 1 deletions(-)
 create mode 100644 block-curl.c
Makefile +6 -0
index 3c70068..421cd41 100644
@@ -77,6 +77,10 @@ endif
 BLOCK_OBJS += block-raw-posix.o
 endif
 
+ifdef CONFIG_CURL
+BLOCK_OBJS += block-curl.o
+endif
+
 ######################################################################
 # libqemu_common.a: Target independent part of system emulation. The
 # long term path is to suppress *all* target specific code in case of
@@ -185,6 +189,8 @@ endif
 
 LIBS+=$(VDE_LIBS)
 
+LIBS+=$(CURL_LIBS)
+
 cocoa.o: cocoa.m
 
 keymaps.o: keymaps.c keymaps.h
Makefile.target +1 -1
index f735105..3a9226c 100644
@@ -735,7 +735,7 @@ endif
 
 vl.o: qemu-options.h
 
-$(QEMU_PROG): LIBS += $(SDL_LIBS) $(COCOA_LIBS) $(CURSES_LIBS) $(BRLAPI_LIBS) $(VDE_LIBS)
+$(QEMU_PROG): LIBS += $(SDL_LIBS) $(COCOA_LIBS) $(CURSES_LIBS) $(BRLAPI_LIBS) $(VDE_LIBS) $(CURL_LIBS)
 
 $(QEMU_PROG): $(OBJS) ../libqemu_common.a libqemu.a
 	$(LINK)
block-curl.c new file +540 -0

540 lines not on this page · raw message

block.c +12 -0
index 3d1223d..55af1eb 100644
@@ -1504,6 +1504,18 @@ void bdrv_init(void)
     bdrv_register(&bdrv_qcow2);
     bdrv_register(&bdrv_parallels);
     bdrv_register(&bdrv_nbd);
+#ifdef CONFIG_CURL
+    bdrv_register(&bdrv_http);
+    bdrv_register(&bdrv_https);
+    bdrv_register(&bdrv_ftp);
+    bdrv_register(&bdrv_ftps);
+    bdrv_register(&bdrv_tftp);
+#if 0
+    /* SSH over curl can't detect file sizes :-( */
+    bdrv_register(&bdrv_sftp);
+    bdrv_register(&bdrv_scp);
+#endif
+#endif
 }
 
 void aio_pool_init(AIOPool *pool, int aiocb_size,
block.h +7 -0
index 5aef076..5e554e8 100644
@@ -20,6 +20,13 @@ extern BlockDriver bdrv_vvfat;
 extern BlockDriver bdrv_qcow2;
 extern BlockDriver bdrv_parallels;
 extern BlockDriver bdrv_nbd;
+extern BlockDriver bdrv_http;
+extern BlockDriver bdrv_https;
+extern BlockDriver bdrv_ftp;
+extern BlockDriver bdrv_ftps;
+extern BlockDriver bdrv_sftp;
+extern BlockDriver bdrv_scp;
+extern BlockDriver bdrv_tftp;
 
 typedef struct BlockDriverInfo {
     /* in bytes, 0 if irrelevant */
configure +25 -0
index 82fb60a..49b5b84 100755
@@ -181,6 +181,7 @@ bsd_user="no"
 build_docs="no"
 uname_release=""
 curses="yes"
+curl="yes"
 pthread="yes"
 aio="yes"
 io_thread="no"
@@ -477,6 +478,8 @@ for opt do
   ;;
   --disable-curses) curses="no"
   ;;
+  --disable-curl) curl="no"
+  ;;
   --disable-nptl) nptl="no"
   ;;
   --enable-mixemu) mixemu="yes"
@@ -600,6 +603,7 @@ echo "  --disable-brlapi         disable BrlAPI"
 echo "  --disable-vnc-tls        disable TLS encryption for VNC server"
 echo "  --disable-vnc-sasl       disable SASL encryption for VNC server"
 echo "  --disable-curses         disable curses output"
+echo "  --disable-curl           disable curl connectivity"
 echo "  --disable-bluez          disable bluez stack connectivity"
 echo "  --disable-kvm            disable KVM acceleration support"
 echo "  --disable-nptl           disable usermode NPTL support"
@@ -1062,6 +1066,21 @@ EOF
 fi # test "$curses"
 
 ##########################################
+# curl probe
+
+if test "$curl" = "yes" ; then
+  curl=no
+  cat > $TMPC << EOF
+#include <curl/curl.h>
+int main(void) { return curl_easy_init(); }
+EOF
+  curl_libs=`curl-config --libs`
+ if $cc $ARCH_CFLAGS $curl_libs -o $TMPE $TMPC > /dev/null 2> /dev/null ; then
+    curl=yes
+  fi
+fi # test "$curl"
+
+##########################################
 # bluez support probe
 if test "$bluez" = "yes" ; then
   `pkg-config bluez 2> /dev/null` || bluez="no"
@@ -1311,6 +1330,7 @@ if test "$sdl" != "no" ; then
     echo "SDL static link   $sdl_static"
 fi
 echo "curses support    $curses"
+echo "curl support      $curl"
 echo "mingw32 support   $mingw32"
 echo "Audio drivers     $audio_drv_list"
 echo "Extra audio cards $audio_card_list"
@@ -1637,6 +1657,11 @@ fi
 if test "$inotify" = "yes" ; then
   echo "#define CONFIG_INOTIFY 1" >> $config_h
 fi
+if test "$curl" = "yes" ; then
+  echo "CONFIG_CURL=yes" >> $config_mak
+  echo "CURL_LIBS=$curl_libs" >> $config_mak
+  echo "#define CONFIG_CURL 1" >> $config_h
+fi
 if test "$brlapi" = "yes" ; then
   echo "CONFIG_BRLAPI=yes" >> $config_mak
   echo "#define CONFIG_BRLAPI 1" >> $config_h
Signature · 1 line
1.6.0.2
#1 lore raw
Quoted 7 lines · Alexander Graf

Currently Qemu can read from posix I/O and NBD. This patch adds a third protocol to the game: HTTP.

In certain situations it can be useful to access HTTP data directly, for example if you want to try out an http provided OS image, but don't know if you want to download it yet.

Quoted 7 lines

Using this patch you can now try it on on the fly. Just use it like:

qemu -cdrom http://host/path/my.iso

In order to not reinvent the wheel, this patch uses libcurl.

Just a comment on the form: changelog should appear after the "---" with diffstat to not be commited with the patch comment.
(they work like that on the linux kernel mailing list...)

Quoted 43 lines

v2 changes:

- fix the segfault (yay!)
- implement AIO

v3 changes:

- remove synchronous API
- implement caching

v4 changes:

- enable other protocols (HTTPS, FTP, FTPS, TFTP, SFTP, SCP) (I only tested FTP so far, but they _should_ work)

v5 changes:

- address Anthony's comments
- deactivate ssh protocols again

v6 changes:

- s/^I/ /g
- s/http/curl/g

v7 changes:

- add Signed-off-by again

---
 Makefile        |    6 +
 Makefile.target |    2 +-
 block-curl.c    |  540 +++++++++++++++++++++++++++++++++++++++++++++++++++++++
 block.c         |   12 ++
 block.h         |    7 +
 configure       |   25 +++
 6 files changed, 591 insertions(+), 1 deletions(-)
 create mode 100644 block-curl.c
Makefile +0 -0
index 3c70068..421cd41 100644

[...]

#2 lore raw
In reply to Alexander Graf

I tried to test out this patch, but unfortunately it doesn't apply to the latest git repo - have you got an updated patch?

Rich.

Signature · 4 lines
Richard Jones, Emerging Technologies, Red Hat  http://et.redhat.com/~rjones
New in Fedora 11: Fedora Windows cross-compiler. Compile Windows
programs, test, and build Windows installers. Over 70 libraries supprt'd
http://fedoraproject.org/wiki/MinGW http://www.annexia.org/fedora_mingw
#3 lore raw
Quoted 2 lines · Richard W.M. Jones

I tried to test out this patch, but unfortunately it doesn't apply to the latest git repo - have you got an updated patch?

Last time I checked it was in qemu.git.

Alex

#4 lore raw
Quoted 7 lines · Alexander Graf
Quoted 2 lines · Richard W.M. Jones

I tried to test out this patch, but unfortunately it doesn't apply to the latest git repo - have you got an updated patch?

Last time I checked it was in qemu.git.

So it is, sorry for the noise.

Great feature BTW ...

Rich.

Signature · 4 lines
Richard Jones, Emerging Technologies, Red Hat  http://et.redhat.com/~rjones
virt-p2v converts physical machines to virtual machines.  Boot with a
live CD or over the network (PXE) and turn machines into Xen guests.
http://et.redhat.com/~rjones/virt-p2v
#5 lore raw
In reply to Richard W.M. Jones
Quoted 3 lines · Richard W.M. Jones

I tried to test out this patch, but unfortunately it doesn't apply to the latest git repo - have you got an updated patch?

It's already in git.

Regards,

Anthony Liguori

Quoted 3 lines

Rich.

#6 lore raw
Quoted 1 line · Anthony Liguori

It's already in git.

Does it work on Windows for anybody? In my case the git version of configure does not even recognize the presence of libcurl. The hack below makes it to compile in, but running for example "qemu -cdrom http://aleksoft.net/fdbasecd.iso" hangs the emulator with the following log. Any hints?

Alex

curl --version
curl 7.19.5 (i686-pc-mingw32) libcurl/7.19.5 OpenSSL/0.9.8g zlib/1.2.3 libssh2/0.18
Protocols: tftp ftp telnet dict ldap http file https ftps scp sftp
Features: Largefile NTLM SSL libz

<log>

C:\qemu-dist>qemu -cdrom http://aleksoft.net/fdbasecd.iso
CURL: Opening http://aleksoft.net/fdbasecd.iso
* About to connect() to aleksoft.net port 80 (#0)
* Trying 74.220.214.207... * connected
* Connected to aleksoft.net (74.220.214.207) port 80 (#0)

Quoted 1 line

HEAD /fdbasecd.iso HTTP/1.1

Host: aleksoft.net
Accept: */*

< HTTP/1.1 200 OK
< Date: Wed, 27 May 2009 16:28:38 GMT
< Server: Apache/2.2.11 (Unix) mod_ssl/2.2.11 OpenSSL/0.9.8i DAV/2 mod_auth_pass through/2.1 mod_bwlimited/1.4 FrontPage/5.0.2.2635
< Last-Modified: Wed, 27 May 2009 16:27:53 GMT
< ETag: "40719c4-7f2800-46ae75337d440"
< Accept-Ranges: bytes
< Content-Length: 8333312
< Content-Type: application/octet-stream
<
* Connection #0 to host aleksoft.net left intact
CURL: Size = 8333312
* Closing connection #0
CURL (AIO): Reading 2048 at 0 (0-0)
CURL (AIO): Sock action 2 on fd 1828
CURL (AIO): Sock action 4 on fd 1828
* About to connect() to aleksoft.net port 80 (#0)
* Trying 74.220.214.207... CURL (AIO): Sock action 2 on fd 1784
* Connected to aleksoft.net (74.220.214.207) port 80 (#0)

Quoted 1 line

GET /fdbasecd.iso HTTP/1.1

Range: bytes=0-0
Host: aleksoft.net
Accept: */*

CURL (AIO): Sock action 1 on fd 1784
< HTTP/1.1 206 Partial Content
< Date: Wed, 27 May 2009 16:28:38 GMT
< Server: Apache/2.2.11 (Unix) mod_ssl/2.2.11 OpenSSL/0.9.8i DAV/2 mod_auth_pass through/2.1 mod_bwlimited/1.4 FrontPage/5.0.2.2635
< Last-Modified: Wed, 27 May 2009 16:27:53 GMT
< ETag: "40719c4-7f2800-46ae75337d440"
< Accept-Ranges: bytes
< Content-Length: 1
< Content-Range: bytes 0-0/8333312
< Content-Type: application/octet-stream
<
CURL: Just reading 1 bytes
read 1
CURL (AIO): Sock action 4 on fd 1784
* Connection #0 to host aleksoft.net left intact

</log>

block/curl.c +7 -2
index 5d1487d..14ce646 100644
@@ -25,8 +25,8 @@
 #include "block_int.h"
 #include <curl/curl.h>
 
-// #define DEBUG
-// #define DEBUG_VERBOSE
+ #define DEBUG_CURL
+ #define DEBUG_VERBOSE
 
 #ifdef DEBUG_CURL
 #define dprintf(fmt, ...) do { printf(fmt, ## __VA_ARGS__); } while (0)
@@ -141,6 +141,7 @@ static size_t curl_read_cb(void *ptr, size_t size, size_t nmemb, void *opaque)
     }
 
 read_end:
+    printf("read %x\n",realsize);
     return realsize;
 }
 
@@ -255,7 +256,11 @@ static CURLState *curl_init_state(BDRVCURLState *s)
             break;
         }
         if (!state) {
+#ifndef _WIN32
             usleep(100);
+#else
+            Sleep(0);
+#endif
             curl_multi_do(s);
         }
     } while(!state);
configure +2 -2
index bcf1207..c85c29e 100755
@@ -640,7 +640,7 @@ if test "$mingw32" = "yes" ; then
     oss="no"
     linux_user="no"
     bsd_user="no"
-    OS_CFLAGS="$OS_CFLAGS -DWIN32_LEAN_AND_MEAN -DWINVER=0x501"
+    OS_CFLAGS="$OS_CFLAGS -DWIN32_LEAN_AND_MEAN -DWINVER=0x501 -DCURL_STATICLIB"
 fi
 
 if test ! -x "$(which cgcc 2>/dev/null)"; then
@@ -1103,7 +1103,7 @@ if test "$curl" = "yes" ; then
 int main(void) { return curl_easy_init(); }
 EOF
   curl_libs=`curl-config --libs 2>/dev/null`
- if $cc $ARCH_CFLAGS $curl_libs -o $TMPE $TMPC > /dev/null 2> /dev/null ; then
+ if $cc $ARCH_CFLAGS -DCURL_STATICLIB -o $TMPE $TMPC $curl_libs > /dev/null 2> /dev/null ; then
     curl=yes
   fi
 fi # test "$curl"
#7 lore raw
Quoted 5 lines · Consul
Quoted 1 line · Anthony Liguori

It's already in git.

Does it work on Windows for anybody? In my case the git version of configure does not even recognize the presence of libcurl. \

Is curl-config in your path? Does the following help?

Quoted 3 lines

The hack below makes it to compile in, but running for example "qemu -cdrom http://aleksoft.net/fdbasecd.iso" hangs the emulator with the following log. Any hints?

It probably doesn't. AIO on Windows is really busted. It needs some serious work.

Regards,

Anthony Liguori

commit 2712d0cbaa816ab6b0976f3d65b7cc9f222dd63f

Author: Anthony Liguori <[email protected]>
Date:   Thu May 28 03:44:23 2009 -0500

Make sure to respect curl-config CFLAGS

Makefile.target +6 -1
index 445d55f..0078668 100644
@@ -548,6 +548,11 @@ ifdef CONFIG_BLUEZ
 LIBS += $(CONFIG_BLUEZ_LIBS)
 endif
 
+ifdef CONFIG_CURL
+CPPFLAGS += $(CURL_CFLAGS)
+LIBS += $(CURL_LIBS)
+endif
+
 # xen backend driver support
 XEN_OBJS := xen_machine_pv.o xen_domainbuild.o
 ifeq ($(CONFIG_XEN), yes)
@@ -732,7 +737,7 @@ endif
 
 vl.o: qemu-options.h
 
-$(QEMU_PROG): LIBS += $(SDL_LIBS) $(COCOA_LIBS) $(CURSES_LIBS) $(BRLAPI_LIBS) $(VDE_LIBS) $(CURL_LIBS)
+$(QEMU_PROG): LIBS += $(SDL_LIBS) $(COCOA_LIBS) $(CURSES_LIBS) $(BRLAPI_LIBS) $(VDE_LIBS)
 $(QEMU_PROG): ARLIBS=../libqemu_common.a libqemu.a $(HWLIB)
 $(QEMU_PROG): $(OBJS) ../libqemu_common.a libqemu.a $(HWLIB)
 	$(call LINK,$(OBJS))
configure +3 -1
index 21c0633..2e68fe3 100755
@@ -1079,7 +1079,8 @@ if test "$curl" = "yes" ; then
 int main(void) { return curl_easy_init(); }
 EOF
   curl_libs=`curl-config --libs 2>/dev/null`
- if $cc $ARCH_CFLAGS $curl_libs -o $TMPE $TMPC > /dev/null 2> /dev/null ; then
+  curl_cflags=`curl-config --cflags 2>/dev/null`
+ if $cc $ARCH_CFLAGS $curl_cflags $curl_libs -o $TMPE $TMPC > /dev/null 2> /dev/null ; then
     curl=yes
   fi
 fi # test "$curl"
@@ -1663,6 +1664,7 @@ fi
 if test "$curl" = "yes" ; then
   echo "CONFIG_CURL=yes" >> $config_mak
   echo "CURL_LIBS=$curl_libs" >> $config_mak
+  echo "CURL_CFLAGS=$curl_cflags" >> $config_mak
   echo "#define CONFIG_CURL 1" >> $config_h
 fi
 if test "$brlapi" = "yes" ; then
#8 lore raw
Quoted 3 lines · Anthony Liguori

Is curl-config in your path? Does the following help?

Yes.
How about this patch on top of yours? Still does not run well on windows, but at least compiles on both linux and windows. Without re-arranging the args order linker can't resolve the symbols.

Makefile +2 -0
index d7b9985..4830285 100644
@@ -202,6 +202,8 @@ endif
 
 LIBS+=$(CURL_LIBS)
 
+block/curl.o: CFLAGS += $(CURL_CFLAGS)
+
 cocoa.o: cocoa.m
 
 keymaps.o: keymaps.c keymaps.h
block/curl.c +4 -0
index e1a553f..5534680 100644
@@ -255,7 +255,11 @@ static CURLState *curl_init_state(BDRVCURLState *s)
             break;
         }
         if (!state) {
+#ifndef _WIN32
             usleep(100);
+#else
+            Sleep(0);
+#endif
             curl_multi_do(s);
         }
     } while(!state);
configure +1 -1
index 2e68fe3..fdbc352 100755
@@ -1080,7 +1080,7 @@ int main(void) { return curl_easy_init(); }
 EOF
   curl_libs=`curl-config --libs 2>/dev/null`
   curl_cflags=`curl-config --cflags 2>/dev/null`
- if $cc $ARCH_CFLAGS $curl_cflags $curl_libs -o $TMPE $TMPC > /dev/null 2> /dev/null ; then
+ if $cc $ARCH_CFLAGS $curl_cflags -o $TMPE $TMPC $curl_libs > /dev/null 2> /dev/null ; then
     curl=yes
   fi
 fi # test "$curl"
Makefile +2 -0
index d7b9985..4830285 100644
@@ -202,6 +202,8 @@ endif
 
 LIBS+=$(CURL_LIBS)
 
+block/curl.o: CFLAGS += $(CURL_CFLAGS)
+
 cocoa.o: cocoa.m
 
 keymaps.o: keymaps.c keymaps.h
block/curl.c +4 -0
index e1a553f..5534680 100644
@@ -255,7 +255,11 @@ static CURLState *curl_init_state(BDRVCURLState *s)
             break;
         }
         if (!state) {
+#ifndef _WIN32
             usleep(100);
+#else
+            Sleep(0);
+#endif
             curl_multi_do(s);
         }
     } while(!state);
configure +1 -1
index 2e68fe3..fdbc352 100755
@@ -1080,7 +1080,7 @@ int main(void) { return curl_easy_init(); }
 EOF
   curl_libs=`curl-config --libs 2>/dev/null`
   curl_cflags=`curl-config --cflags 2>/dev/null`
- if $cc $ARCH_CFLAGS $curl_cflags $curl_libs -o $TMPE $TMPC > /dev/null 2> /dev/null ; then
+ if $cc $ARCH_CFLAGS $curl_cflags -o $TMPE $TMPC $curl_libs > /dev/null 2> /dev/null ; then
     curl=yes
   fi
 fi # test "$curl"
#9 lore raw
In reply to Anthony Liguori
Quoted 3 lines · Anthony Liguori

Is curl-config in your path? Does the following help?

Yes.
How about this patch on top of yours? Still does not run well on windows, but at least compiles on both linux and windows. Without re-arranging the args order linker can't resolve the symbols.

Makefile +2 -0
index d7b9985..4830285 100644
@@ -202,6 +202,8 @@ endif
 
 LIBS+=$(CURL_LIBS)
 
+block/curl.o: CFLAGS += $(CURL_CFLAGS)
+
 cocoa.o: cocoa.m
 
 keymaps.o: keymaps.c keymaps.h
block/curl.c +4 -0
index e1a553f..5534680 100644
@@ -255,7 +255,11 @@ static CURLState *curl_init_state(BDRVCURLState *s)
             break;
         }
         if (!state) {
+#ifndef _WIN32
             usleep(100);
+#else
+            Sleep(0);
+#endif
             curl_multi_do(s);
         }
     } while(!state);
configure +1 -1
index 2e68fe3..fdbc352 100755
@@ -1080,7 +1080,7 @@ int main(void) { return curl_easy_init(); }
 EOF
   curl_libs=`curl-config --libs 2>/dev/null`
   curl_cflags=`curl-config --cflags 2>/dev/null`
- if $cc $ARCH_CFLAGS $curl_cflags $curl_libs -o $TMPE $TMPC > /dev/null 2> /dev/null ; then
+ if $cc $ARCH_CFLAGS $curl_cflags -o $TMPE $TMPC $curl_libs > /dev/null 2> /dev/null ; then
     curl=yes
   fi
 fi # test "$curl"
Makefile +2 -0
index d7b9985..4830285 100644
@@ -202,6 +202,8 @@ endif
 
 LIBS+=$(CURL_LIBS)
 
+block/curl.o: CFLAGS += $(CURL_CFLAGS)
+
 cocoa.o: cocoa.m
 
 keymaps.o: keymaps.c keymaps.h
block/curl.c +4 -0
index e1a553f..5534680 100644
@@ -255,7 +255,11 @@ static CURLState *curl_init_state(BDRVCURLState *s)
             break;
         }
         if (!state) {
+#ifndef _WIN32
             usleep(100);
+#else
+            Sleep(0);
+#endif
             curl_multi_do(s);
         }
     } while(!state);
configure +1 -1
index 2e68fe3..fdbc352 100755
@@ -1080,7 +1080,7 @@ int main(void) { return curl_easy_init(); }
 EOF
   curl_libs=`curl-config --libs 2>/dev/null`
   curl_cflags=`curl-config --cflags 2>/dev/null`
- if $cc $ARCH_CFLAGS $curl_cflags $curl_libs -o $TMPE $TMPC > /dev/null 2> /dev/null ; then
+ if $cc $ARCH_CFLAGS $curl_cflags -o $TMPE $TMPC $curl_libs > /dev/null 2> /dev/null ; then
     curl=yes
   fi
 fi # test "$curl"