qemu-img: make convert async
RFC PATCH 1/1 qemu-img: make convert async
this is something I have been thinking about for almost 2 years now. we heavily have the following two use cases when using qemu-img convert.
a) reading from NFS and writing to iSCSI for deploying templates
b) reading from iSCSI and writing to NFS for backups
In both processes we use libiscsi and libnfs so we have no kernel pagecache. As qemu-img convert is implemented with sync operations that means we read one buffer and then write it. No parallelism and each sync request takes as long as it takes until it is completed.
What I put together is an approach to use aio routines for the conversion process to have ideally read and write happening in parallel.
The code is far from clean or complete, but I would appreaciate comments and thoughts from you.
So far I have the following runtimes when reading an uncompressed QCOW2 from NFS and writing it to iSCSI (raw):
qemu-img (master)
nfs -> iscsi 33 secs
nfs -> ram 19 secs
ram -> iscsi 14 secs
qemu-img-async
nfs -> iscsi 23 secs
nfs -> ram 17 secs
ram -> iscsi 14 secs
Its visible that on master the runtimes add up as expected. The async branch is faster, but not as fast as I would have expected. I would expect the runtime to be as slow as the slowest of the two involved transfers.
Thank you,
Peter
--- qemu-img.c | 271 +++++++++++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 199 insertions(+), 72 deletions(-)
qemu-img.c +199 -72
@@ -1446,6 +1446,29 @@ enum ImgConvertBlockStatus { BLK_BACKING_FILE, }; +#define CONVERT_MAX_REQS 4 + +typedef struct ImgConvertState ImgConvertState ; + +typedef struct ImgConvertReq { +ImgConvertState *s; +int64_t sector_num; +int64_t sector_offs; +int64_t next_sector; +int nb_sectors; +QEMUIOVector qiov; +uint8_t *buf; +enum ImgConvertBlockStatus status; +bool rd_completed; +}ImgConvertReq ; + +typedef struct ImgConvertQueueElt { +int64_t sector_num; +enum ImgConvertBlockStatus status; +int nb_sectors; +QSIMPLEQ_ENTRY (ImgConvertQueueElt) next; +}ImgConvertQueueElt ; + typedef struct ImgConvertState { BlockBackend **src; int64_t *src_sectors;
@@ -1453,6 +1476,8 @@ typedef struct ImgConvertState {int64_t src_cur_offset;int64_t total_sectors;int64_t allocated_sectors; +int64_t allocated_done; +int64_t wr_offs;enum ImgConvertBlockStatus status;int64_t sector_next_status;BlockBackend *target;
@@ -1462,11 +1487,15 @@ typedef struct ImgConvertState {int min_sparse;size_t cluster_sectors;size_t buf_sectors; +ImgConvertReq reqs[CONVERT_MAX_REQS ]; +int ret; + QSIMPLEQ_HEAD(, ImgConvertQueueElt) queue; } ImgConvertState;static void convert_select_part (ImgConvertState *s,int64_t sector_num) { -assert (sector_num >= s->src_cur_offset); + s->src_cur_offset =0 ; + s->src_cur =0 ;while (sector_num - s->src_cur_offset >= s->src_sectors[s->src_cur]) { s->src_cur_offset += s->src_sectors[s->src_cur]; s->src_cur++;
@@ -1542,16 +1571,89 @@ static int convert_iteration_sectors(ImgConvertState *s, int64_t sector_num)return n; } -static int convert_read (ImgConvertState *s,int64_t sector_num,int nb_sectors, -uint8_t *buf) +static void convert_aio_read (void *opaque,int ret); +static void convert_aio_write (void *opaque,int ret); + +static void convert_fill_write_queue (ImgConvertState *s) { +int i; +for (i =0 ; i <CONVERT_MAX_REQS ; i++) { +if (s->reqs[i].sector_num == s->wr_offs && s->reqs[i].rd_completed) { + s->reqs[i].rd_completed = false; + s->reqs[i].sector_offs =0 ; +convert_aio_write (&s->reqs[i],0 ); +break ; + } + } +} + +static void convert_fill_read_queue (ImgConvertState *s) +{ +ImgConvertQueueElt *elt; + +while ((elt =QSIMPLEQ_FIRST (&s->queue)) && !s->ret) { +ImgConvertReq *req =NULL ; +int i; +for (i =0 ; i <CONVERT_MAX_REQS ; i++) { +if (s->reqs[i].sector_num ==-1 ) { + req = &s->reqs[i]; +break ; + } + } +if (!req) { +return ; + } + +QSIMPLEQ_REMOVE_HEAD (&s->queue, next); + +printf ("convert_fill_read_queue req #%d @%p sector_num %ld nb_sectors %d status %d\n" , i, req, elt->sector_num, elt->nb_sectors, elt->status); + +if (elt->status ==BLK_DATA || (!s->min_sparse && elt->status ==BLK_ZERO )) + { + s->allocated_done += elt->nb_sectors; +qemu_progress_print (100.0 * s->allocated_done / s->allocated_sectors, +0 ); + } + req->sector_num = elt->sector_num; + req->nb_sectors = elt->nb_sectors; + req->status = elt->status; + req->sector_offs =0 ; +g_free (elt); + +if ((elt =QSIMPLEQ_FIRST (&s->queue))) { + req->next_sector = elt->sector_num; + }else { + req->next_sector = s->total_sectors; + } + +if (req->status ==BLK_DATA ) { + req->rd_completed = false; +convert_aio_read (req,0 ); + }else if (!s->min_sparse && s->status ==BLK_ZERO ) { +memset (req->buf,0 , req->nb_sectors *BDRV_SECTOR_SIZE ); + req->status =BLK_DATA ; + req->rd_completed = true; +convert_fill_write_queue (s); + } + } +} + +static void convert_aio_read (void *opaque,int ret) +{ +ImgConvertReq *req = opaque; +ImgConvertState *s = req->s; +int64_t sector_num = req->sector_num + req->sector_offs; +int nb_sectors = req->nb_sectors - req->sector_offs; +uint8_t *buf = req->buf + req->sector_offs *BDRV_SECTOR_SIZE ;int n; -int ret; + +printf ("convert_aio_read enter req %p sector_num %ld nb_sectors %d offs %ld ret %d\n" , req, sector_num, nb_sectors, req->sector_offs, ret);assert (nb_sectors <= s->buf_sectors); -while (nb_sectors >0 ) { +if (nb_sectors >0 ) {BlockBackend *blk;int64_t bs_sectors; +BlockAIOCB *acb;/* In the case of compression with multiple source files, we can get a * nb_sectors that spreads into the next part. So we must be able to
@@ -1561,30 +1663,49 @@ static int convert_read(ImgConvertState *s, int64_t sector_num, int nb_sectors, bs_sectors = s->src_sectors[s->src_cur]; n =MIN (nb_sectors, bs_sectors - (sector_num - s->src_cur_offset)); - ret =blk_pread (blk, +qemu_iovec_reset (&req->qiov); +qemu_iovec_add (&req->qiov, buf, n <<BDRV_SECTOR_BITS ); + req->sector_offs += n; + acb =blk_aio_preadv (blk, (sector_num - s->src_cur_offset) <<BDRV_SECTOR_BITS , - buf, n <<BDRV_SECTOR_BITS ); -if (ret <0 ) { -return ret; + &req->qiov,0 , convert_aio_read, req); +if (!acb) { + s->ret = -ENOMEM ; +return ; } - - sector_num += n; - nb_sectors -= n; - buf += n *BDRV_SECTOR_SIZE ; +return ; } -return 0 ; + req->rd_completed = true; +convert_fill_write_queue (s); } -static int convert_write (ImgConvertState *s,int64_t sector_num,int nb_sectors, -const uint8_t *buf) +static void convert_aio_write (void *opaque,int ret) { -int ret; +ImgConvertReq *req = opaque; +ImgConvertState *s = req->s; +int64_t sector_num; +int nb_sectors; +uint8_t *buf; + +again: + sector_num = req->sector_num + req->sector_offs; + nb_sectors = req->nb_sectors - req->sector_offs; + buf = req->buf + req->sector_offs *BDRV_SECTOR_SIZE ; +printf ("convert_aio_write enter req %p sector_num %ld nb_sectors %d offs %ld ret %d\n" , req, sector_num, nb_sectors, req->sector_offs, ret); -while (nb_sectors >0 ) { +if (ret) { +error_report ("error while writing sector %" PRId64 +": %s" , req->sector_num,strerror (-ret)); + s->ret = ret; +return ; + } + +if (nb_sectors >0 ) {int n = nb_sectors; +BlockAIOCB *acb; -switch (s->status) { +switch (req->status) {case BLK_BACKING_FILE :/* If we have a backing file, leave clusters unallocated that are * unallocated in the source image, so that the backing file is
@@ -1598,6 +1719,7 @@ static int convert_write(ImgConvertState *s, int64_t sector_num, int nb_sectors,* write if the buffer is completely zeroed and we're allowed to * keep the target sparse. */ if (s->compressed) { +assert (0 );//XXX TODO if (s->has_zero_init && s->min_sparse &&buffer_is_zero (buf, n *BDRV_SECTOR_SIZE )) {
@@ -1609,7 +1731,8 @@ static int convert_write(ImgConvertState *s, int64_t sector_num, int nb_sectors, sector_num <<BDRV_SECTOR_BITS , buf, n << BDRV_SECTOR_BITS);if (ret <0 ) { -return ret; + s->ret = ret; +return ; }break ; }
@@ -1620,10 +1743,15 @@ static int convert_write(ImgConvertState *s, int64_t sector_num, int nb_sectors,if (!s->min_sparse ||is_allocated_sectors_min (buf, n, &n, s->min_sparse)) { - ret =blk_pwrite (s->target, sector_num <<BDRV_SECTOR_BITS , - buf, n <<BDRV_SECTOR_BITS ,0 ); -if (ret <0 ) { -return ret; +qemu_iovec_reset (&req->qiov); +qemu_iovec_add (&req->qiov, buf, n <<BDRV_SECTOR_BITS ); + req->sector_offs += n; + acb =blk_aio_pwritev (s->target, + sector_num <<BDRV_SECTOR_BITS , + &req->qiov,0 , convert_aio_write, req); +if (!acb) { + s->ret = -ENOMEM ; +return ; }break ; }
@@ -1631,30 +1759,39 @@ static int convert_write(ImgConvertState *s, int64_t sector_num, int nb_sectors,case BLK_ZERO :if (s->has_zero_init) { -break ; +printf ("convert_aio_write n %d blocks are zero\n" , n); + req->sector_offs += n; + goto again; } +assert (0 );//XXX TODO ret =blk_pwrite_zeroes (s->target, sector_num <<BDRV_SECTOR_BITS , n <<BDRV_SECTOR_BITS ,0 );if (ret <0 ) { -return ret; + s->ret = ret; +return ; }break ; } - sector_num += n; - nb_sectors -= n; - buf += n *BDRV_SECTOR_SIZE ; +return ; } -return 0 ; +/* request is available for reading again */ +printf ("wr offs new = %ld\n" , req->next_sector); + req->sector_num =-1 ; + s->wr_offs = req->next_sector; + +convert_fill_read_queue (s); +convert_fill_write_queue (s); +return ; }static int convert_do_copy (ImgConvertState *s) { -uint8_t *buf =NULL ; -int64_t sector_num, allocated_done;int ret;int n; +int i; +int64_t sector_num;/* Check whether we have zero initialisation or can get it efficiently */ s->has_zero_init = s->min_sparse && !s->target_has_backing
@@ -1680,20 +1817,38 @@ static int convert_do_copy(ImgConvertState *s) } s->buf_sectors = s->cluster_sectors; } - buf =blk_blockalign (s->target, s->buf_sectors *BDRV_SECTOR_SIZE ); + +for (i =0 ; i <CONVERT_MAX_REQS ; i++) { +qemu_iovec_init (&s->reqs[i].qiov,1 ); + s->reqs[i].buf =blk_blockalign (s->target, s->buf_sectors *BDRV_SECTOR_SIZE ); + s->reqs[i].sector_num =-1 ; + s->reqs[i].s = s; + }/* Calculate allocated sectors for progress */ s->allocated_sectors =0 ; sector_num =0 ; +QSIMPLEQ_INIT (&s->queue);while (sector_num < s->total_sectors) { n =convert_iteration_sectors (s, sector_num);if (n <0 ) { ret = n; goto fail; } + +if (!s->min_sparse && s->status ==BLK_ZERO ) { + n =MIN (n, s->buf_sectors); + } + +printf ("convert_iteration_sectors %ld n %d status %d\n" , sector_num, n, s->status);if (s->status ==BLK_DATA || (!s->min_sparse && s->status ==BLK_ZERO )) { +ImgConvertQueueElt *elt =g_malloc (sizeof (ImgConvertQueueElt)); + elt->sector_num = sector_num; + elt->status = s->status; + elt->nb_sectors = n; s->allocated_sectors += n; +QSIMPLEQ_INSERT_TAIL (&s->queue, elt, next); } sector_num += n; }
@@ -1702,47 +1857,16 @@ static int convert_do_copy(ImgConvertState *s) s->src_cur =0 ; s->src_cur_offset =0 ; s->sector_next_status =0 ; + s->wr_offs =0 ; + s->allocated_done =0 ; - sector_num =0 ; - allocated_done =0 ; +convert_fill_read_queue (s); -while (sector_num < s->total_sectors) { - n =convert_iteration_sectors (s, sector_num); -if (n <0 ) { - ret = n; - goto fail; - } -if (s->status ==BLK_DATA || (!s->min_sparse && s->status ==BLK_ZERO )) - { - allocated_done += n; -qemu_progress_print (100.0 * allocated_done / s->allocated_sectors, -0 ); - } - -if (s->status ==BLK_DATA ) { - ret =convert_read (s, sector_num, n, buf); -if (ret <0 ) { -error_report ("error while reading sector %" PRId64 -": %s" , sector_num,strerror (-ret)); - goto fail; - } - }else if (!s->min_sparse && s->status ==BLK_ZERO ) { - n =MIN (n, s->buf_sectors); -memset (buf,0 , n *BDRV_SECTOR_SIZE ); - s->status =BLK_DATA ; - } - - ret =convert_write (s, sector_num, n, buf); -if (ret <0 ) { -error_report ("error while writing sector %" PRId64 -": %s" , sector_num,strerror (-ret)); - goto fail; - } - - sector_num += n; +while (s->wr_offs < s->total_sectors && !s->ret) { +main_loop_wait (false); } -if (s->compressed) { +if (s->compressed && !s->ret) {/* signal EOF to align */ ret =blk_pwrite_compressed (s->target,0 ,NULL ,0 );if (ret <0 ) {
@@ -1750,9 +1874,12 @@ static int convert_do_copy(ImgConvertState *s) } } - ret =0 ; + ret = s->ret; fail: -qemu_vfree (buf); +for (i =0 ; i <CONVERT_MAX_REQS ; i++) { +qemu_iovec_destroy (&s->reqs[i].qiov); +qemu_vfree (s->reqs[i].buf); + }return ret; }
Signature · 1 line
1.9.1
Your series seems to have some coding style problems. See output below for more information:
Hi,
Your series seems to have some coding style problems. See output below for more information:
Type: series
Subject: [Qemu-devel] [RFC][PATCH] qemu-img: make convert async
Message-id: [email protected]
=== TEST SCRIPT BEGIN ===
#!/bin/bash
BASE=base
n=1
total=$(git log --oneline $BASE.. | wc -l)
failed=0
# Useful git options
git config --local diff.renamelimit 0
git config --local diff.renames True
commits="$(git log --format=%H --reverse $BASE..)" for c in $commits; do
echo "Checking PATCH $n/$total: $(git log -n 1 --format=%s $c)..."
if ! git show $c --format=email | ./scripts/checkpatch.pl --mailback -; then
failed=1
echo
fi
n=$((n+1))done
exit $failed
=== TEST SCRIPT END ===
Updating 3c8cf5a9c21ff8782164d1def7f44bd888713384
Switched to a new branch 'test'
3a167eb qemu-img: make convert async
=== OUTPUT BEGIN ===
Checking PATCH 1/1: qemu-img: make convert async...
ERROR: suspect code indent for conditional statements (4, 9) #118: FILE: qemu-img.c:1580:
+ for (i = 0; i < CONVERT_MAX_REQS; i++) {
+ if (s->reqs[i].sector_num == s->wr_offs && s->reqs[i].rd_completed) {
ERROR: line over 90 characters
#147: FILE: qemu-img.c:1609:
+ printf("convert_fill_read_queue req #%d @%p sector_num %ld nb_sectors %d status %d\n", i, req, elt->sector_num, elt->nb_sectors, elt->status);
WARNING: line over 80 characters
#149: FILE: qemu-img.c:1611:
+ if (elt->status == BLK_DATA || (!s->min_sparse && elt->status == BLK_ZERO))
WARNING: line over 80 characters
#152: FILE: qemu-img.c:1614:
+ qemu_progress_print(100.0 * s->allocated_done / s->allocated_sectors,
ERROR: do not use assignment in if condition
#161: FILE: qemu-img.c:1623:
+ if ((elt = QSIMPLEQ_FIRST(&s->queue))) {
ERROR: line over 90 characters
#189: FILE: qemu-img.c:1650:
+ printf("convert_aio_read enter req %p sector_num %ld nb_sectors %d offs %ld ret %d\n", req, sector_num, nb_sectors, req->sector_offs, ret);
ERROR: line over 90 characters
#245: FILE: qemu-img.c:1695:
+ printf("convert_aio_write enter req %p sector_num %ld nb_sectors %d offs %ld ret %d\n", req, sector_num, nb_sectors, req->sector_offs, ret);
ERROR: do not use C99 // comments
#268: FILE: qemu-img.c:1722:
+ assert(0); //XXX TODO
ERROR: do not use C99 // comments
#311: FILE: qemu-img.c:1766:
+ assert(0); //XXX TODO
WARNING: line over 80 characters
#358: FILE: qemu-img.c:1823:
+ s->reqs[i].buf = blk_blockalign(s->target, s->buf_sectors * BDRV_SECTOR_SIZE);
ERROR: line over 90 characters
#378: FILE: qemu-img.c:1843:
+ printf("convert_iteration_sectors %ld n %d status %d\n", sector_num, n, s->status);
ERROR: suspect code indent for conditional statements (4, 9) #452: FILE: qemu-img.c:1879:
+ for (i = 0; i < CONVERT_MAX_REQS; i++) {
+ qemu_iovec_destroy(&s->reqs[i].qiov);
total: 9 errors, 3 warnings, 397 lines checked
Your patch has style problems, please review. If any of these errors are false positives report them to the maintainer, see CHECKPATCH in MAINTAINERS.
=== OUTPUT END ===
Test command exited with code: 1
---
Email generated automatically by Patchew [http://patchew.org/].
Please send your feedback to [email protected]
Your series failed automatic build test. Please find the testing commands and their output below. If you have docker installed, you can probably reproduce it locally.
Hi,
Your series failed automatic build test. Please find the testing commands and their output below. If you have docker installed, you can probably reproduce it locally.
Type: series
Subject: [Qemu-devel] [RFC][PATCH] qemu-img: make convert async
Message-id: [email protected]
=== TEST SCRIPT BEGIN ===
#!/bin/bash
set -e
git submodule update --init dtc
# Let docker tests dump environment info
export SHOW_ENV=1
export J=16
make docker-test-quick@centos6
make docker-test-mingw@fedora
make docker-test-build@min-glib
=== TEST SCRIPT END ===
Updating 3c8cf5a9c21ff8782164d1def7f44bd888713384
Switched to a new branch 'test'
3a167eb qemu-img: make convert async
=== OUTPUT BEGIN ===
Submodule 'dtc' (git://git.qemu-project.org/dtc.git) registered for path 'dtc' Cloning into 'dtc'...
Submodule path 'dtc': checked out '65cc4d2748a2c2e6f27f1cf39e07a5dbabd80ebf'
BUILD centos6
make[1]: Entering directory `/var/tmp/patchew-tester-tmp-oelmqzb0/src'
ARCHIVE qemu.tgz
ARCHIVE dtc.tgz
COPY RUNNER
RUN test-quick in qemu:centos6
Packages installed:
SDL-devel-1.2.14-7.el6_7.1.x86_64
ccache-3.1.6-2.el6.x86_64
epel-release-6-8.noarch
gcc-4.4.7-17.el6.x86_64
git-1.7.1-4.el6_7.1.x86_64
glib2-devel-2.28.8-5.el6.x86_64
libfdt-devel-1.4.0-1.el6.x86_64
make-3.81-23.el6.x86_64
package g++ is not installed
pixman-devel-0.32.8-1.el6.x86_64
tar-1.23-15.el6_8.x86_64
zlib-devel-1.2.3-29.el6.x86_64
Environment variables:
PACKAGES=libfdt-devel ccache tar git make gcc g++ zlib-devel glib2-devel SDL-devel pixman-devel epel-release
HOSTNAME=66ea1b2e6c36
TERM=xterm
MAKEFLAGS= -j16
HISTSIZE=1000
J=16
USER=root
CCACHE_DIR=/var/tmp/ccache
EXTRA_CONFIGURE_OPTS=
V=
SHOW_ENV=1
MAIL=/var/spool/mail/root PATH=/usr/lib/ccache:/usr/lib64/ccache:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin PWD=/
LANG=en_US.UTF-8
TARGET_LIST=
HISTCONTROL=ignoredups
SHLVL=1
HOME=/root
TEST_DIR=/tmp/qemu-test
LOGNAME=root
LESSOPEN=||/usr/bin/lesspipe.sh %s
FEATURES= dtc
DEBUG=
G_BROKEN_FILENAMES=1
CCACHE_HASHDIR=
_=/usr/bin/env
Configure options:
--enable-werror --target-list=x86_64-softmmu,aarch64-softmmu --prefix=/var/tmp/qemu-build/install No C++ compiler available; disabling C++ specific optional code
Install prefix /var/tmp/qemu-build/install BIOS directory /var/tmp/qemu-build/install/share/qemu binary directory /var/tmp/qemu-build/install/bin
library directory /var/tmp/qemu-build/install/lib module directory /var/tmp/qemu-build/install/lib/qemu libexec directory /var/tmp/qemu-build/install/libexec include directory /var/tmp/qemu-build/install/include config directory /var/tmp/qemu-build/install/etc
local state directory /var/tmp/qemu-build/install/var
Manual directory /var/tmp/qemu-build/install/share/man ELF interp prefix /usr/gnemul/qemu-%M
Source path /tmp/qemu-test/src C compiler cc Host C compiler cc C++ compiler Objective-C compiler cc ARFLAGS rv CFLAGS -O2 -U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=2 -g QEMU_CFLAGS -I/usr/include/pixman-1 -pthread -I/usr/include/glib-2.0 -I/usr/lib64/glib-2.0/include -fPIE -DPIE -m64 -mcx16 -D_GNU_SOURCE -D_FILE_OFFSET_BITS=64 -D_LARGEFILE_SOURCE -Wstrict-prototypes -Wredundant-decls -Wall -Wundef -Wwrite-strings -Wmissing-prototypes -fno-strict-aliasing -fno-common -fwrapv -Wendif-labels -Wmissing-include-dirs -Wempty-body -Wnested-externs -Wformat-security -Wformat-y2k -Winit-self -Wignored-qualifiers -Wold-style-declaration -Wold-style-definition -Wtype-limits -fstack-protector-all LDFLAGS -Wl,--warn-common -Wl,-z,relro -Wl,-z,now -pie -m64 -g make make install install python python -B smbd /usr/sbin/smbd module support no host CPU x86_64 host big endian no target list x86_64-softmmu aarch64-softmmu tcg debug enabled no gprof enabled no sparse enabled no strip binaries yes profiler no static build no pixman system SDL support yes (1.2.14) GTK support no GTK GL support no VTE support no TLS priority NORMAL GNUTLS support no GNUTLS rnd no libgcrypt no libgcrypt kdf no nettle no nettle kdf no libtasn1 no curses support no virgl support no curl support no mingw32 support no Audio drivers oss Block whitelist (rw) Block whitelist (ro) VirtFS support no VNC support yes VNC SASL support no VNC JPEG support no VNC PNG support no xen support no brlapi support no bluez support no Documentation no PIE yes vde support no netmap support no Linux AIO support no ATTR/XATTR support yes Install blobs yes KVM support yes HAX support no RDMA support no TCG interpreter no fdt support yes preadv support yes fdatasync yes madvise yes posix_madvise yes
libcap-ng support no
vhost-net support yes
vhost-scsi support yes
vhost-vsock support yes
Trace backends log spice support no rbd support no xfsctl support no smartcard support no libusb no usb net redir no OpenGL support no OpenGL dmabufs no libiscsi support no libnfs support no
build guest agent yes
QGA VSS support no
QGA w32 disk info no
QGA MSI support no
seccomp support no
coroutine backend ucontext
coroutine pool yes
debug stack usage no
GlusterFS support no
Archipelago support no
gcov gcov gcov enabled no TPM support yes libssh2 support no TPM passthrough yes QOM debugging yes lzo support no snappy support no bzip2 support no
NUMA host support no
tcmalloc support no
jemalloc support no
avx2 optimization no
replication support yes
GEN x86_64-softmmu/config-devices.mak.tmp GEN aarch64-softmmu/config-devices.mak.tmp GEN config-host.h GEN qemu-options.def GEN qapi-types.h GEN qmp-commands.h GEN qapi-visit.h GEN qapi-event.h GEN qmp-introspect.h GEN module_block.h GEN tests/test-qapi-types.h GEN tests/test-qapi-visit.h GEN tests/test-qmp-commands.h GEN tests/test-qapi-event.h GEN tests/test-qmp-introspect.h GEN trace/generated-tracers.h GEN x86_64-softmmu/config-devices.mak GEN aarch64-softmmu/config-devices.mak GEN trace/generated-tcg-tracers.h GEN trace/generated-helpers-wrappers.h GEN trace/generated-helpers.h GEN config-all-devices.mak CC tests/qemu-iotests/socket_scm_helper.o GEN qga/qapi-generated/qga-qmp-commands.h GEN qga/qapi-generated/qga-qapi-types.h GEN qga/qapi-generated/qga-qapi-visit.h GEN qga/qapi-generated/qga-qapi-types.c GEN qmp-introspect.c GEN qga/qapi-generated/qga-qapi-visit.c GEN qga/qapi-generated/qga-qmp-marshal.c GEN qapi-types.c GEN qapi-visit.c GEN qapi-event.c CC qapi/qapi-dealloc-visitor.o CC qapi/qapi-visit-core.o CC qapi/qobject-input-visitor.o CC qapi/qobject-output-visitor.o CC qapi/qmp-registry.o CC qapi/qmp-dispatch.o CC qapi/string-input-visitor.o CC qapi/string-output-visitor.o CC qapi/opts-visitor.o CC qapi/qapi-clone-visitor.o CC qapi/qmp-event.o CC qapi/qapi-util.o CC qobject/qnull.o CC qobject/qint.o CC qobject/qstring.o CC qobject/qdict.o CC qobject/qlist.o CC qobject/qfloat.o CC qobject/qbool.o CC qobject/qjson.o CC qobject/qobject.o CC qobject/json-lexer.o CC qobject/json-streamer.o CC qobject/json-parser.o GEN trace/generated-tracers.c CC trace/qmp.o CC trace/control.o CC util/osdep.o CC util/cutils.o CC util/unicode.o CC util/qemu-timer-common.o CC util/bufferiszero.o CC util/lockcnt.o CC util/compatfd.o CC util/event_notifier-posix.o CC util/mmap-alloc.o CC util/oslib-posix.o CC util/qemu-openpty.o CC util/qemu-thread-posix.o CC util/memfd.o CC util/envlist.o CC util/path.o CC util/module.o CC util/bitmap.o CC util/bitops.o CC util/hbitmap.o CC util/fifo8.o CC util/acl.o CC util/error.o CC util/qemu-error.o CC util/id.o CC util/iov.o CC util/qemu-config.o CC util/qemu-sockets.o CC util/uri.o CC util/notify.o CC util/qemu-option.o CC util/qemu-progress.o CC util/hexdump.o CC util/crc32c.o CC util/uuid.o CC util/throttle.o CC util/getauxval.o CC util/readline.o CC util/rcu.o CC util/qemu-coroutine.o CC util/qemu-coroutine-lock.o CC util/qemu-coroutine-io.o CC util/qemu-coroutine-sleep.o CC util/coroutine-ucontext.o CC util/buffer.o CC util/timed-average.o CC util/base64.o CC util/log.o CC util/qdist.o CC util/qht.o CC util/range.o CC crypto/pbkdf-stub.o CC stubs/arch-query-cpu-def.o CC stubs/arch-query-cpu-model-expansion.o CC stubs/arch-query-cpu-model-comparison.o CC stubs/arch-query-cpu-model-baseline.o CC stubs/bdrv-next-monitor-owned.o CC stubs/blk-commit-all.o CC stubs/blockdev-close-all-bdrv-states.o CC stubs/clock-warp.o CC stubs/cpu-get-clock.o CC stubs/cpu-get-icount.o CC stubs/dump.o CC stubs/error-printf.o CC stubs/fdset.o CC stubs/get-vm-name.o CC stubs/gdbstub.o CC stubs/iothread.o CC stubs/iothread-lock.o CC stubs/is-daemonized.o CC stubs/machine-init-done.o CC stubs/migr-blocker.o CC stubs/monitor.o CC stubs/notify-event.o CC stubs/qtest.o CC stubs/runstate-check.o CC stubs/set-fd-handler.o CC stubs/replay.o CC stubs/trace-control.o CC stubs/slirp.o CC stubs/sysbus.o CC stubs/uuid.o CC stubs/vmstate.o CC stubs/vm-stop.o CC stubs/qmp_pc_dimm_device_list.o CC stubs/target-monitor-defs.o CC stubs/target-get-monitor-def.o CC stubs/pc_madt_cpu_entry.o CC contrib/ivshmem-client/ivshmem-client.o CC contrib/ivshmem-client/main.o CC contrib/ivshmem-server/ivshmem-server.o CC contrib/ivshmem-server/main.o CC qemu-nbd.o CC async.o CC thread-pool.o CC block.o CC blockjob.o CC main-loop.o CC iohandler.o CC qemu-timer.o CC aio-posix.o CC qemu-io-cmds.o CC replication.o CC block/raw-format.o CC block/qcow.o CC block/vdi.o CC block/vmdk.o CC block/cloop.o CC block/bochs.o CC block/vpc.o CC block/vvfat.o CC block/dmg.o CC block/qcow2.o CC block/qcow2-refcount.o CC block/qcow2-cluster.o CC block/qcow2-snapshot.o CC block/qcow2-cache.o CC block/qed.o CC block/qed-gencb.o CC block/qed-l2-cache.o CC block/qed-table.o CC block/qed-cluster.o CC block/qed-check.o CC block/vhdx.o CC block/vhdx-endian.o CC block/vhdx-log.o CC block/quorum.o CC block/parallels.o CC block/blkdebug.o CC block/blkverify.o CC block/blkreplay.o CC block/block-backend.o CC block/snapshot.o CC block/qapi.o CC block/file-posix.o CC block/null.o CC block/mirror.o CC block/commit.o CC block/io.o CC block/throttle-groups.o CC block/nbd.o CC block/nbd-client.o CC block/sheepdog.o CC block/accounting.o CC block/dirty-bitmap.o CC block/write-threshold.o CC block/backup.o CC block/replication.o CC block/crypto.o CC nbd/server.o CC nbd/client.o CC nbd/common.o CC crypto/init.o CC crypto/hash.o CC crypto/hash-glib.o CC crypto/hmac.o CC crypto/hmac-glib.o CC crypto/aes.o CC crypto/desrfb.o CC crypto/cipher.o CC crypto/tlscreds.o CC crypto/tlscredsanon.o CC crypto/tlscredsx509.o CC crypto/tlssession.o CC crypto/secret.o CC crypto/random-platform.o CC crypto/pbkdf.o CC crypto/ivgen-essiv.o CC crypto/ivgen.o CC crypto/ivgen-plain.o CC crypto/ivgen-plain64.o CC crypto/afsplit.o CC crypto/xts.o CC crypto/block.o CC crypto/block-qcow.o CC crypto/block-luks.o CC io/channel.o CC io/channel-buffer.o CC io/channel-file.o CC io/channel-socket.o CC io/channel-command.o CC io/channel-tls.o CC io/channel-watch.o CC io/channel-websock.o CC io/channel-util.o CC io/dns-resolver.o CC qom/object.o CC io/task.o CC qom/container.o CC qom/qom-qobject.o CC qom/object_interfaces.o GEN qemu-img-cmds.h CC qemu-io.o CC qemu-bridge-helper.o CC blockdev.o CC blockdev-nbd.o CC iothread.o CC qdev-monitor.o CC os-posix.o CC device-hotplug.o CC page_cache.o CC accel.o CC bt-host.o CC dma-helpers.o CC bt-vhci.o CC vl.o CC tpm.o GEN qmp-marshal.c CC device_tree.o CC qmp.o CC hmp.o CC audio/noaudio.o CC cpus-common.o CC audio/audio.o CC audio/wavaudio.o CC audio/mixeng.o CC audio/sdlaudio.o CC audio/ossaudio.o CC backends/rng.o CC audio/wavcapture.o CC backends/rng-egd.o CC backends/testdev.o CC backends/rng-random.o CC backends/msmouse.o CC backends/tpm.o CC backends/hostmem.o CC backends/hostmem-ram.o CC backends/cryptodev.o CC backends/hostmem-file.o CC backends/cryptodev-builtin.o CC block/stream.o CC disas/arm.o CC disas/i386.o CC fsdev/qemu-fsdev-dummy.o CC fsdev/qemu-fsdev-opts.o CC hw/acpi/core.o CC hw/acpi/piix4.o CC hw/acpi/pcihp.o CC hw/acpi/ich9.o CC hw/acpi/memory_hotplug.o CC hw/acpi/cpu_hotplug.o CC hw/acpi/tco.o CC hw/acpi/cpu.o CC hw/acpi/nvdimm.o CC hw/acpi/acpi_interface.o CC hw/acpi/bios-linker-loader.o CC hw/acpi/ipmi.o CC hw/acpi/aml-build.o CC hw/acpi/acpi-stub.o CC hw/acpi/ipmi-stub.o CC hw/audio/sb16.o CC hw/audio/es1370.o CC hw/audio/ac97.o CC hw/audio/fmopl.o CC hw/audio/adlib.o CC hw/audio/gusemu_mixer.o CC hw/audio/gus.o CC hw/audio/gusemu_hal.o CC hw/audio/intel-hda.o CC hw/audio/cs4231a.o CC hw/audio/hda-codec.o CC hw/audio/pcspk.o CC hw/audio/wm8750.o CC hw/audio/pl041.o CC hw/audio/lm4549.o CC hw/audio/marvell_88w8618.o CC hw/block/block.o CC hw/block/cdrom.o CC hw/block/hd-geometry.o CC hw/block/fdc.o CC hw/block/pflash_cfi01.o CC hw/block/nand.o CC hw/block/m25p80.o CC hw/block/pflash_cfi02.o CC hw/block/ecc.o CC hw/block/onenand.o CC hw/block/nvme.o CC hw/bt/core.o CC hw/bt/l2cap.o CC hw/bt/hid.o CC hw/bt/hci-csr.o CC hw/bt/sdp.o CC hw/bt/hci.o CC hw/char/parallel.o CC hw/char/ipoctal232.o CC hw/char/pl011.o CC hw/char/serial.o CC hw/char/serial-isa.o CC hw/char/serial-pci.o CC hw/char/debugcon.o CC hw/char/imx_serial.o CC hw/char/virtio-console.o CC hw/char/cadence_uart.o CC hw/core/qdev.o CC hw/core/qdev-properties.o CC hw/core/bus.o CC hw/core/reset.o CC hw/core/fw-path-provider.o CC hw/core/irq.o CC hw/core/hotplug.o CC hw/core/ptimer.o CC hw/core/sysbus.o CC hw/core/loader.o CC hw/core/machine.o CC hw/core/qdev-properties-system.o CC hw/core/register.o CC hw/core/or-irq.o CC hw/core/platform-bus.o CC hw/display/ads7846.o CC hw/display/cirrus_vga.o CC hw/display/ssd0323.o CC hw/display/pl110.o CC hw/display/ssd0303.o CC hw/display/vga-pci.o CC hw/display/vmware_vga.o CC hw/display/exynos4210_fimd.o CC hw/display/vga-isa.o CC hw/display/blizzard.o CC hw/display/framebuffer.o CC hw/display/tc6393xb.o CC hw/dma/pl080.o CC hw/dma/i8257.o CC hw/dma/pl330.o CC hw/dma/xlnx-zynq-devcfg.o CC hw/gpio/max7310.o CC hw/gpio/pl061.o CC hw/gpio/zaurus.o CC hw/gpio/gpio_key.o CC hw/i2c/core.o CC hw/i2c/smbus.o CC hw/i2c/smbus_eeprom.o CC hw/i2c/i2c-ddc.o CC hw/i2c/versatile_i2c.o CC hw/i2c/pm_smbus.o CC hw/i2c/smbus_ich9.o CC hw/i2c/bitbang_i2c.o CC hw/i2c/exynos4210_i2c.o CC hw/i2c/imx_i2c.o CC hw/i2c/aspeed_i2c.o CC hw/ide/qdev.o CC hw/ide/pci.o CC hw/ide/isa.o CC hw/ide/core.o CC hw/ide/atapi.o CC hw/ide/piix.o CC hw/ide/ahci.o CC hw/ide/microdrive.o CC hw/ide/ich.o CC hw/input/lm832x.o CC hw/input/hid.o CC hw/input/pckbd.o CC hw/input/pl050.o CC hw/input/stellaris_input.o CC hw/input/ps2.o CC hw/input/vmmouse.o CC hw/input/tsc2005.o CC hw/input/virtio-input.o CC hw/input/virtio-input-hid.o CC hw/input/virtio-input-host.o CC hw/intc/i8259.o CC hw/intc/i8259_common.o CC hw/intc/pl190.o CC hw/intc/imx_avic.o CC hw/intc/realview_gic.o CC hw/intc/ioapic_common.o CC hw/intc/arm_gic_common.o CC hw/intc/arm_gicv2m.o CC hw/intc/arm_gic.o CC hw/intc/arm_gicv3_common.o CC hw/intc/arm_gicv3.o CC hw/intc/arm_gicv3_redist.o CC hw/intc/arm_gicv3_dist.o CC hw/intc/arm_gicv3_its_common.o CC hw/intc/intc.o CC hw/ipack/tpci200.o CC hw/ipack/ipack.o CC hw/ipmi/ipmi.o CC hw/ipmi/isa_ipmi_kcs.o CC hw/ipmi/ipmi_bmc_extern.o CC hw/ipmi/ipmi_bmc_sim.o CC hw/ipmi/isa_ipmi_bt.o CC hw/isa/isa-bus.o CC hw/isa/apm.o CC hw/mem/pc-dimm.o CC hw/mem/nvdimm.o CC hw/misc/applesmc.o CC hw/misc/max111x.o CC hw/misc/tmp105.o CC hw/misc/debugexit.o CC hw/misc/sga.o CC hw/misc/pc-testdev.o CC hw/misc/pci-testdev.o CC hw/misc/arm_integrator_debug.o CC hw/misc/arm_l2x0.o CC hw/misc/a9scu.o CC hw/misc/arm11scu.o CC hw/net/ne2000.o CC hw/net/eepro100.o CC hw/net/pcnet-pci.o CC hw/net/pcnet.o CC hw/net/e1000.o CC hw/net/e1000x_common.o CC hw/net/net_tx_pkt.o CC hw/net/net_rx_pkt.o CC hw/net/e1000e.o CC hw/net/e1000e_core.o CC hw/net/rtl8139.o CC hw/net/vmxnet3.o CC hw/net/smc91c111.o CC hw/net/lan9118.o CC hw/net/ne2000-isa.o CC hw/net/xgmac.o CC hw/net/allwinner_emac.o CC hw/net/imx_fec.o CC hw/net/cadence_gem.o CC hw/net/rocker/rocker.o CC hw/net/stellaris_enet.o CC hw/net/rocker/rocker_fp.o CC hw/net/rocker/rocker_desc.o CC hw/net/rocker/rocker_of_dpa.o CC hw/net/rocker/rocker_world.o CC hw/nvram/eeprom93xx.o CC hw/nvram/fw_cfg.o CC hw/nvram/chrp_nvram.o CC hw/pci-bridge/pci_expander_bridge.o CC hw/pci-bridge/pci_bridge_dev.o CC hw/pci-bridge/pcie_root_port.o CC hw/pci-bridge/xio3130_downstream.o CC hw/pci-bridge/gen_pcie_root_port.o CC hw/pci-bridge/xio3130_upstream.o CC hw/pci-bridge/ioh3420.o CC hw/pci-bridge/i82801b11.o CC hw/pci-host/pam.o CC hw/pci-host/versatile.o CC hw/pci-host/piix.o CC hw/pci-host/q35.o CC hw/pci-host/gpex.o CC hw/pci/pci.o CC hw/pci/pci_bridge.o CC hw/pci/msix.o CC hw/pci/msi.o CC hw/pci/shpc.o CC hw/pci/pci_host.o CC hw/pci/slotid_cap.o CC hw/pci/pcie_host.o CC hw/pci/pcie.o CC hw/pci/pcie_aer.o CC hw/pci/pcie_port.o CC hw/pci/pci-stub.o CC hw/pcmcia/pcmcia.o CC hw/scsi/scsi-disk.o CC hw/scsi/lsi53c895a.o CC hw/scsi/scsi-bus.o CC hw/scsi/scsi-generic.o CC hw/scsi/mptconfig.o CC hw/scsi/mptendian.o CC hw/scsi/megasas.o CC hw/scsi/mptsas.o CC hw/scsi/vmw_pvscsi.o CC hw/scsi/esp.o CC hw/scsi/esp-pci.o CC hw/sd/pl181.o CC hw/sd/ssi-sd.o CC hw/sd/sd.o CC hw/sd/core.o CC hw/sd/sdhci.o CC hw/smbios/smbios.o CC hw/smbios/smbios_type_38.o CC hw/smbios/smbios-stub.o CC hw/smbios/smbios_type_38-stub.o CC hw/ssi/pl022.o CC hw/ssi/ssi.o CC hw/ssi/xilinx_spips.o CC hw/ssi/stm32f2xx_spi.o CC hw/ssi/aspeed_smc.o CC hw/timer/arm_timer.o CC hw/timer/arm_mptimer.o CC hw/timer/ds1338.o CC hw/timer/a9gtimer.o CC hw/timer/cadence_ttc.o CC hw/timer/hpet.o CC hw/timer/i8254_common.o CC hw/timer/imx_epit.o CC hw/timer/i8254.o CC hw/timer/pl031.o CC hw/timer/twl92230.o CC hw/timer/imx_gpt.o CC hw/timer/stm32f2xx_timer.o CC hw/timer/aspeed_timer.o CC hw/tpm/tpm_util.o CC hw/tpm/tpm_tis.o CC hw/tpm/tpm_passthrough.o CC hw/usb/core.o CC hw/usb/combined-packet.o CC hw/usb/bus.o CC hw/usb/libhw.o CC hw/usb/desc.o CC hw/usb/hcd-uhci.o CC hw/usb/desc-msos.o CC hw/usb/hcd-ohci.o CC hw/usb/hcd-ehci.o CC hw/usb/hcd-ehci-sysbus.o CC hw/usb/hcd-xhci.o CC hw/usb/hcd-ehci-pci.o CC hw/usb/hcd-musb.o CC hw/usb/dev-hub.o CC hw/usb/dev-hid.o CC hw/usb/dev-wacom.o CC hw/usb/dev-storage.o CC hw/usb/dev-uas.o CC hw/usb/dev-audio.o CC hw/usb/dev-network.o CC hw/usb/dev-serial.o CC hw/usb/dev-bluetooth.o CC hw/usb/dev-smartcard-reader.o CC hw/usb/dev-mtp.o CC hw/usb/host-stub.o CC hw/virtio/virtio-rng.o CC hw/virtio/virtio-pci.o CC hw/virtio/virtio-bus.o CC hw/virtio/virtio-mmio.o CC hw/virtio/vhost-stub.o CC hw/watchdog/watchdog.o CC hw/watchdog/wdt_ib700.o CC hw/watchdog/wdt_i6300esb.o CC migration/socket.o CC migration/fd.o CC migration/migration.o CC migration/exec.o CC migration/tls.o CC migration/colo-comm.o CC migration/colo.o CC migration/colo-failover.o CC migration/vmstate.o CC migration/qemu-file.o CC migration/qemu-file-channel.o CC migration/xbzrle.o CC migration/postcopy-ram.o CC migration/qjson.o CC migration/block.o CC net/net.o CC net/queue.o CC net/checksum.o CC net/util.o CC net/hub.o CC net/socket.o CC net/tap.o CC net/dump.o CC net/eth.o CC net/l2tpv3.o CC net/vhost-user.o CC net/tap-linux.o CC net/slirp.o CC net/filter.o CC net/filter-mirror.o CC net/filter-buffer.o CC net/filter-replay.o CC net/colo-compare.o CC net/colo.o CC qom/cpu.o CC net/filter-rewriter.o CC replay/replay.o CC replay/replay-internal.o CC replay/replay-events.o CC replay/replay-time.o CC replay/replay-input.o CC replay/replay-char.o CC replay/replay-net.o CC replay/replay-snapshot.o CC slirp/cksum.o CC slirp/if.o /tmp/qemu-test/src/replay/replay-internal.c: In function ‘replay_put_array’: /tmp/qemu-test/src/replay/replay-internal.c:65: warning: ignoring return value of ‘fwrite’, declared with attribute warn_unused_result CC slirp/ip_icmp.o CC slirp/ip6_icmp.o CC slirp/ip6_input.o CC slirp/ip6_output.o CC slirp/ip_input.o CC slirp/ip_output.o CC slirp/dnssearch.o CC slirp/dhcpv6.o CC slirp/slirp.o CC slirp/mbuf.o CC slirp/misc.o CC slirp/sbuf.o CC slirp/socket.o CC slirp/tcp_input.o CC slirp/tcp_output.o CC slirp/tcp_subr.o CC slirp/tcp_timer.o CC slirp/udp.o CC slirp/udp6.o CC slirp/bootp.o CC slirp/tftp.o CC slirp/arp_table.o CC slirp/ndp_table.o
/tmp/qemu-test/src/slirp/tcp_input.c: In function ‘tcp_input’:/tmp/qemu-test/src/slirp/tcp_input.c:219: warning: ‘save_ip.ip_p’ may be used uninitialized in this function /tmp/qemu-test/src/slirp/tcp_input.c:219: warning: ‘save_ip.ip_len’ may be used uninitialized in this function /tmp/qemu-test/src/slirp/tcp_input.c:219: warning: ‘save_ip.ip_tos’ may be used uninitialized in this function /tmp/qemu-test/src/slirp/tcp_input.c:219: warning: ‘save_ip.ip_id’ may be used uninitialized in this function /tmp/qemu-test/src/slirp/tcp_input.c:219: warning: ‘save_ip.ip_off’ may be used uninitialized in this function /tmp/qemu-test/src/slirp/tcp_input.c:219: warning: ‘save_ip.ip_ttl’ may be used uninitialized in this function /tmp/qemu-test/src/slirp/tcp_input.c:219: warning: ‘save_ip.ip_sum’ may be used uninitialized in this function /tmp/qemu-test/src/slirp/tcp_input.c:219: warning: ‘save_ip.ip_src.s_addr’ may be used uninitialized in this function /tmp/qemu-test/src/slirp/tcp_input.c:219: warning: ‘save_ip.ip_dst.s_addr’ may be used uninitialized in this function /tmp/qemu-test/src/slirp/tcp_input.c:220: warning: ‘save_ip6.ip_nh’ may be used uninitialized in this function
CC ui/keymaps.o CC ui/console.o CC ui/cursor.o CC ui/qemu-pixman.o CC ui/input.o CC ui/input-keymap.o CC ui/input-legacy.o CC ui/input-linux.o CC ui/sdl.o CC ui/x_keymap.o CC ui/sdl_zoom.o CC ui/vnc.o CC ui/vnc-enc-zlib.o CC ui/vnc-enc-hextile.o CC ui/vnc-enc-tight.o CC ui/vnc-palette.o CC ui/vnc-enc-zrle.o CC ui/vnc-auth-vencrypt.o CC ui/vnc-jobs.o CC ui/vnc-ws.o CC chardev/char.o CC chardev/char-fd.o CC chardev/char-file.o CC chardev/char-mux.o CC chardev/char-io.o CC chardev/char-null.o CC chardev/char-parallel.o CC chardev/char-pipe.o CC chardev/char-pty.o CC chardev/char-ringbuf.o CC chardev/char-serial.o CC chardev/char-socket.o CC chardev/char-stdio.o CC chardev/char-udp.o LINK tests/qemu-iotests/socket_scm_helper CC qga/commands.o CC qga/guest-agent-command-state.o CC qga/main.o CC qga/commands-posix.o CC qga/channel-posix.o CC qga/qapi-generated/qga-qapi-visit.o CC qga/qapi-generated/qga-qapi-types.o CC qga/qapi-generated/qga-qmp-marshal.o CC qapi-types.o CC qmp-introspect.o CC qapi-visit.o CC qapi-event.o AR libqemustub.a CC qmp-marshal.o CC qemu-img.o CC trace/generated-tracers.o AR libqemuutil.a AS optionrom/multiboot.o AS optionrom/kvmvapic.o CC optionrom/linuxboot_dma.o AS optionrom/linuxboot.o
BUILD optionrom/linuxboot_dma.img BUILD optionrom/linuxboot_dma.raw BUILD optionrom/multiboot.img BUILD optionrom/kvmvapic.img BUILD optionrom/linuxboot.img SIGN optionrom/linuxboot_dma.bin BUILD optionrom/multiboot.raw BUILD optionrom/linuxboot.raw BUILD optionrom/kvmvapic.raw SIGN optionrom/multiboot.bin SIGN optionrom/linuxboot.bin SIGN optionrom/kvmvapic.bin LINK qemu-ga LINK ivshmem-client LINK ivshmem-server LINK qemu-nbd LINK qemu-io LINK qemu-bridge-helper
/tmp/qemu-test/src/qemu-img.c:1493: error: redefinition of typedef ‘ImgConvertState’ /tmp/qemu-test/src/qemu-img.c:1451: note: previous declaration of ‘ImgConvertState’ was here
make: *** [qemu-img.o] Error 1
make: *** Waiting for unfinished jobs....
GEN x86_64-softmmu/hmp-commands.h GEN x86_64-softmmu/hmp-commands-info.h GEN x86_64-softmmu/config-target.h GEN aarch64-softmmu/hmp-commands.h GEN aarch64-softmmu/hmp-commands-info.h GEN aarch64-softmmu/config-target.h CC x86_64-softmmu/exec.o CC x86_64-softmmu/translate-all.o CC x86_64-softmmu/tcg/tcg.o CC x86_64-softmmu/tcg/tcg-op.o CC x86_64-softmmu/cpu-exec.o CC x86_64-softmmu/translate-common.o CC x86_64-softmmu/cpu-exec-common.o CC x86_64-softmmu/tcg/optimize.o CC x86_64-softmmu/tcg/tcg-common.o CC x86_64-softmmu/fpu/softfloat.o CC x86_64-softmmu/hax-stub.o CC x86_64-softmmu/disas.o CC x86_64-softmmu/tcg-runtime.o CC x86_64-softmmu/arch_init.o CC x86_64-softmmu/cpus.o CC x86_64-softmmu/monitor.o CC x86_64-softmmu/gdbstub.o CC aarch64-softmmu/exec.o CC aarch64-softmmu/translate-all.o CC aarch64-softmmu/cpu-exec.o CC x86_64-softmmu/balloon.o CC aarch64-softmmu/translate-common.o CC aarch64-softmmu/cpu-exec-common.o CC x86_64-softmmu/ioport.o CC aarch64-softmmu/tcg/tcg.o CC aarch64-softmmu/tcg/tcg-op.o CC aarch64-softmmu/tcg/optimize.o CC aarch64-softmmu/tcg/tcg-common.o CC aarch64-softmmu/fpu/softfloat.o CC x86_64-softmmu/qtest.o CC x86_64-softmmu/numa.o CC aarch64-softmmu/disas.o CC x86_64-softmmu/bootdevice.o CC aarch64-softmmu/tcg-runtime.o GEN aarch64-softmmu/gdbstub-xml.c CC x86_64-softmmu/kvm-all.o CC aarch64-softmmu/kvm-stub.o CC aarch64-softmmu/hax-stub.o CC aarch64-softmmu/arch_init.o CC aarch64-softmmu/cpus.o CC aarch64-softmmu/balloon.o CC aarch64-softmmu/monitor.o CC x86_64-softmmu/memory.o CC aarch64-softmmu/gdbstub.o CC aarch64-softmmu/ioport.o CC aarch64-softmmu/numa.o CC x86_64-softmmu/cputlb.o CC x86_64-softmmu/memory_mapping.o CC aarch64-softmmu/qtest.o CC x86_64-softmmu/dump.o CC aarch64-softmmu/bootdevice.o CC x86_64-softmmu/migration/ram.o CC x86_64-softmmu/migration/savevm.o CC aarch64-softmmu/memory.o CC aarch64-softmmu/cputlb.o CC aarch64-softmmu/memory_mapping.o CC aarch64-softmmu/migration/ram.o CC aarch64-softmmu/dump.o CC aarch64-softmmu/migration/savevm.o CC x86_64-softmmu/xen-common-stub.o CC aarch64-softmmu/xen-common-stub.o CC x86_64-softmmu/xen-hvm-stub.o CC x86_64-softmmu/hw/block/virtio-blk.o CC x86_64-softmmu/hw/block/dataplane/virtio-blk.o CC aarch64-softmmu/hw/block/virtio-blk.o CC aarch64-softmmu/xen-hvm-stub.o CC aarch64-softmmu/hw/adc/stm32f2xx_adc.o CC aarch64-softmmu/hw/block/dataplane/virtio-blk.o CC aarch64-softmmu/hw/char/exynos4210_uart.o CC x86_64-softmmu/hw/char/virtio-serial-bus.o CC aarch64-softmmu/hw/char/omap_uart.o CC x86_64-softmmu/hw/core/nmi.o CC x86_64-softmmu/hw/core/generic-loader.o CC x86_64-softmmu/hw/core/null-machine.o CC x86_64-softmmu/hw/cpu/core.o CC aarch64-softmmu/hw/char/digic-uart.o CC x86_64-softmmu/hw/display/vga.o CC x86_64-softmmu/hw/display/virtio-gpu-3d.o CC x86_64-softmmu/hw/display/virtio-gpu.o CC x86_64-softmmu/hw/display/virtio-gpu-pci.o CC aarch64-softmmu/hw/char/stm32f2xx_usart.o CC aarch64-softmmu/hw/char/bcm2835_aux.o CC x86_64-softmmu/hw/display/virtio-vga.o CC aarch64-softmmu/hw/char/virtio-serial-bus.o CC x86_64-softmmu/hw/intc/apic.o CC x86_64-softmmu/hw/intc/apic_common.o CC aarch64-softmmu/hw/core/nmi.o CC aarch64-softmmu/hw/core/generic-loader.o CC aarch64-softmmu/hw/core/null-machine.o CC aarch64-softmmu/hw/cpu/realview_mpcore.o CC aarch64-softmmu/hw/cpu/arm11mpcore.o CC aarch64-softmmu/hw/cpu/a9mpcore.o CC aarch64-softmmu/hw/cpu/a15mpcore.o CC x86_64-softmmu/hw/intc/ioapic.o CC aarch64-softmmu/hw/cpu/core.o CC aarch64-softmmu/hw/display/omap_dss.o CC aarch64-softmmu/hw/display/omap_lcdc.o CC aarch64-softmmu/hw/display/pxa2xx_lcd.o CC aarch64-softmmu/hw/display/bcm2835_fb.o CC x86_64-softmmu/hw/isa/lpc_ich9.o CC x86_64-softmmu/hw/misc/vmport.o CC aarch64-softmmu/hw/display/vga.o CC aarch64-softmmu/hw/display/virtio-gpu.o CC aarch64-softmmu/hw/display/virtio-gpu-3d.o CC x86_64-softmmu/hw/misc/ivshmem.o CC x86_64-softmmu/hw/misc/pvpanic.o CC aarch64-softmmu/hw/display/virtio-gpu-pci.o CC aarch64-softmmu/hw/display/dpcd.o CC x86_64-softmmu/hw/misc/edu.o CC x86_64-softmmu/hw/misc/hyperv_testdev.o CC aarch64-softmmu/hw/display/xlnx_dp.o CC x86_64-softmmu/hw/net/virtio-net.o CC x86_64-softmmu/hw/net/vhost_net.o CC x86_64-softmmu/hw/scsi/virtio-scsi.o CC x86_64-softmmu/hw/scsi/virtio-scsi-dataplane.o CC aarch64-softmmu/hw/dma/xlnx_dpdma.o CC aarch64-softmmu/hw/dma/omap_dma.o CC x86_64-softmmu/hw/scsi/vhost-scsi.o CC aarch64-softmmu/hw/dma/soc_dma.o CC x86_64-softmmu/hw/timer/mc146818rtc.o CC x86_64-softmmu/hw/vfio/common.o CC x86_64-softmmu/hw/vfio/pci.o CC aarch64-softmmu/hw/dma/pxa2xx_dma.o CC aarch64-softmmu/hw/dma/bcm2835_dma.o CC x86_64-softmmu/hw/vfio/pci-quirks.o CC x86_64-softmmu/hw/vfio/platform.o CC aarch64-softmmu/hw/gpio/omap_gpio.o CC aarch64-softmmu/hw/gpio/imx_gpio.o CC x86_64-softmmu/hw/vfio/calxeda-xgmac.o CC aarch64-softmmu/hw/i2c/omap_i2c.o CC x86_64-softmmu/hw/vfio/amd-xgbe.o CC aarch64-softmmu/hw/input/pxa2xx_keypad.o CC aarch64-softmmu/hw/input/tsc210x.o CC x86_64-softmmu/hw/vfio/spapr.o CC aarch64-softmmu/hw/intc/armv7m_nvic.o CC aarch64-softmmu/hw/intc/exynos4210_gic.o CC x86_64-softmmu/hw/virtio/virtio.o CC aarch64-softmmu/hw/intc/exynos4210_combiner.o CC aarch64-softmmu/hw/intc/omap_intc.o CC aarch64-softmmu/hw/intc/bcm2835_ic.o CC x86_64-softmmu/hw/virtio/virtio-balloon.o CC aarch64-softmmu/hw/intc/bcm2836_control.o CC x86_64-softmmu/hw/virtio/vhost.o CC x86_64-softmmu/hw/virtio/vhost-backend.o CC aarch64-softmmu/hw/intc/allwinner-a10-pic.o CC x86_64-softmmu/hw/virtio/vhost-user.o CC aarch64-softmmu/hw/intc/aspeed_vic.o CC x86_64-softmmu/hw/virtio/vhost-vsock.o CC aarch64-softmmu/hw/misc/ivshmem.o CC aarch64-softmmu/hw/intc/arm_gicv3_cpuif.o CC x86_64-softmmu/hw/virtio/virtio-crypto-pci.o CC x86_64-softmmu/hw/virtio/virtio-crypto.o CC x86_64-softmmu/hw/i386/multiboot.o CC x86_64-softmmu/hw/i386/pc.o CC x86_64-softmmu/hw/i386/pc_piix.o CC x86_64-softmmu/hw/i386/pc_q35.o CC aarch64-softmmu/hw/misc/arm_sysctl.o CC x86_64-softmmu/hw/i386/pc_sysfw.o CC aarch64-softmmu/hw/misc/cbus.o CC x86_64-softmmu/hw/i386/x86-iommu.o CC aarch64-softmmu/hw/misc/exynos4210_pmu.o CC aarch64-softmmu/hw/misc/imx_ccm.o CC aarch64-softmmu/hw/misc/imx31_ccm.o CC x86_64-softmmu/hw/i386/intel_iommu.o CC x86_64-softmmu/hw/i386/amd_iommu.o CC aarch64-softmmu/hw/misc/imx25_ccm.o CC aarch64-softmmu/hw/misc/imx6_ccm.o CC aarch64-softmmu/hw/misc/imx6_src.o CC x86_64-softmmu/hw/i386/kvmvapic.o CC x86_64-softmmu/hw/i386/acpi-build.o CC aarch64-softmmu/hw/misc/mst_fpga.o /tmp/qemu-test/src/hw/i386/pc_piix.c: In function ‘igd_passthrough_isa_bridge_create’: /tmp/qemu-test/src/hw/i386/pc_piix.c:1055: warning: ‘pch_rev_id’ may be used uninitialized in this function CC x86_64-softmmu/hw/i386/kvm/clock.o CC x86_64-softmmu/hw/i386/pci-assign-load-rom.o CC aarch64-softmmu/hw/misc/omap_clk.o CC x86_64-softmmu/hw/i386/kvm/apic.o CC x86_64-softmmu/hw/i386/kvm/i8259.o CC x86_64-softmmu/hw/i386/kvm/ioapic.o CC x86_64-softmmu/hw/i386/kvm/i8254.o CC aarch64-softmmu/hw/misc/omap_gpmc.o CC aarch64-softmmu/hw/misc/omap_l4.o CC aarch64-softmmu/hw/misc/omap_sdrc.o CC x86_64-softmmu/hw/i386/kvm/pci-assign.o /tmp/qemu-test/src/hw/i386/acpi-build.c: In function ‘build_append_pci_bus_devices’: /tmp/qemu-test/src/hw/i386/acpi-build.c:495: warning: ‘notify_method’ may be used uninitialized in this function CC x86_64-softmmu/target/i386/translate.o CC aarch64-softmmu/hw/misc/omap_tap.o CC aarch64-softmmu/hw/misc/bcm2835_mbox.o CC aarch64-softmmu/hw/misc/bcm2835_property.o CC aarch64-softmmu/hw/misc/zynq_slcr.o CC aarch64-softmmu/hw/misc/zynq-xadc.o CC aarch64-softmmu/hw/misc/stm32f2xx_syscfg.o CC aarch64-softmmu/hw/misc/edu.o CC aarch64-softmmu/hw/misc/auxbus.o CC x86_64-softmmu/target/i386/helper.o CC x86_64-softmmu/target/i386/cpu.o CC x86_64-softmmu/target/i386/bpt_helper.o CC aarch64-softmmu/hw/misc/aspeed_scu.o CC x86_64-softmmu/target/i386/excp_helper.o CC x86_64-softmmu/target/i386/fpu_helper.o CC x86_64-softmmu/target/i386/cc_helper.o CC aarch64-softmmu/hw/misc/aspeed_sdmc.o CC x86_64-softmmu/target/i386/int_helper.o CC aarch64-softmmu/hw/net/virtio-net.o CC x86_64-softmmu/target/i386/svm_helper.o CC x86_64-softmmu/target/i386/smm_helper.o CC aarch64-softmmu/hw/net/vhost_net.o CC aarch64-softmmu/hw/pcmcia/pxa2xx.o CC x86_64-softmmu/target/i386/misc_helper.o CC x86_64-softmmu/target/i386/mem_helper.o CC aarch64-softmmu/hw/scsi/virtio-scsi.o CC x86_64-softmmu/target/i386/seg_helper.o CC aarch64-softmmu/hw/scsi/virtio-scsi-dataplane.o CC aarch64-softmmu/hw/scsi/vhost-scsi.o CC x86_64-softmmu/target/i386/mpx_helper.o CC aarch64-softmmu/hw/sd/omap_mmc.o CC x86_64-softmmu/target/i386/gdbstub.o CC aarch64-softmmu/hw/sd/pxa2xx_mmci.o CC x86_64-softmmu/target/i386/machine.o CC x86_64-softmmu/target/i386/arch_memory_mapping.o CC x86_64-softmmu/target/i386/arch_dump.o CC aarch64-softmmu/hw/ssi/omap_spi.o CC x86_64-softmmu/target/i386/monitor.o CC aarch64-softmmu/hw/ssi/imx_spi.o CC aarch64-softmmu/hw/timer/exynos4210_mct.o CC x86_64-softmmu/target/i386/kvm.o CC aarch64-softmmu/hw/timer/exynos4210_pwm.o CC x86_64-softmmu/target/i386/hyperv.o CC aarch64-softmmu/hw/timer/exynos4210_rtc.o CC aarch64-softmmu/hw/timer/omap_gptimer.o CC aarch64-softmmu/hw/timer/omap_synctimer.o CC aarch64-softmmu/hw/timer/pxa2xx_timer.o GEN trace/generated-helpers.c CC aarch64-softmmu/hw/timer/digic-timer.o CC aarch64-softmmu/hw/timer/allwinner-a10-pit.o CC x86_64-softmmu/trace/control-target.o CC aarch64-softmmu/hw/usb/tusb6010.o CC aarch64-softmmu/hw/vfio/common.o CC aarch64-softmmu/hw/vfio/pci.o CC aarch64-softmmu/hw/vfio/platform.o CC aarch64-softmmu/hw/vfio/pci-quirks.o CC aarch64-softmmu/hw/vfio/calxeda-xgmac.o CC aarch64-softmmu/hw/vfio/amd-xgbe.o CC aarch64-softmmu/hw/vfio/spapr.o CC aarch64-softmmu/hw/virtio/virtio.o CC aarch64-softmmu/hw/virtio/virtio-balloon.o CC aarch64-softmmu/hw/virtio/vhost.o CC aarch64-softmmu/hw/virtio/vhost-backend.o CC aarch64-softmmu/hw/virtio/vhost-user.o CC aarch64-softmmu/hw/virtio/vhost-vsock.o CC aarch64-softmmu/hw/virtio/virtio-crypto.o CC aarch64-softmmu/hw/virtio/virtio-crypto-pci.o CC aarch64-softmmu/hw/arm/boot.o CC aarch64-softmmu/hw/arm/collie.o CC aarch64-softmmu/hw/arm/exynos4_boards.o CC aarch64-softmmu/hw/arm/gumstix.o CC aarch64-softmmu/hw/arm/highbank.o CC aarch64-softmmu/hw/arm/digic_boards.o CC aarch64-softmmu/hw/arm/integratorcp.o CC aarch64-softmmu/hw/arm/mainstone.o CC aarch64-softmmu/hw/arm/musicpal.o CC aarch64-softmmu/hw/arm/nseries.o CC aarch64-softmmu/hw/arm/omap_sx1.o CC aarch64-softmmu/hw/arm/palm.o CC aarch64-softmmu/hw/arm/realview.o CC aarch64-softmmu/hw/arm/spitz.o CC x86_64-softmmu/trace/generated-helpers.o CC aarch64-softmmu/hw/arm/stellaris.o CC aarch64-softmmu/hw/arm/tosa.o CC aarch64-softmmu/hw/arm/versatilepb.o CC aarch64-softmmu/hw/arm/vexpress.o CC aarch64-softmmu/hw/arm/virt.o CC aarch64-softmmu/hw/arm/xilinx_zynq.o CC aarch64-softmmu/hw/arm/z2.o CC aarch64-softmmu/hw/arm/virt-acpi-build.o CC aarch64-softmmu/hw/arm/netduino2.o CC aarch64-softmmu/hw/arm/sysbus-fdt.o CC aarch64-softmmu/hw/arm/armv7m.o CC aarch64-softmmu/hw/arm/exynos4210.o CC aarch64-softmmu/hw/arm/pxa2xx.o CC aarch64-softmmu/hw/arm/pxa2xx_gpio.o CC aarch64-softmmu/hw/arm/pxa2xx_pic.o CC aarch64-softmmu/hw/arm/digic.o CC aarch64-softmmu/hw/arm/omap1.o CC aarch64-softmmu/hw/arm/omap2.o CC aarch64-softmmu/hw/arm/strongarm.o CC aarch64-softmmu/hw/arm/allwinner-a10.o CC aarch64-softmmu/hw/arm/cubieboard.o CC aarch64-softmmu/hw/arm/bcm2835_peripherals.o CC aarch64-softmmu/hw/arm/bcm2836.o CC aarch64-softmmu/hw/arm/raspi.o CC aarch64-softmmu/hw/arm/stm32f205_soc.o CC aarch64-softmmu/hw/arm/xlnx-zynqmp.o CC aarch64-softmmu/hw/arm/xlnx-ep108.o CC aarch64-softmmu/hw/arm/fsl-imx25.o CC aarch64-softmmu/hw/arm/imx25_pdk.o CC aarch64-softmmu/hw/arm/fsl-imx31.o CC aarch64-softmmu/hw/arm/kzm.o CC aarch64-softmmu/hw/arm/fsl-imx6.o CC aarch64-softmmu/hw/arm/sabrelite.o CC aarch64-softmmu/hw/arm/aspeed_soc.o CC aarch64-softmmu/hw/arm/aspeed.o CC aarch64-softmmu/target/arm/arm-semi.o CC aarch64-softmmu/target/arm/machine.o CC aarch64-softmmu/target/arm/psci.o CC aarch64-softmmu/target/arm/arch_dump.o CC aarch64-softmmu/target/arm/monitor.o CC aarch64-softmmu/target/arm/kvm-stub.o LINK x86_64-softmmu/qemu-system-x86_64 CC aarch64-softmmu/target/arm/translate.o CC aarch64-softmmu/target/arm/op_helper.o CC aarch64-softmmu/target/arm/helper.o CC aarch64-softmmu/target/arm/cpu.o CC aarch64-softmmu/target/arm/neon_helper.o CC aarch64-softmmu/target/arm/iwmmxt_helper.o CC aarch64-softmmu/target/arm/gdbstub.o CC aarch64-softmmu/target/arm/cpu64.o CC aarch64-softmmu/target/arm/gdbstub64.o CC aarch64-softmmu/target/arm/translate-a64.o CC aarch64-softmmu/target/arm/helper-a64.o CC aarch64-softmmu/target/arm/crypto_helper.o CC aarch64-softmmu/target/arm/arm-powerctl.o GEN trace/generated-helpers.c CC aarch64-softmmu/trace/control-target.o
/tmp/qemu-test/src/target/arm/translate-a64.c: In function ‘handle_shri_with_rndacc’:/tmp/qemu-test/src/target/arm/translate-a64.c:6369: warning: ‘tcg_src_hi’ may be used uninitialized in this function /tmp/qemu-test/src/target/arm/translate-a64.c: In function ‘disas_simd_scalar_two_reg_misc’:/tmp/qemu-test/src/target/arm/translate-a64.c:8096: warning: ‘rmode’ may be used uninitialized in this function
CC aarch64-softmmu/gdbstub-xml.o CC aarch64-softmmu/trace/generated-helpers.o LINK aarch64-softmmu/qemu-system-aarch64
make[1]: *** [docker-run] Error 2
make[1]: Leaving directory `/var/tmp/patchew-tester-tmp-oelmqzb0/src'
make: *** [docker-run-test-quick@centos6] Error 2
=== OUTPUT END ===
Test command exited with code: 2
---
Email generated automatically by Patchew [http://patchew.org/].
Please send your feedback to [email protected]
Quoted 34 lines · Peter Lieven
this is something I have been thinking about for almost 2 years now. we heavily have the following two use cases when using qemu-img convert.
a) reading from NFS and writing to iSCSI for deploying templates
b) reading from iSCSI and writing to NFS for backupsIn both processes we use libiscsi and libnfs so we have no kernel pagecache. As qemu-img convert is implemented with sync operations that means we read one buffer and then write it. No parallelism and each sync request takes as long as it takes until it is completed.
What I put together is an approach to use aio routines for the conversion process to have ideally read and write happening in parallel.
The code is far from clean or complete, but I would appreaciate comments and thoughts from you.
So far I have the following runtimes when reading an uncompressed QCOW2 from NFS and writing it to iSCSI (raw):
qemu-img (master)
nfs -> iscsi 33 secs
nfs -> ram 19 secs
ram -> iscsi 14 secsqemu-img-async
nfs -> iscsi 23 secs
nfs -> ram 17 secs
ram -> iscsi 14 secsIts visible that on master the runtimes add up as expected. The async branch is faster, but not as fast as I would have expected. I would expect the runtime to be as slow as the slowest of the two involved transfers.
Quoted 7 lines
Thank you,
PeterSigned-off-by Peter Lieven <[email protected]>--- qemu-img.c | 271 +++++++++++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 199 insertions(+), 72 deletions(-)
Asynchronous convert sounds good. But your implementation looks a bit weird to me.
Your implementation has four "slots" which receive work from a central work queue that they then process. You can do that, but it looks counter-intuitive to me. (Or if you do that, I would do it using
coroutines: Start up four coroutines that simply submit blk_co_* requests.)
What I would have done (if using AIO) is the following: Seek through the image, finding the next bit of work to do (without having a central work queue). Then submit an AIO request with a newly allocated piece of data (not using fixed slots). Continue until four requests are in flight, then wait until one is settled.
I think this would simplify the design. Also, it's basically what the mirror block job does.
Which brings me to a totally different point: At some point we intended to convert as many qemu-img functions to block jobs as possible. Unfortunately, we only ever did one and that's commit. If we were to convert convert to mirror, then we'd get async for free.
But the effort of converting convert to mirror is probably much larger than adding async to convert...
Max
Quoted 60 lines · Max Reitz
Quoted 34 lines · Peter Lieven
this is something I have been thinking about for almost 2 years now. we heavily have the following two use cases when using qemu-img convert.
a) reading from NFS and writing to iSCSI for deploying templates
b) reading from iSCSI and writing to NFS for backupsIn both processes we use libiscsi and libnfs so we have no kernel pagecache. As qemu-img convert is implemented with sync operations that means we read one buffer and then write it. No parallelism and each sync request takes as long as it takes until it is completed.
What I put together is an approach to use aio routines for the conversion process to have ideally read and write happening in parallel.
The code is far from clean or complete, but I would appreaciate comments and thoughts from you.
So far I have the following runtimes when reading an uncompressed QCOW2 from NFS and writing it to iSCSI (raw):
qemu-img (master)
nfs -> iscsi 33 secs
nfs -> ram 19 secs
ram -> iscsi 14 secsqemu-img-async
nfs -> iscsi 23 secs
nfs -> ram 17 secs
ram -> iscsi 14 secsIts visible that on master the runtimes add up as expected. The async branch is faster, but not as fast as I would have expected. I would expect the runtime to be as slow as the slowest of the two involved transfers.
Quoted 7 lines
Thank you,
PeterSigned-off-by Peter Lieven <[email protected]>--- qemu-img.c | 271 +++++++++++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 199 insertions(+), 72 deletions(-)Asynchronous convert sounds good. But your implementation looks a bit weird to me.
Your implementation has four "slots" which receive work from a central work queue that they then process. You can do that, but it looks counter-intuitive to me. (Or if you do that, I would do it using
coroutines: Start up four coroutines that simply submit blk_co_* requests.)What I would have done (if using AIO) is the following: Seek through the image, finding the next bit of work to do (without having a central work queue). Then submit an AIO request with a newly allocated piece of data (not using fixed slots). Continue until four requests are in flight, then wait until one is settled.
I think this would simplify the design. Also, it's basically what the mirror block job does.
Quoted 7 lines
Which brings me to a totally different point: At some point we intended to convert as many qemu-img functions to block jobs as possible. Unfortunately, we only ever did one and that's commit. If we were to convert convert to mirror, then we'd get async for free.
But the effort of converting convert to mirror is probably much larger than adding async to convert...
My 2 cents.
In the long run it still seems to me as a worthwhile deal. Now we know it will have a useful advantage by bringing in async, maybe it's a good time to do it!
Related, I have had the feeling that block jobs can be cleaned up too: backup, commit, stream can all reuse mirror code to achieve the "async" feature that Vladmir is proposing, in an AIO way which is more resource friendly than coroutine worker pool.
Fam
Am 12.02.2017 um 03:06 schrieb Max Reitz:
Quoted 50 lines · Peter Lieven
Quoted 34 lines
this is something I have been thinking about for almost 2 years now. we heavily have the following two use cases when using qemu-img convert.
a) reading from NFS and writing to iSCSI for deploying templates
b) reading from iSCSI and writing to NFS for backupsIn both processes we use libiscsi and libnfs so we have no kernel pagecache. As qemu-img convert is implemented with sync operations that means we read one buffer and then write it. No parallelism and each sync request takes as long as it takes until it is completed.
What I put together is an approach to use aio routines for the conversion process to have ideally read and write happening in parallel.
The code is far from clean or complete, but I would appreaciate comments and thoughts from you.
So far I have the following runtimes when reading an uncompressed QCOW2 from NFS and writing it to iSCSI (raw):
qemu-img (master)
nfs -> iscsi 33 secs
nfs -> ram 19 secs
ram -> iscsi 14 secsqemu-img-async
nfs -> iscsi 23 secs
nfs -> ram 17 secs
ram -> iscsi 14 secsIts visible that on master the runtimes add up as expected. The async branch is faster, but not as fast as I would have expected. I would expect the runtime to be as slow as the slowest of the two involved transfers.
Quoted 7 lines
Thank you,
PeterSigned-off-by Peter Lieven <[email protected]>--- qemu-img.c | 271 +++++++++++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 199 insertions(+), 72 deletions(-)Asynchronous convert sounds good. But your implementation looks a bit weird to me.
Your implementation has four "slots" which receive work from a central work queue that they then process. You can do that, but it looks counter-intuitive to me. (Or if you do that, I would do it using
coroutines: Start up four coroutines that simply submit blk_co_* requests.)
Quoted 5 lines
What I would have done (if using AIO) is the following: Seek through the image, finding the next bit of work to do (without having a central work queue). Then submit an AIO request with a newly allocated piece of data (not using fixed slots). Continue until four requests are in flight, then wait until one is settled.
Hi Max,
thank you very much for you feedback. The reason why I have this worker queue is that I ended up with recursive invocations of the fill_read/write_queue functions. The problem is convert_iterate_sectors. Its not only quite expensive it also leads to callback invocation.
I also tried with coroutines, but ended up in the same problem. But I did not fire up worker "threads", I fired up a coroutine for each read/write. Let my try to have x worker threads and let them do the work. The 2 issues that have to be adressed however are: convert_iterate_sectors must not be called in parallel and all writes should be sequential.
I will have a look.
Peter