live block copy (v2)

Support live image copy + switch. That is, copy an image backing a guest hard disk to a destination image (destination image must be created separately), and switch to this copy.

Command syntax:

block_copy device filename [commit_filename] [-i] -- live block copy device to image
optional commit filename
-i for incremental copy (base image shared between src and destination)

Signed-off-by Marcelo Tosatti<[email protected]>

Index: qemu/block-copy.c
===================================================================

block-copy.c new file +400 -0

@@ -0,0 +1,741 @@
+/*
+ * QEMU live block copy
+ *
+ * Copyright (C) 2010 Red Hat Inc.
+ *
+ * Authors: Marcelo Tosatti<[email protected]>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2.  See
+ * the COPYING file in the top-level directory.
+ *
+ */
+
+#include "qemu-common.h"
+#include "block_int.h"
+#include "blockdev.h"
+#include "qemu-queue.h"
+#include "qemu-timer.h"
+#include "monitor.h"
+#include "block-copy.h"
+#include "migration.h"
+#include "sysemu.h"
+#include "qjson.h"
+#include<assert.h>
+
+#define BLOCK_SIZE (BDRV_SECTORS_PER_DIRTY_CHUNK<<  BDRV_SECTOR_BITS)
+#define MAX_IS_ALLOCATED_SEARCH 65536
+
+/*
+ * Stages:
+ *
+ * STAGE_BULK: bulk reads/writes in progress
+ * STAGE_BULK_FINISHED: bulk reads finished, bulk writes in progress
+ * STAGE_DIRTY: bulk writes finished, dirty reads/writes in progress
+ * STAGE_SWITCH_FINISHED: switched to new image.
+ */
+
+enum BdrvCopyStage {
+    STAGE_BULK,
+    STAGE_BULK_FINISHED,
+    STAGE_DIRTY,
+    STAGE_SWITCH_FINISHED,
+};
+
+typedef struct BdrvCopyState {
+    BlockDriverState *src;
+    BlockDriverState *dst;
+    bool shared_base;
+
+    int64_t curr_sector;
+    int64_t completed_sectors;
+    int64_t nr_sectors;
+
+    enum BdrvCopyStage stage;
+    int inflight_reads;
+    int error;
+    int failed;
+    int cancelled;
+    QLIST_HEAD(, BdrvCopyBlock) io_list;
+    unsigned long *aio_bitmap;
+    QEMUTimer *aio_timer;
+    QLIST_ENTRY(BdrvCopyState) list;
+
+    int64_t blocks;
+    int64_t total_time;
+
+    char src_device_name[32];
+    char dst_filename[1024];
+    int commit_fd;
+} BdrvCopyState;
+
+typedef struct BdrvCopyBlock {
+    BdrvCopyState *state;
+    uint8_t *buf;
+    int64_t sector;
+    int64_t nr_sectors;
+    struct iovec iov;
+    QEMUIOVector qiov;
+    BlockDriverAIOCB *aiocb;
+    int64_t time;
+    QLIST_ENTRY(BdrvCopyBlock) list;
+} BdrvCopyBlock;
+
+static QLIST_HEAD(, BdrvCopyState) block_copy_list =
+    QLIST_HEAD_INITIALIZER(block_copy_list);
+
+static void alloc_aio_bitmap(BdrvCopyState *s)
+{
+    BlockDriverState *bs = s->src;
+    int64_t bitmap_size;
+
+    bitmap_size = (bdrv_getlength(bs)>>  BDRV_SECTOR_BITS) +
+            BDRV_SECTORS_PER_DIRTY_CHUNK * 8 - 1;
+    bitmap_size /= BDRV_SECTORS_PER_DIRTY_CHUNK * 8;
+
+    s->aio_bitmap = qemu_mallocz(bitmap_size);
+}
+
+static bool aio_inflight(BdrvCopyState *s, int64_t sector)
+{
+    int64_t chunk = sector / (int64_t)BDRV_SECTORS_PER_DIRTY_CHUNK;
+
+    if (s->aio_bitmap&&
+        (sector<<  BDRV_SECTOR_BITS)<  bdrv_getlength(s->src)) {
+        return !!(s->aio_bitmap[chunk / (sizeof(unsigned long) * 8)]&
+            (1UL<<  (chunk % (sizeof(unsigned long) * 8))));
+    } else {
+        return 0;
+    }
+}
+
+static void set_aio_inflight(BdrvCopyState *s, int64_t sector_num,
+                             int nb_sectors, int set)
+{
+    int64_t start, end;
+    unsigned long val, idx, bit;
+
+    start = sector_num / BDRV_SECTORS_PER_DIRTY_CHUNK;
+    end = (sector_num + nb_sectors - 1) / BDRV_SECTORS_PER_DIRTY_CHUNK;
+
+    for (; start<= end; start++) {
+        idx = start / (sizeof(unsigned long) * 8);
+        bit = start % (sizeof(unsigned long) * 8);
+        val = s->aio_bitmap[idx];
+        if (set) {
+            if (!(val&  (1UL<<  bit))) {
+                val |= 1UL<<  bit;
+            }
+        } else {
+            if (val&  (1UL<<  bit)) {
+                val&= ~(1UL<<  bit);
+            }
+        }
+        s->aio_bitmap[idx] = val;
+    }
+}
+
+static void blkcopy_set_stage(BdrvCopyState *s, enum BdrvCopyStage stage)
+{
+    s->stage = stage;
+
+    switch (stage) {
+    case STAGE_BULK:
+        BLKDBG_EVENT(s->dst->file, BLKDBG_BLKCOPY_STAGE_BULK);
+        break;
+    case STAGE_BULK_FINISHED:
+        BLKDBG_EVENT(s->dst->file, BLKDBG_BLKCOPY_STAGE_BULK_FINISHED);
+        break;
+    case STAGE_DIRTY:
+        BLKDBG_EVENT(s->dst->file, BLKDBG_BLKCOPY_STAGE_DIRTY);
+        break;
+    case STAGE_SWITCH_FINISHED:
+        BLKDBG_EVENT(s->dst->file, BLKDBG_BLKCOPY_STAGE_SWITCH_FINISHED);
+        break;
+    default:
+        break;
+    }
+}
+
+static void blk_copy_handle_cb_error(BdrvCopyState *s, int ret)
+{
+    s->error = ret;
+    qemu_mod_timer(s->aio_timer, qemu_get_clock(rt_clock));
+}
+
+static inline void add_avg_transfer_time(BdrvCopyState *s, int64_t time)
+{
+    s->blocks++;
+    s->total_time += time;
+}
+
+static void blk_copy_write_cb(void *opaque, int ret)
+{
+    BdrvCopyBlock *blk = opaque;
+    BdrvCopyState *s = blk->state;
+
+    if (ret<  0) {
+        QLIST_REMOVE(blk, list);
+        qemu_free(blk->buf);
+        qemu_free(blk);
+        blk_copy_handle_cb_error(s, ret);
+        return;
+    }
+
+    QLIST_REMOVE(blk, list);
+    add_avg_transfer_time(s, qemu_get_clock_ns(rt_clock) - blk->time);
+
+    /* schedule switch to STAGE_DIRTY on last bulk write completion */
+    if (blk->state->stage == STAGE_BULK_FINISHED) {
+        qemu_mod_timer(s->aio_timer, qemu_get_clock(rt_clock));
+    }
+
+    if (blk->state->stage>  STAGE_BULK_FINISHED) {
+        set_aio_inflight(blk->state, blk->sector, blk->nr_sectors, 0);
+    }
+
+    qemu_free(blk->buf);
+    qemu_free(blk);
+}
+
+static void blk_copy_issue_write(BdrvCopyState *s, BdrvCopyBlock *read_blk)
+{
+    BdrvCopyBlock *blk = qemu_mallocz(sizeof(BdrvCopyBlock));
+    blk->state = s;
+    blk->sector = read_blk->sector;
+    blk->nr_sectors = read_blk->nr_sectors;
+    blk->time = read_blk->time;
+    blk->buf = read_blk->buf;
+    QLIST_INSERT_HEAD(&s->io_list, blk, list);
+
+    blk->iov.iov_base = read_blk->buf;
+    blk->iov.iov_len = read_blk->iov.iov_len;
+    qemu_iovec_init_external(&blk->qiov,&blk->iov, 1);
+
+    BLKDBG_EVENT(s->dst->file, BLKDBG_BLKCOPY_AIO_WRITE);
+    blk->aiocb = bdrv_aio_writev(s->dst, blk->sector,&blk->qiov,
+                                 blk->iov.iov_len / BDRV_SECTOR_SIZE,
+                                 blk_copy_write_cb, blk);
+    if (!blk->aiocb) {
+        s->error = 1;
+        goto error;
+    }
+
+    return;
+
+error:
+    QLIST_REMOVE(blk, list);
+    qemu_free(read_blk->buf);
+    qemu_free(blk);
+}
+
+static void blk_copy_read_cb(void *opaque, int ret)
+{
+    BdrvCopyBlock *blk = opaque;
+    BdrvCopyState *s = blk->state;
+
+    s->inflight_reads--;
+    if (ret<  0) {
+        QLIST_REMOVE(blk, list);
+        qemu_free(blk->buf);
+        qemu_free(blk);
+        blk_copy_handle_cb_error(s, ret);
+        return;
+    }
+    blk_copy_issue_write(s, blk);
+    QLIST_REMOVE(blk, list);
+    qemu_free(blk);
+    qemu_mod_timer(s->aio_timer, qemu_get_clock(rt_clock));
+}
+
+static void blk_copy_issue_read(BdrvCopyState *s, int64_t sector,
+                                int nr_sectors)
+{
+    BdrvCopyBlock *blk = qemu_mallocz(sizeof(BdrvCopyBlock));
+    blk->buf = qemu_mallocz(BLOCK_SIZE);
+    blk->state = s;
+    blk->sector = sector;
+    blk->nr_sectors = nr_sectors;
+    QLIST_INSERT_HEAD(&s->io_list, blk, list);
+
+    blk->iov.iov_base = blk->buf;
+    blk->iov.iov_len = nr_sectors * BDRV_SECTOR_SIZE;
+    qemu_iovec_init_external(&blk->qiov,&blk->iov, 1);
+
+    s->inflight_reads++;
+    blk->time = qemu_get_clock_ns(rt_clock);
+    blk->aiocb = bdrv_aio_readv(s->src, sector,&blk->qiov, nr_sectors,
+                                blk_copy_read_cb, blk);
+    if (!blk->aiocb) {
+        s->error = 1;
+        goto error;
+    }
+
+    return;
+
+error:
+    s->inflight_reads--;
+    QLIST_REMOVE(blk, list);
+    qemu_free(blk->buf);
+    qemu_free(blk);
+}
+
+static bool blkcopy_can_switch(BdrvCopyState *s)
+{
+    int64_t remaining_dirty;
+    int64_t avg_transfer_time;
+
+    remaining_dirty = bdrv_get_dirty_count(s->src);
+    if (remaining_dirty == 0 || s->blocks == 0) {
+        return true;
+    }
+
+    avg_transfer_time = s->total_time / s->blocks;
+    if ((remaining_dirty * avg_transfer_time)<= migrate_max_downtime()) {
+        return true;
+    }
+    return false;
+}
+
+static int blk_issue_reads_dirty(BdrvCopyState *s)
+{
+    int64_t sector;
+
+    for (sector = s->curr_sector; sector<  s->nr_sectors;) {
+        if (bdrv_get_dirty(s->src, sector)&&  !aio_inflight(s, sector)) {
+            int nr_sectors = MIN(s->nr_sectors - s->curr_sector,
+                                 BDRV_SECTORS_PER_DIRTY_CHUNK);
+
+            blk_copy_issue_read(s, sector, nr_sectors);
+            bdrv_reset_dirty(s->src, sector, nr_sectors);
+            set_aio_inflight(s, sector, nr_sectors, 1);
+            break;
+        }
+
+        sector += BDRV_SECTORS_PER_DIRTY_CHUNK;
+        s->curr_sector = sector;
+    }
+
+    if (sector>= s->nr_sectors) {
+        s->curr_sector = 0;
+    }
+    return 0;
+}
+
+static int blk_issue_reads_bulk(BdrvCopyState *s)
+{
+    int nr_sectors;
+    int64_t curr_sector = s->curr_sector;
+
+    if (s->shared_base) {
+        while (curr_sector<  s->nr_sectors&&
+                !bdrv_is_allocated(s->src, curr_sector,
+                                   MAX_IS_ALLOCATED_SEARCH,&nr_sectors)) {
+                curr_sector += nr_sectors;
+        }
+    }
+
+    if (curr_sector>= s->nr_sectors) {
+        s->curr_sector = 0;
+        return 1;
+    }
+
+    curr_sector&= ~((int64_t)BDRV_SECTORS_PER_DIRTY_CHUNK - 1);
+    nr_sectors = BDRV_SECTORS_PER_DIRTY_CHUNK;
+
+    blk_copy_issue_read(s, s->curr_sector, nr_sectors);
+    s->curr_sector += nr_sectors;
+    s->completed_sectors = curr_sector;
+    return 0;
+}
+
+static void blkcopy_finish(BdrvCopyState *s)
+{
+    int64_t sector;
+    uint8_t *buf;
+
+    buf = qemu_malloc(BLOCK_SIZE);
+
+    /* FIXME: speed up loop, get_next_dirty_block? */
+    for (sector = 0; sector<  s->nr_sectors;
+         sector += BDRV_SECTORS_PER_DIRTY_CHUNK) {
+        if (bdrv_get_dirty(s->src, sector)) {
+            int nr_sectors = MIN(s->nr_sectors - sector,
+                                 BDRV_SECTORS_PER_DIRTY_CHUNK);
+
+            memset(buf, 0, BLOCK_SIZE);
+            if (bdrv_read(s->src, sector, buf, nr_sectors)<  0) {
+                goto error;
+            }
+            if (bdrv_write(s->dst, sector, buf, nr_sectors)<  0) {
+                goto error;
+            }
+            bdrv_reset_dirty(s->src, sector, nr_sectors);
+        }
+
+        if (bdrv_get_dirty_count(s->src) == 0)
+            break;
+    }
+    qemu_free(buf);
+    return;
+
+error:
+    qemu_free(buf);
+    s->error = 1;
+}
+
+static int write_commit_file(BdrvCopyState *s)
+{
+    char commit_msg[1400];
+    const char *buf = commit_msg;
+    int len, ret;
+
+    sprintf(commit_msg, "commit QEMU block_copy %s ->  %s\n", s->src_device_name,
+                        s->dst_filename);
+
+    len = strlen(commit_msg);
+    while (len>  0) {
+        ret = write(s->commit_fd, buf, len);
+        if (ret == -1&&  errno == EINTR) {
+            continue;
+        }

Quoted 21 lines

block-copy.c new file +21 -0

+static int write_commit_file(BdrvCopyState *s)
+{
+    char commit_msg[1400];
+    const char *buf = commit_msg;
+    int len, ret;
+
+    sprintf(commit_msg, "commit QEMU block_copy %s ->  %s\n", s->src_device_name,
+                        s->dst_filename);
+
+    len = strlen(commit_msg);
+    while (len>  0) {
+        ret = write(s->commit_fd, buf, len);
+        if (ret == -1&&  errno == EINTR) {
+            continue;
+        }
+        if (ret<= 0) {
+            return -errno;
+        }
+        buf += ret;
+        len -= ret;
+    }

block-copy.c line 410

Quoted 5 lines

block-copy.c new file +4 -0

+
+    if (fsync(s->commit_fd) == -1) {
+        return -errno;
+    }

This is more or less black magic. What is this commit file used for and why aren't we using something like a QMP event?

Quoted 21 lines · Anthony Liguori

Quoted 21 lines

block-copy.c new file +21 -0

+static int write_commit_file(BdrvCopyState *s)
+{
+    char commit_msg[1400];
+    const char *buf = commit_msg;
+    int len, ret;
+
+    sprintf(commit_msg, "commit QEMU block_copy %s ->   %s\n", s->src_device_name,
+                        s->dst_filename);
+
+    len = strlen(commit_msg);
+    while (len>   0) {
+        ret = write(s->commit_fd, buf, len);
+        if (ret == -1&&   errno == EINTR) {
+            continue;
+        }
+        if (ret<= 0) {
+            return -errno;
+        }
+        buf += ret;
+        len -= ret;
+    }

Quoted 9 lines

block-copy.c line 410

Quoted 6 lines

block-copy.c new file +4 -0

+
+    if (fsync(s->commit_fd) == -1) {
+        return -errno;
+    }

This is more or less black magic. What is this commit file used for and why aren't we using something like a QMP event?

The commit file is considered reliable storage for the result of image switch operation. Think of the following scenario:

- mgmt app requests live copy of guests ide1-hd0 from /a/image.img to /b/image.img.
- mgmt app dies.
- guest switches to /b/image.img, /a/image.img is outdated.
- guest dies.

Notifying the switch via QMP would not be reliable in this case.

Will add this as a comment in the code.

Quoted 38 lines · Marcelo Tosatti

Quoted 20 lines · Anthony Liguori

Quoted 20 lines

block-copy.c new file +20 -0

+static int write_commit_file(BdrvCopyState *s)
+{
+    char commit_msg[1400];
+    const char *buf = commit_msg;
+    int len, ret;
+
+    sprintf(commit_msg, "commit QEMU block_copy %s ->   %s\n", s->src_device_name,
+                        s->dst_filename);
+
+    len = strlen(commit_msg);
+    while (len>   0) {
+        ret = write(s->commit_fd, buf, len);
+        if (ret == -1&&   errno == EINTR) {
+            continue;
+        }
+        if (ret<= 0) {
+            return -errno;
+        }
+        buf += ret;
+        len -= ret;

Quoted 8 lines

block-copy.c line 410

Quoted 6 lines

block-copy.c new file +5 -0

+    }
+
+    if (fsync(s->commit_fd) == -1) {
+        return -errno;
+    }

This is more or less black magic. What is this commit file used for and why aren't we using something like a QMP event?

The commit file is considered reliable storage for the result of image switch operation. Think of the following scenario:

- mgmt app requests live copy of guests ide1-hd0 from /a/image.img to /b/image.img.
- mgmt app dies.
- guest switches to /b/image.img, /a/image.img is outdated.
- guest dies.

Quoted 1 line

Notifying the switch via QMP would not be reliable in this case.

But this is true of any number of operations in QEMU such as hotplug where if a management tool dies after requesting hotplug and then the guest dies before the management tool reconnects, it's never known whether it's truly connected or not.

The only way to robustly solve this is for QEMU to maintain a stateful config file. A one-off for this particular command doesn't seem wise to me.

Quoted 45 lines · Anthony Liguori

Quoted 43 lines · Marcelo Tosatti

Quoted 21 lines · Anthony Liguori

Quoted 21 lines

block-copy.c new file +21 -0

+static int write_commit_file(BdrvCopyState *s)
+{
+    char commit_msg[1400];
+    const char *buf = commit_msg;
+    int len, ret;
+
+    sprintf(commit_msg, "commit QEMU block_copy %s ->    %s\n", s->src_device_name,
+                        s->dst_filename);
+
+    len = strlen(commit_msg);
+    while (len>    0) {
+        ret = write(s->commit_fd, buf, len);
+        if (ret == -1&&    errno == EINTR) {
+            continue;
+        }
+        if (ret<= 0) {
+            return -errno;
+        }
+        buf += ret;
+        len -= ret;
+    }

Quoted 9 lines

block-copy.c line 410

Quoted 6 lines

block-copy.c new file +4 -0

+
+    if (fsync(s->commit_fd) == -1) {
+        return -errno;
+    }

This is more or less black magic. What is this commit file used for and why aren't we using something like a QMP event?

The commit file is considered reliable storage for the result of image switch operation. Think of the following scenario:

- mgmt app requests live copy of guests ide1-hd0

from /a/image.img to /b/image.img.

- mgmt app dies.
- guest switches to /b/image.img, /a/image.img is outdated.
- guest dies.

Quoted 7 lines

Quoted 2 lines

Notifying the switch via QMP would not be reliable in this case.

But this is true of any number of operations in QEMU such as hotplug where if a management tool dies after requesting hotplug and then the guest dies before the management tool reconnects, it's never known whether it's truly connected or not.

This is a different case. The mgmt tool should be able to safely store the desired device tree before hotplugging a device.

And even if you consider that should be done by QEMU, in a config file, restarting the guest with or without the device is not going to result in data corruption/loss.

Quoted 4 lines

The only way to robustly solve this is for QEMU to maintain a stateful config file. A one-off for this particular command doesn't seem wise to me.

I don't disagree the commit message could be in a config file, but since that does not exist, a separate file is a workable solution.

Also, the
separate commit file is not incompatible with future improvements.

-drive already ties into the qemuopts infrastructure and we have
readconfig and writeconfig. I don't think we're missing any major pieces to do this in a more proper fashion.

Quoted 3 lines · Anthony Liguori

-drive already ties into the qemuopts infrastructure and we have
readconfig and writeconfig. I don't think we're missing any major pieces to do this in a more proper fashion.

The problem with qemu config files is that it splits the authoritative source of where images are stored into two. Is it in the management tool's database or is it in qemu's config file?

For the problem at hand, one solution is to make qemu stop after the copy, and then management can issue an additional command to rearrange the disk and resume the guest. A drawback here is that if management dies, the guest is stopped until it restarts. We also make management latency guest visible, even if it doesn't die at an inconvenient place.

An alternative approach is to have the copy be performed by a new layered block format driver:

- create a new image, type = live-copy, containing three pieces of information
- source image
- destination image
- copy state (initially nothing is copied)
- tell qemu switch to the new image
- qemu starts copying, updates copy state as needed
- copy finishes, event is emitted; reads and writes still serviced
- management receives event, switches qemu to destination image
- management removes live-copy image

If management dies while this is happening, it can simply query the state of the copy. Similarly, if qemu dies, the copy state is persistent (could be 0/1 or real range of blocks).

Quoted 6 lines · Avi Kivity

Quoted 3 lines · Anthony Liguori

-drive already ties into the qemuopts infrastructure and we have
readconfig and writeconfig. I don't think we're missing any major pieces to do this in a more proper fashion.

The problem with qemu config files is that it splits the authoritative source of where images are stored into two. Is it in the management tool's database or is it in qemu's config file?

I like to use the phrase "stateful config file". To me, it's just a database for QEMU to persist data about the VM. It's the only way for QEMU to make certain transactions atomic in the face of QEMU crashing.

The user visible config file is a totally different concept. A management tool launches QEMU and tells it where to keep it's state database. The management application may prepopulate the state database or it may just use an empty file.

QEMU uses the state database to store information that is created dynamically. For instance, devices added through device_add. A device added via -device wouldn't necessary get added to the state database.

Practically speaking, it let's you invoke QEMU with a fixed command line, while still using the monitor to make changes that would otherwise require the command line being updated.

Quoted 15 lines

For the problem at hand, one solution is to make qemu stop after the copy, and then management can issue an additional command to rearrange the disk and resume the guest. A drawback here is that if management dies, the guest is stopped until it restarts. We also make management latency guest visible, even if it doesn't die at an inconvenient place.

An alternative approach is to have the copy be performed by a new layered block format driver:

- create a new image, type = live-copy, containing three pieces of information
- source image
- destination image
- copy state (initially nothing is copied)
- tell qemu switch to the new image
- qemu starts copying, updates copy state as needed
- copy finishes, event is emitted; reads and writes still serviced
- management receives event, switches qemu to destination image
- management removes live-copy image

Quoted 1 line

If management dies while this is happening, it can simply query the state of the copy. Similarly, if qemu dies, the copy state is persistent (could be 0/1 or real range of blocks).

This is a more elegant solution to the problem than the commit problem but it's also a one-off. I think we have a generic problem here and we ought to try to solve it generically (within reason).

Quoted 6 lines · Anthony Liguori

Quoted 5 lines · Avi Kivity

Quoted 3 lines · Anthony Liguori

-drive already ties into the qemuopts infrastructure and we have
readconfig and writeconfig. I don't think we're missing any major pieces to do this in a more proper fashion.

Quoted 5 lines

Quoted 1 line

The problem with qemu config files is that it splits the authoritative source of where images are stored into two. Is it in the management tool's database or is it in qemu's config file?

I like to use the phrase "stateful config file". To me, it's just a database for QEMU to persist data about the VM. It's the only way for QEMU to make certain transactions atomic in the face of QEMU crashing.

The user visible config file is a totally different concept. A management tool launches QEMU and tells it where to keep it's state database. The management application may prepopulate the state database or it may just use an empty file.

In that case the word 'config' is misleading. To me, it implies that the user configures something, and qemu reads it, not something mostly internal to qemu.

Qemu does keep state. Currently only images, but in theory also the on-board NVRAM.

Quoted 4 lines

QEMU uses the state database to store information that is created dynamically. For instance, devices added through device_add. A device added via -device wouldn't necessary get added to the state database.

Practically speaking, it let's you invoke QEMU with a fixed command line, while still using the monitor to make changes that would otherwise require the command line being updated.

Then the invoker quickly loses track of what the actual state is. It can't just remember which commands it issued (presumably in response to the user updating user visible state). It has to parse the stateful config file qemu outputs.

But at which points should it parse it?

I don't think it's reasonable to have three different ways to interact with qemu, all needed: the command line, reading and writing the stateful config file, and the monitor. I'd rather push for starting qemu with a blank guest and assembling (cold-plugging) all the hardware via the monitor before starting the guest.

Quoted 15 lines

Quoted 15 lines

For the problem at hand, one solution is to make qemu stop after the copy, and then management can issue an additional command to rearrange the disk and resume the guest. A drawback here is that if management dies, the guest is stopped until it restarts. We also make management latency guest visible, even if it doesn't die at an inconvenient place.

An alternative approach is to have the copy be performed by a new layered block format driver:

- create a new image, type = live-copy, containing three pieces of information
- source image
- destination image
- copy state (initially nothing is copied)
- tell qemu switch to the new image
- qemu starts copying, updates copy state as needed
- copy finishes, event is emitted; reads and writes still serviced
- management receives event, switches qemu to destination image
- management removes live-copy image

Quoted 3 lines

Quoted 1 line

If management dies while this is happening, it can simply query the state of the copy. Similarly, if qemu dies, the copy state is persistent (could be 0/1 or real range of blocks).

This is a more elegant solution to the problem than the commit problem but it's also a one-off. I think we have a generic problem here and we ought to try to solve it generically (within reason).

Can you give more examples?

I think I demonstrated that hot-plug can be solved via the existing interfaces.

Quoted 3 lines

Qemu does keep state. Currently only images, but in theory also the on-board NVRAM.

Yeah, this is a good example of an area where a "stateful config file" would be useful. I like the idea of storing this sort of thing in a text file with a config structure because a user certainly wants to be able to specify the boot order. Being able to tweak this kind of stuff adds a lot of interesting capabilities.

Quoted 7 lines

Quoted 5 lines

QEMU uses the state database to store information that is created dynamically. For instance, devices added through device_add. A device added via -device wouldn't necessary get added to the state database.

Practically speaking, it let's you invoke QEMU with a fixed command line, while still using the monitor to make changes that would otherwise require the command line being updated.

Then the invoker quickly loses track of what the actual state is. It can't just remember which commands it issued (presumably in response to the user updating user visible state). It has to parse the stateful config file qemu outputs.

Well specifically, it has to ask QEMU and QEMU can tell it the current state via a nice structured data format over QMP. It's a hell of a lot easier than the management tool trying to do this outside of QEMU.

Quoted 2 lines

But at which points should it parse it?

I was thinking that we should post events whenever we change the stateful config. That would let the management tool have a mechanism for determining when settings have been changed. Of course, if the management tool crashes, it should re-read at startup.

Quoted 1 line

I don't think it's reasonable to have three different ways to interact with qemu, all needed: the command line, reading and writing the stateful config file, and the monitor. I'd rather push for starting qemu with a blank guest and assembling (cold-plugging) all the hardware via the monitor before starting the guest.

Yes. I view the command line as optional. To me, this is the ideal interaction:

1) start qemu with an empty stateful config file

2) issue monitor commands to create all devices and backends

3) the stateful config file totally captures the state of all of the issued QMP commands. The management tool can relaunch the guest just by passing the stateful config file to QEMU.

4) when the management tool needs to "extract" a config file, it can read the stateful config (through the monitor) and generate it's own config.

5) the management tool should treat the stateful config file as more or less opaque. It shouldn't be visible to end user.

In the non-managed case, users should interact directly with the config file.

Quoted 3 lines

Can you give more examples?

I think I demonstrated that hot-plug can be solved via the existing interfaces.

Sure. CMOS settings right now are not persisted across reboot.

Guest initiated activities like IDE or PCI eject are tricky to persist correctly within a management tool.

We could add events for all of this things but it's all racy since events are posted. If we have a stateful config file, we can make all of these things non-racy and post an event that the config has changed. If there's a crash, the management tool can read the config on startup to catch up on missed events.

I think the nature of a posted event management interface is such that we need a stateful config that persists across QEMU invocations.

Quoted 5 lines · Anthony Liguori

Quoted 3 lines

Qemu does keep state. Currently only images, but in theory also the on-board NVRAM.

Yeah, this is a good example of an area where a "stateful config file" would be useful. I like the idea of storing this sort of thing in a text file with a config structure because a user certainly wants to be able to specify the boot order. Being able to tweak this kind of stuff adds a lot of interesting capabilities.

My preference would be a binary file (a disk image, in fact), with a tool to play with the known fields. It allows a management tool to reuse its storage stack.

Quoted 1 line

Well specifically, it has to ask QEMU and QEMU can tell it the current state via a nice structured data format over QMP. It's a hell of a lot easier than the management tool trying to do this outside of QEMU.

So, if qemu crashes, the management tool has to start it up to find out what the current state is.

Quoted 15 lines

Quoted 2 lines

But at which points should it parse it?

I was thinking that we should post events whenever we change the stateful config. That would let the management tool have a mechanism for determining when settings have been changed. Of course, if the management tool crashes, it should re-read at startup.

Quoted 1 line

I don't think it's reasonable to have three different ways to interact with qemu, all needed: the command line, reading and writing the stateful config file, and the monitor. I'd rather push for starting qemu with a blank guest and assembling (cold-plugging) all the hardware via the monitor before starting the guest.

Yes. I view the command line as optional. To me, this is the ideal interaction:

1) start qemu with an empty stateful config file

2) issue monitor commands to create all devices and backends

3) the stateful config file totally captures the state of all of the issued QMP commands. The management tool can relaunch the guest just by passing the stateful config file to QEMU.

Quoted 5 lines

4) when the management tool needs to "extract" a config file, it can read the stateful config (through the monitor) and generate it's own config.

5) the management tool should treat the stateful config file as more or less opaque. It shouldn't be visible to end user.

In the non-managed case, users should interact directly with the config file.

Doesn't the stateful non-config file becomes a failure point? It has to be on shared and redundant storage?

To me, it seems a lot easier to require management to replay any commands that hadn't been acknowledged (due to management failure), or to query qemu as to its current state (if it is alive).

Management already needs stable, redundant config storage anyway (often a database).

Quoted 5 lines

Quoted 3 lines

Can you give more examples?

I think I demonstrated that hot-plug can be solved via the existing interfaces.

Sure. CMOS settings right now are not persisted across reboot.

That is handled easily with an NVRAM disk image.

Quoted 3 lines

Guest initiated activities like IDE or PCI eject are tricky to persist correctly within a management tool.

We could add events for all of this things but it's all racy since events are posted. If we have a stateful config file, we can make all of these things non-racy and post an event that the config has changed. If there's a crash, the management tool can read the config on startup to catch up on missed events.

If qemu crashes, these events are meaningless. If management crashes, it has to query qemu for all state that it wants to keep track of via events.

Quoted 2 lines

I think the nature of a posted event management interface is such that we need a stateful config that persists across QEMU invocations.

I'm not convinced, and I think making qemu manage even more state creates more problems.

Quoted 4 lines

Quoted 2 lines

Well specifically, it has to ask QEMU and QEMU can tell it the current state via a nice structured data format over QMP. It's a hell of a lot easier than the management tool trying to do this outside of QEMU.

So, if qemu crashes, the management tool has to start it up to find out what the current state is.

Depends on how opaque we make the state file. I've been thinking a simple ini syntax with a well supported set of keys. In that case, a management tool can read it without starting QEMU.

Quoted 2 lines

Doesn't the stateful non-config file becomes a failure point? It has to be on shared and redundant storage?

It depends on what your availability model is and how frequently your management tool backs up the config. As of right now, we have a pretty glaring reliability hole here so adding a stateful "non-config" can only improve things.

Quoted 2 lines

To me, it seems a lot easier to require management to replay any commands that hadn't been acknowledged (due to management failure), or to query qemu as to its current state (if it is alive).

You still have the race condition around guest initiated events like eject. Unless you have an acknowledged event from a management tool (which we can't do in QMP today) whereas you don't complete the guest initiated eject operation until management ack's it, we need to store that state ourself.

I don't like the idea of making a management tool such an integral part of the functional paths.

Not having a stateful config file also means that this problem isn't solved in any form without a really sophisticated management stack. I'm a big fan of being robust in the face of not-so sophisticated management tools.

Quoted 1 line

If qemu crashes, these events are meaningless. If management crashes, it has to query qemu for all state that it wants to keep track of via events.

Think power failure, not qemu crash. In the event of a power failure, any hardware change initiated by the guest ought to be consistent with when the guest has restarted. If you eject the CDROM tray and then lose power, its still ejected after the power comes back on.

Quoted 5 lines

Quoted 3 lines

I think the nature of a posted event management interface is such that we need a stateful config that persists across QEMU invocations.

I'm not convinced, and I think making qemu manage even more state creates more problems.

Well this patch series is making qemu management more state. The only question is whether we do this as a one-off mechanism or whether we architect a general mechanism to do it.

How much state we store can always be up for discussion but I think it's undeniable that we need to store more state than we're storing today (none).

Quoted 6 lines · Anthony Liguori

Quoted 4 lines

Quoted 2 lines

Well specifically, it has to ask QEMU and QEMU can tell it the current state via a nice structured data format over QMP. It's a hell of a lot easier than the management tool trying to do this outside of QEMU.

So, if qemu crashes, the management tool has to start it up to find out what the current state is.

Depends on how opaque we make the state file. I've been thinking a simple ini syntax with a well supported set of keys. In that case, a management tool can read it without starting QEMU.

Then the management stack has to worry about yet another way of interacting via qemu.

I'd like to limit it to the monitor.

Quoted 4 lines

Quoted 2 lines

Doesn't the stateful non-config file becomes a failure point? It has to be on shared and redundant storage?

It depends on what your availability model is and how frequently your management tool backs up the config. As of right now, we have a pretty glaring reliability hole here so adding a stateful "non-config" can only improve things.

I think the solutions I pointed out close the hole with the existing interfaces.

Quoted 4 lines

Quoted 2 lines

To me, it seems a lot easier to require management to replay any commands that hadn't been acknowledged (due to management failure), or to query qemu as to its current state (if it is alive).

You still have the race condition around guest initiated events like eject. Unless you have an acknowledged event from a management tool (which we can't do in QMP today) whereas you don't complete the guest initiated eject operation until management ack's it, we need to store that state ourself.

I don't see why.

If management crashes, it queries the eject state when it reconnects to qemu.
If qemu crashes, the eject state is lost, but that is fine. My CD-ROM drive tray pulls itself in when the machine is started.

Quoted 2 lines

I don't like the idea of making a management tool such an integral part of the functional paths.

I agree that we don't want qemu to wait on the management stack any more than necessary.

Quoted 1 line

Not having a stateful config file also means that this problem isn't solved in any form without a really sophisticated management stack. I'm a big fan of being robust in the face of not-so sophisticated management tools.

You're introducing the need for additional code in the management layer, the care and feeding for the stateful non-config file.

Quoted 3 lines

Quoted 1 line

If qemu crashes, these events are meaningless. If management crashes, it has to query qemu for all state that it wants to keep track of via events.

Think power failure, not qemu crash. In the event of a power failure, any hardware change initiated by the guest ought to be consistent with when the guest has restarted. If you eject the CDROM tray and then lose power, its still ejected after the power comes back on.

Not on all machines.

Let's list guest state which is independent of power. That would be wither NVRAM of various types, or physical alterations. CD-ROM eject is one. Are there others?

Quoted 9 lines

Quoted 5 lines

Quoted 3 lines

I think the nature of a posted event management interface is such that we need a stateful config that persists across QEMU invocations.

I'm not convinced, and I think making qemu manage even more state creates more problems.

Well this patch series is making qemu management more state. The only question is whether we do this as a one-off mechanism or whether we architect a general mechanism to do it.

How much state we store can always be up for discussion but I think it's undeniable that we need to store more state than we're storing today (none).

I think my solution (multiplexing block format driver) fits the requirements for live-copy perfectly. In fact it has a name - it's a RAID-1 driver started in degraded mode. It could be useful other use cases.

Quoted 10 lines · Avi Kivity

Quoted 9 lines · Anthony Liguori

Quoted 8 lines

Quoted 5 lines

Well specifically, it has to ask QEMU and QEMU can tell it the current state via a nice structured data format over QMP. It's a hell of a lot easier than the management tool trying to do this outside of QEMU.

So, if qemu crashes, the management tool has to start it up to find out what the current state is.

Quoted 7 lines

Quoted 4 lines

Depends on how opaque we make the state file. I've been thinking a simple ini syntax with a well supported set of keys. In that case, a management tool can read it without starting QEMU.

Then the management stack has to worry about yet another way of interacting via qemu.

{ 'StateItem': { 'key': 'str', 'value': 'str' } } { 'StateSection': { 'kind': 'str', 'name': 'str', 'items': [ 'StateItem' ] } }
{ 'StateInfo': { 'sections': [ 'StateSection' ] } }

{ 'query-state', {}, {}, 'StateInfo' }

A management tool never need to worry about anything other than this command if it so chooses. If we have the pre-machine init mode for 0.16, then this can even be used to inspect state without running a guest.

The fact that the state is visible in the filesystem is an implementation detail.

Quoted 6 lines

I'd like to limit it to the monitor.

Quoted 4 lines

Quoted 3 lines

Doesn't the stateful non-config file becomes a failure point? It has to be on shared and redundant storage?

Quoted 7 lines

Quoted 4 lines

It depends on what your availability model is and how frequently your management tool backs up the config. As of right now, we have a pretty glaring reliability hole here so adding a stateful "non-config" can only improve things.

I think the solutions I pointed out close the hole with the existing interfaces.

It doesn't work for eject unless you interpose an acknowledged event. Ultimately, this is a simple problem. If you want reliability, we either need symmetric RPCs so that the device model can call (and wait) to the management layer to acknowledge a change or QEMU can post an event to the management layer, and maintain the state in a reliable fashion.

Quoted 12 lines

Quoted 11 lines

Quoted 5 lines

To me, it seems a lot easier to require management to replay any commands that hadn't been acknowledged (due to management failure), or to query qemu as to its current state (if it is alive).

You still have the race condition around guest initiated events like eject. Unless you have an acknowledged event from a management tool (which we can't do in QMP today) whereas you don't complete the guest initiated eject operation until management ack's it, we need to store that state ourself.

Quoted 6 lines

I don't see why.

If management crashes, it queries the eject state when it reconnects to qemu.
If qemu crashes, the eject state is lost, but that is fine. My CD-ROM drive tray pulls itself in when the machine is started.

Pick any of a number of possible events that change the machine's state. We can wave our hands at some things saying they don't matter and do one off solutions for others, or we can just have a robust way of handling this consistently.

Quoted 7 lines

Quoted 3 lines

I don't like the idea of making a management tool such an integral part of the functional paths.

I agree that we don't want qemu to wait on the management stack any more than necessary.

Quoted 7 lines

Quoted 4 lines

Not having a stateful config file also means that this problem isn't solved in any form without a really sophisticated management stack. I'm a big fan of being robust in the face of not-so sophisticated management tools.

You're introducing the need for additional code in the management layer, the care and feeding for the stateful non-config file.

If a management layer ignores the stateful non-config file, as you like to call it, it'll get the same semantics it has today. I think managing a single thing is a whole lot easier than managing an NVRAM file, a block migration layering file, and all of the future things we're going to add once we decide they are important too.

Quoted 10 lines

Quoted 9 lines

Quoted 3 lines

If qemu crashes, these events are meaningless. If management crashes, it has to query qemu for all state that it wants to keep track of via events.

Think power failure, not qemu crash. In the event of a power failure, any hardware change initiated by the guest ought to be consistent with when the guest has restarted. If you eject the CDROM tray and then lose power, its still ejected after the power comes back on.

Quoted 5 lines

Not on all machines.

Let's list guest state which is independent of power. That would be wither NVRAM of various types, or physical alterations. CD-ROM eject is one. Are there others?

Any indirect qemu state. Block migration is an example, but other examples would be VNC server information (like current password), WCE setting (depending on whether we modelled eeprom for the drivers), and persisted device settings (lots of devices have eeprom these days).

Quoted 7 lines

Any indirect qemu state. Block migration is an example, but other examples would be VNC server information (like current password), WCE setting (depending on whether we modelled eeprom for the drivers), and persisted device settings (lots of devices have eeprom these days).

Agreed.

Why a separate location for this "stateful non-config" section, however?

The state in question (backing image property, device presence, VNC info, etc) is already represented in either host or guest configuration, so why not simply expose that?

Quoted 10 lines · Marcelo Tosatti

Quoted 6 lines

Any indirect qemu state. Block migration is an example, but other examples would be VNC server information (like current password), WCE setting (depending on whether we modelled eeprom for the drivers), and persisted device settings (lots of devices have eeprom these days).

Agreed.

Why a separate location for this "stateful non-config" section, however?

Quoted 3 lines

The state in question (backing image property, device presence, VNC info, etc) is already represented in either host or guest configuration, so why not simply expose that?

Hrm, are you suggesting that the "stateful non-config" be hidden directly from the user such that only existing monitor interfaces are used to query it's contents?

I'll have to think about that a bit. Obviously, the existing commands would still be authoritative even with a query-state command. I think the only question is whether there's value in making this totally opaque.

Regards,

Anthony Liguori

Quoted 10 lines · Avi Kivity

Quoted 9 lines · Anthony Liguori

Quoted 8 lines

Quoted 5 lines

Well specifically, it has to ask QEMU and QEMU can tell it the current state via a nice structured data format over QMP. It's a hell of a lot easier than the management tool trying to do this outside of QEMU.

So, if qemu crashes, the management tool has to start it up to find out what the current state is.

Quoted 6 lines

Quoted 3 lines

Depends on how opaque we make the state file. I've been thinking a simple ini syntax with a well supported set of keys. In that case, a management tool can read it without starting QEMU.

Then the management stack has to worry about yet another way of interacting via qemu.

{ 'StateItem': { 'key': 'str', 'value': 'str' } } { 'StateSection': { 'kind': 'str', 'name': 'str', 'items': [ StateItem' ] } }
{ 'StateInfo': { 'sections': [ 'StateSection' ] } }

{ 'query-state', {}, {}, 'StateInfo' }

A management tool never need to worry about anything other than this command if it so chooses. If we have the pre-machine init mode for 0.16, then this can even be used to inspect state without running a guest.

The fact that the state is visible in the filesystem is an implementation detail.

Quoted 6 lines

I'd like to limit it to the monitor.

Quoted 4 lines

Quoted 3 lines

Doesn't the stateful non-config file becomes a failure point? It has to be on shared and redundant storage?

Quoted 7 lines

Quoted 4 lines

It depends on what your availability model is and how frequently your management tool backs up the config. As of right now, we have a pretty glaring reliability hole here so adding a stateful "non-config" can only improve things.

I think the solutions I pointed out close the hole with the existing interfaces.

It doesn't work for eject unless you interpose an acknowledged event. Ultimately, this is a simple problem. If you want reliability, we either need symmetric RPCs so that the device model can call (and wait) to the management layer to acknowledge a change or QEMU can post an event to the management layer, and maintain the state in a reliable fashion.

Quoted 12 lines

Quoted 11 lines

Quoted 5 lines

To me, it seems a lot easier to require management to replay any commands that hadn't been acknowledged (due to management failure), or to query qemu as to its current state (if it is alive).

You still have the race condition around guest initiated events like eject. Unless you have an acknowledged event from a management tool (which we can't do in QMP today) whereas you don't complete the guest initiated eject operation until management ack's it, we need to store that state ourself.

Quoted 6 lines

I don't see why.

If management crashes, it queries the eject state when it reconnects to qemu.
If qemu crashes, the eject state is lost, but that is fine. My CD-ROM drive tray pulls itself in when the machine is started.

Pick any of a number of possible events that change the machine's state. We can wave our hands at some things saying they don't matter and do one off solutions for others, or we can just have a robust way of handling this consistently.

Quoted 1 line

Then the management stack has to worry about yet another way of interacting via qemu.

{ 'StateItem': { 'key': 'str', 'value': 'str' } }
{ 'StateSection': { 'kind': 'str', 'name': 'str', 'items': [ 'StateItem' ] } } { 'StateInfo': { 'sections': [ 'StateSection' ] } }

{ 'query-state', {}, {}, 'StateInfo' }

A management tool never need to worry about anything other than this command if it so chooses. If we have the pre-machine init mode for 0.16, then this can even be used to inspect state without running a guest.

The fact that the state is visible in the filesystem is an implementation detail.

Quoted 9 lines

I'd like to limit it to the monitor.

Quoted 4 lines

Quoted 2 lines

Doesn't the stateful non-config file becomes a failure point? It has to be on shared and redundant storage?

It depends on what your availability model is and how frequently your management tool backs up the config. As of right now, we have a pretty glaring reliability hole here so adding a stateful "non-config" can only improve things.

I think the solutions I pointed out close the hole with the existing interfaces.

It doesn't work for eject unless you interpose an acknowledged event. Ultimately, this is a simple problem. If you want reliability, we either need symmetric RPCs so that the device model can call (and wait) to the management layer to acknowledge a change or QEMU can post an event to the management layer, and maintain the state in a reliable fashion.

Quoted 6 lines

Quoted 1 line

You still have the race condition around guest initiated events like eject. Unless you have an acknowledged event from a management tool (which we can't do in QMP today) whereas you don't complete the guest initiated eject operation until management ack's it, we need to store that state ourself.

I don't see why.

If management crashes, it queries the eject state when it reconnects to qemu.
If qemu crashes, the eject state is lost, but that is fine. My CD-ROM drive tray pulls itself in when the machine is started.

Pick any of a number of possible events that change the machine's state. We can wave our hands at some things saying they don't matter and do one off solutions for others, or we can just have a robust way of handling this consistently.

Quoted 2 lines

You're introducing the need for additional code in the management layer, the care and feeding for the stateful non-config file.

If a management layer ignores the stateful non-config file, as you like to call it, it'll get the same semantics it has today. I think managing a single thing is a whole lot easier than managing an NVRAM file, a block migration layering file, and all of the future things we're going to add once we decide they are important too.

Quoted 8 lines

Quoted 4 lines

Quoted 2 lines

If qemu crashes, these events are meaningless. If management crashes, it has to query qemu for all state that it wants to keep track of via events.

Think power failure, not qemu crash. In the event of a power failure, any hardware change initiated by the guest ought to be consistent with when the guest has restarted. If you eject the CDROM tray and then lose power, its still ejected after the power comes back on.

Not on all machines.

Let's list guest state which is independent of power. That would be wither NVRAM of various types, or physical alterations. CD-ROM eject is one. Are there others?

Any indirect qemu state. Block migration is an example, but other examples would be VNC server information (like current password), WCE setting (depending on whether we modelled eeprom for the drivers), and persisted device settings (lots of devices have eeprom these days).

Quoted 1 line

I think my solution (multiplexing block format driver) fits the requirements for live-copy perfectly. In fact it has a name - it's a RAID-1 driver started in degraded mode. It could be useful other use cases.

It feels a bit awkward to me to be honest.

Quoted 7 lines · Anthony Liguori

Quoted 1 line

Then the management stack has to worry about yet another way of interacting via qemu.

{ 'StateItem': { 'key': 'str', 'value': 'str' } }
{ 'StateSection': { 'kind': 'str', 'name': 'str', 'items': [ 'StateItem' ] } } { 'StateInfo': { 'sections': [ 'StateSection' ] } }

Quoted 3 lines

{ 'query-state', {}, {}, 'StateInfo' }

A management tool never need to worry about anything other than this command if it so chooses. If we have the pre-machine init mode for 0.16, then this can even be used to inspect state without running a guest.

So we have yet another information tree. If we store the cd-rom eject state here, then we need to make an association between the device path of the cd-rom, and the StateItem key.

Far better to store it in the device itself. For example, we could make a layered block format driver that stores the eject state and a "backing file" containing the actual media. Eject and media change would be recorded in the block format driver's state. You could then hot-unplug a USB cd-writer and hot-plug it back into a different guest, implementing a virtual sneakernet.

Quoted 2 lines

The fact that the state is visible in the filesystem is an implementation detail.

A detail that has to be catered for by the management stack - it has to provide a safe place for it, back it up, etc.

Quoted 1 line

It doesn't work for eject unless you interpose an acknowledged event. Ultimately, this is a simple problem. If you want reliability, we either need symmetric RPCs so that the device model can call (and wait) to the management layer to acknowledge a change or QEMU can post an event to the management layer, and maintain the state in a reliable fashion.

I don't see why it doesn't work. Please explain.

Quoted 8 lines

Quoted 6 lines

Quoted 1 line

You still have the race condition around guest initiated events like eject. Unless you have an acknowledged event from a management tool (which we can't do in QMP today) whereas you don't complete the guest initiated eject operation until management ack's it, we need to store that state ourself.

I don't see why.

If management crashes, it queries the eject state when it reconnects to qemu.
If qemu crashes, the eject state is lost, but that is fine. My CD-ROM drive tray pulls itself in when the machine is started.

Pick any of a number of possible events that change the machine's state. We can wave our hands at some things saying they don't matter and do one off solutions for others, or we can just have a robust way of handling this consistently.

Both block live copy and cd-rom eject state can be solved with layered block format drivers. I don't think a central place for random data makes sense. State belongs near the device that maintains it, esp. if the device is hot-pluggable, so it's easy to associate the state with the device.

Quoted 4 lines

Quoted 2 lines

You're introducing the need for additional code in the management layer, the care and feeding for the stateful non-config file.

If a management layer ignores the stateful non-config file, as you like to call it, it'll get the same semantics it has today. I think managing a single thing is a whole lot easier than managing an NVRAM file, a block migration layering file, and all of the future things we're going to add once we decide they are important too.

I disagree. Storing NVRAM as a disk image is a simple extension of existing management tools. Block live-copy and cd-rom eject state also make sense as per-image state if you take hotunplug and hotplug into account.

Quoted 7 lines

Quoted 7 lines

Quoted 4 lines

Quoted 2 lines

If qemu crashes, these events are meaningless. If management crashes, it has to query qemu for all state that it wants to keep track of via events.

Think power failure, not qemu crash. In the event of a power failure, any hardware change initiated by the guest ought to be consistent with when the guest has restarted. If you eject the CDROM tray and then lose power, its still ejected after the power comes back on.

Not on all machines.

Quoted 3 lines

Quoted 1 line

Let's list guest state which is independent of power. That would be wither NVRAM of various types, or physical alterations. CD-ROM eject is one. Are there others?

Any indirect qemu state. Block migration is an example, but other examples would be VNC server information (like current password), WCE setting (depending on whether we modelled eeprom for the drivers), and persisted device settings (lots of devices have eeprom these days).

Device settings should be stored with the devices, not with qemu.

Suppose we take the cold-plug on startup via the monitor approach. So we start with a bare machine, cold plug stuff into it. Now qemu has to reconcile the stateful non-config file with the hardware. What if something has changed? A device moved into a different slot?

If a network card has eeprom, we can specify it with -device rtl8139,eeprom=id, where id specifies a disk image for the eeprom.

Quoted 4 lines

Quoted 1 line

I think my solution (multiplexing block format driver) fits the requirements for live-copy perfectly. In fact it has a name - it's a RAID-1 driver started in degraded mode. It could be useful other use cases.

It feels a bit awkward to me to be honest.

Not to me.

Quoted 8 lines · Avi Kivity

Quoted 7 lines · Anthony Liguori

Quoted 1 line

Then the management stack has to worry about yet another way of interacting via qemu.

{ 'StateItem': { 'key': 'str', 'value': 'str' } }
{ 'StateSection': { 'kind': 'str', 'name': 'str', 'items': [ 'StateItem' ] } } { 'StateInfo': { 'sections': [ 'StateSection' ] } }

Quoted 5 lines

Quoted 3 lines

{ 'query-state', {}, {}, 'StateInfo' }

A management tool never need to worry about anything other than this command if it so chooses. If we have the pre-machine init mode for 0.16, then this can even be used to inspect state without running a guest.

So we have yet another information tree. If we store the cd-rom eject state here, then we need to make an association between the device path of the cd-rom, and the StateItem key.

And this linkage is key.

Let's say I launch QEMU with:

qemu -cdrom ~/foo.img

And then in the monitor, I do:

(qemu) eject ide1-cd0

The question is, what command can I now use to launch the same qemu instance?

When I think of stateful config, what I really think of is a way to spit out a command line that essentially becomes, "this is how you now launch QEMU".

In this case, it would be:

qemu -cdrom ~/foo.img -device ide-disk,id=ide1-cd0,drive=

Or, we could think of this in terms of:

qemu -cdrom ~/foo.img -readconfig foo.cfg

Where foo.cfg contained:

[device "ide1-cd0"]
driver="ide-disk"
drive=""

So what I'm really suggesting is that we generate foo.cfg whenever monitor commands do things that change the command line and introduce a new option to reflect this, IOW:

qemu -cdrom ~/foo.img -config foo.cfg

Quoted 2 lines

Far better to store it in the device itself. For example, we could make a layered block format driver that stores the eject state and a "backing file" containing the actual media. Eject and media change would be recorded in the block format driver's state. You could then hot-unplug a USB cd-writer and hot-plug it back into a different guest, implementing a virtual sneakernet.

I think you're far too hung up on "store it in the device itself". The recipe to create the device model is not intrinsic to the device model. It's an independent thing that's a combination of the command line arguments and any executed monitor commands.

Maybe a better way to think about the stateful config file is a mechanism to replay the monitor history.

Quoted 4 lines

Quoted 2 lines

The fact that the state is visible in the filesystem is an implementation detail.

A detail that has to be catered for by the management stack - it has to provide a safe place for it, back it up, etc.

If it cares for QEMU to preserve state. Today, this all gets thrown away.

Quoted 4 lines

Quoted 2 lines

It doesn't work for eject unless you interpose an acknowledged event. Ultimately, this is a simple problem. If you want reliability, we either need symmetric RPCs so that the device model can call (and wait) to the management layer to acknowledge a change or QEMU can post an event to the management layer, and maintain the state in a reliable fashion.

I don't see why it doesn't work. Please explain.

1) guest eject
2) qemu posts eject event
3) qemu acknowledges eject to the guest
4) management tool sees eject event and updates guest config

There's a race between 3 & 4. It can only be addressed by interposing 4 between 2 and 3 OR making qemu persist this state between 2 and 3 such that the management tool can reliably query it.

Quoted 1 line

I disagree. Storing NVRAM as a disk image is a simple extension of existing management tools. Block live-copy and cd-rom eject state also make sense as per-image state if you take hotunplug and hotplug into account.

Everything can be stored in a block driver but when the data is highly structured, isn't it nice to expose it in a structured, human readable way? I know I'd personally prefer a text representation of CMOS than a binary blob.

Quoted 3 lines

Device settings should be stored with the devices, not with qemu.

Suppose we take the cold-plug on startup via the monitor approach. So we start with a bare machine, cold plug stuff into it. Now qemu has to reconcile the stateful non-config file with the hardware. What if something has changed? A device moved into a different slot?

Sorry, I'm confused. Is there anything in the stateful config file when we start up? If so, the act of starting up will add a bunch of hardware.

Quoted 2 lines

If a network card has eeprom, we can specify it with -device rtl8139,eeprom=id, where id specifies a disk image for the eeprom.

We could, but then we'll end up with a bunch of little block devices. That seems less than ideal to me.

Technically, mac address is stored on eeprom and we store that as a device property today. We can't persist device properties even though you can change the mac address of a network card and it does persist across reboots. Are you advocating that we introduce an eeprom for every network card (all in a slightly different format) and have special tools to manipulate the eeprom to store and view the mac address?

Quoted 39 lines · Anthony Liguori

Quoted 8 lines · Avi Kivity

Quoted 7 lines · Anthony Liguori

Quoted 1 line

Then the management stack has to worry about yet another way of interacting via qemu.

{ 'StateItem': { 'key': 'str', 'value': 'str' } }
{ 'StateSection': { 'kind': 'str', 'name': 'str', 'items': [ 'StateItem' ] } } { 'StateInfo': { 'sections': [ 'StateSection' ] } }

Quoted 5 lines

Quoted 3 lines

{ 'query-state', {}, {}, 'StateInfo' }

A management tool never need to worry about anything other than this command if it so chooses. If we have the pre-machine init mode for 0.16, then this can even be used to inspect state without running a guest.

So we have yet another information tree. If we store the cd-rom eject state here, then we need to make an association between the device path of the cd-rom, and the StateItem key.

And this linkage is key.

Let's say I launch QEMU with:

qemu -cdrom ~/foo.img

And then in the monitor, I do:

(qemu) eject ide1-cd0

The question is, what command can I now use to launch the same qemu instance?

When I think of stateful config, what I really think of is a way to spit out a command line that essentially becomes, "this is how you now launch QEMU".

In this case, it would be:

qemu -cdrom ~/foo.img -device ide-disk,id=ide1-cd0,drive=

Or, we could think of this in terms of:

qemu -cdrom ~/foo.img -readconfig foo.cfg

Where foo.cfg contained:

Quoted 7 lines

[device "ide1-cd0"]
driver="ide-disk"
drive=""

So what I'm really suggesting is that we generate foo.cfg whenever monitor commands do things that change the command line and introduce a new option to reflect this, IOW:

qemu -cdrom ~/foo.img -config foo.cfg

If you move the cdrom to a different IDE channel, you have to update the stateful non-config file.

Whereas if you do

   $ qemu-img create -f cd-tray -b ~/foo.img ~/foo-media-tray.img
   $ qemu -cdrom ~/foo-media-tray.img

the cd-rom tray state will be tracked in the image file.

Quoted 5 lines

Quoted 1 line

Far better to store it in the device itself. For example, we could make a layered block format driver that stores the eject state and a "backing file" containing the actual media. Eject and media change would be recorded in the block format driver's state. You could then hot-unplug a USB cd-writer and hot-plug it back into a different guest, implementing a virtual sneakernet.

I think you're far too hung up on "store it in the device itself". The recipe to create the device model is not intrinsic to the device model. It's an independent thing that's a combination of the command line arguments and any executed monitor commands.

Maybe a better way to think about the stateful config file is a mechanism to replay the monitor history.

Again the question is who is the authoritative source of the configuration. Is it the management tool or is it qemu?

The management tool already has to keep track of (the optional parts of) the guest device tree. It cannot start reading the stateful non-config file at random points in time. So all that is left is the guest controlled portions of the device tree, which are pretty rare, and random events like live-copy migration. I think that introducing a new authoritative source of information will create a lot of problems.

Quoted 7 lines

Quoted 5 lines

Quoted 3 lines

The fact that the state is visible in the filesystem is an implementation detail.

A detail that has to be catered for by the management stack - it has to provide a safe place for it, back it up, etc.

If it cares for QEMU to preserve state. Today, this all gets thrown away.

Right, but we should make it easy, not hard.

Quoted 5 lines

Quoted 4 lines

Quoted 2 lines

It doesn't work for eject unless you interpose an acknowledged event. Ultimately, this is a simple problem. If you want reliability, we either need symmetric RPCs so that the device model can call (and wait) to the management layer to acknowledge a change or QEMU can post an event to the management layer, and maintain the state in a reliable fashion.

I don't see why it doesn't work. Please explain.

Quoted 6 lines

1) guest eject
2) qemu posts eject event
3) qemu acknowledges eject to the guest
4) management tool sees eject event and updates guest config

There's a race between 3 & 4. It can only be addressed by interposing 4 between 2 and 3 OR making qemu persist this state between 2 and 3 such that the management tool can reliably query it.

If "it" is my cd-rom tray block format driver, it works. It's really the same in action as the stateful non-config, except it's part of the device/image, not a central location.

Quoted 3 lines

Quoted 1 line

I disagree. Storing NVRAM as a disk image is a simple extension of existing management tools. Block live-copy and cd-rom eject state also make sense as per-image state if you take hotunplug and hotplug into account.

Everything can be stored in a block driver but when the data is highly structured, isn't it nice to expose it in a structured, human readable way? I know I'd personally prefer a text representation of CMOS than a binary blob.

Have a tool expose it. Part of the range is unspecified anyway.

Using a block format driver means that we don't have to care about a crash during a write, that we can snapshot it, etc.

Quoted 5 lines

Quoted 3 lines

Device settings should be stored with the devices, not with qemu.

Suppose we take the cold-plug on startup via the monitor approach. So we start with a bare machine, cold plug stuff into it. Now qemu has to reconcile the stateful non-config file with the hardware. What if something has changed? A device moved into a different slot?

Sorry, I'm confused. Is there anything in the stateful config file when we start up? If so, the act of starting up will add a bunch of hardware.

Suppose it has information about ide1-cd0's media tray. Now we restart qemu and cold-plug the cdrom into ide0-cd0. What happens to the information?

Quoted 1 line

Technically, mac address is stored on eeprom and we store that as a device property today. We can't persist device properties even though you can change the mac address of a network card and it does persist across reboots. Are you advocating that we introduce an eeprom for every network card (all in a slightly different format) and have special tools to manipulate the eeprom to store and view the mac address?

Yes -- if we really want to support it. Obviously we have to store the entire eeprom, not just the portion containing the MAC address, so it's not just a key/value store. A card may even have its firmware in flash.

Quoted 8 lines

If you move the cdrom to a different IDE channel, you have to update the stateful non-config file.

Whereas if you do

   $ qemu-img create -f cd-tray -b ~/foo.img ~/foo-media-tray.img
   $ qemu -cdrom ~/foo-media-tray.img

the cd-rom tray state will be tracked in the image file.

Yeah, but how do you move it?

If you do a remove/add through QMP, then the config file will reflect things just fine.

If you want to do it outside of QEMU, then you can just ignore the config file and manage all of the state yourself. But it's never going to work as well (it will be racy) and you're pushing a tremendous amount of knowledge that ultimately belongs in QEMU (what state needs to persist) to something that isn't QEMU which means it's probably not going to be done correctly.

I know you're a big fan of the omnipotent management tool but my experience has been that we need to help the management tooling folks more by expecting less from them.

Quoted 2 lines

Again the question is who is the authoritative source of the configuration. Is it the management tool or is it qemu?

QEMU. No question about it. At any point in time, we are the authoritative source of what the guest's configuration is. There's no doubt about it. A management tool can try to keep up with us, but ultimately we are the only ones that know for sure.

We have all of this information internally. Just persisting it is not a major architectural change. It's something we should have been doing (arguably) from the very beginning.

Quoted 2 lines

The management tool already has to keep track of (the optional parts of) the guest device tree. It cannot start reading the stateful non-config file at random points in time. So all that is left is the guest controlled portions of the device tree, which are pretty rare, and random events like live-copy migration. I think that introducing a new authoritative source of information will create a lot of problems.

QEMU has always been the authoritative source. Nothing new has been introduced. We never persisted the machine's configuration which meant management tools had to try to aggressively keep up with us which is intrinsically error prone. Fixing this will only improve existing management tools.

Quoted 2 lines

Right, but we should make it easy, not hard.

Yeah, I fail to see how this makes it hard. We conveniently are saying, hey, this is all the state that needs to be persisted. We'll persist it for you if you want, otherwise, we'll expose it in a central location.

If the tool wants to ignore it and guess based on various combinations of other commands, more power to it.

Quoted 7 lines

Quoted 7 lines

Quoted 1 line

I don't see why it doesn't work. Please explain.

1) guest eject
2) qemu posts eject event
3) qemu acknowledges eject to the guest
4) management tool sees eject event and updates guest config

Quoted 3 lines

Quoted 1 line

There's a race between 3 & 4. It can only be addressed by interposing 4 between 2 and 3 OR making qemu persist this state between 2 and 3 such that the management tool can reliably query it.

If "it" is my cd-rom tray block format driver, it works. It's really the same in action as the stateful non-config, except it's part of the device/image, not a central location.

Because you've introduced a one-off. Having a bunch of one-offs (especially being a bunch of new block formats!) is not going to make things simpler for management tools.

Quoted 2 lines

Have a tool expose it. Part of the range is unspecified anyway.

I guess we need to agree to disagree then.

Quoted 2 lines

Using a block format driver means that we don't have to care about a crash during a write, that we can snapshot it, etc.

Why?

We always need to care about a crash during write. What I've been thinking for a config file is the class approach of using a ~ and .# file to make sure that we write out the new file and then atomically rename it to get the new contents. Yeah, it's a bit heavy weight but this shouldn't be a very common thing to update.

Quoted 2 lines

Suppose it has information about ide1-cd0's media tray. Now we restart qemu and cold-plug the cdrom into ide0-cd0. What happens to the information?

Whether media is present is not a property of a blockdev, it's a property of a device.

What does it even mean to use your media-tray format with something
like a CMOS device?

Quoted 5 lines

Quoted 2 lines

Technically, mac address is stored on eeprom and we store that as a device property today. We can't persist device properties even though you can change the mac address of a network card and it does persist across reboots. Are you advocating that we introduce an eeprom for every network card (all in a slightly different format) and have special tools to manipulate the eeprom to store and view the mac address?

Yes -- if we really want to support it. Obviously we have to store the entire eeprom, not just the portion containing the MAC address, so it's not just a key/value store. A card may even have its firmware in flash.

I think that's overengineering. I think we can go very far by just persisting small amounts of information in a central location. We're not building a cycle-accurate simulator here afterall.

Quoted 5 lines · Anthony Liguori

Quoted 5 lines

If you move the cdrom to a different IDE channel, you have to update the stateful non-config file.

Whereas if you do

Quoted 7 lines

Quoted 4 lines

$ qemu-img create -f cd-tray -b ~/foo.img ~/foo-media-tray.img
$ qemu -cdrom ~/foo-media-tray.img

the cd-rom tray state will be tracked in the image file.

Yeah, but how do you move it?

There is no need to move the file at all. Simply point the new drive at the media tray.

Quoted 2 lines

If you do a remove/add through QMP, then the config file will reflect things just fine.

If all access to the state file is through QMP then it becomes more palatable. A bit on that later.

Quoted 11 lines

If you want to do it outside of QEMU, then you can just ignore the config file and manage all of the state yourself. But it's never going to work as well (it will be racy) and you're pushing a tremendous amount of knowledge that ultimately belongs in QEMU (what state needs to persist) to something that isn't QEMU which means it's probably not going to be done correctly.

I know you're a big fan of the omnipotent management tool but my experience has been that we need to help the management tooling folks more by expecting less from them.

I thought that's what I'm doing by separating the state out. It's easy for management to assemble configuration from their database and convert it into a centralized representation (like a qemu command line). It's a lot harder to disassemble a central state representation and move it back to the database.

Using QMP is better than directly accessing the state file since qemu does the disassembly for you (provided the command references the device using its normal path, not some random key). The file just becomes a way to survive a crash, and all management needs to know about is to make it available and back it up. But it means that everything must be done via QMP, including assembly of the machine, otherwise the state file can become stale.

Separating the state out to the device is even easier, since management is already expected to take care of disk images. All that's needed is to create the media tray image once, then you can forget about it completely.

Quoted 4 lines

Quoted 3 lines

Again the question is who is the authoritative source of the configuration. Is it the management tool or is it qemu?

Quoted 8 lines

QEMU. No question about it. At any point in time, we are the authoritative source of what the guest's configuration is. There's no doubt about it. A management tool can try to keep up with us, but ultimately we are the only ones that know for sure.

We have all of this information internally. Just persisting it is not a major architectural change. It's something we should have been doing (arguably) from the very beginning.

That's a huge divergence from how management tools are written. Currently they contain the required guest configuration, a representation of what's the current live configuration, and they issue monitor commands to move the live configuration towards the required configuration (or just generate a qemu command line). What you're describing is completely different, I'm not even sure what it is.

Quoted 8 lines

Quoted 7 lines

The management tool already has to keep track of (the optional parts of) the guest device tree. It cannot start reading the stateful non-config file at random points in time. So all that is left is the guest controlled portions of the device tree, which are pretty rare, and random events like live-copy migration. I think that introducing a new authoritative source of information will create a lot of problems.

Quoted 5 lines

QEMU has always been the authoritative source. Nothing new has been introduced. We never persisted the machine's configuration which meant management tools had to try to aggressively keep up with us which is intrinsically error prone. Fixing this will only improve existing management tools.

If you look at management tools, they believe they are the authoritative source of configuration information (not guest state, which is more or less ignored).

Quoted 7 lines

Quoted 2 lines

Right, but we should make it easy, not hard.

Yeah, I fail to see how this makes it hard. We conveniently are saying, hey, this is all the state that needs to be persisted. We'll persist it for you if you want, otherwise, we'll expose it in a central location.

The state-in-a-file is just a blob. Don't expect the tool to parse it and reassociate the various bits to its own representation. Exposing it via QMP commands is a lot better though.

Quoted 13 lines

If the tool wants to ignore it and guess based on various combinations of other commands, more power to it.

Quoted 11 lines

Quoted 10 lines

Quoted 1 line

I don't see why it doesn't work. Please explain.

1) guest eject
2) qemu posts eject event
3) qemu acknowledges eject to the guest
4) management tool sees eject event and updates guest config

There's a race between 3 & 4. It can only be addressed by interposing 4 between 2 and 3 OR making qemu persist this state between 2 and 3 such that the management tool can reliably query it.

Quoted 7 lines

Quoted 3 lines

If "it" is my cd-rom tray block format driver, it works. It's really the same in action as the stateful non-config, except it's part of the device/image, not a central location.

Because you've introduced a one-off. Having a bunch of one-offs (especially being a bunch of new block formats!) is not going to make things simpler for management tools.

I don't see it as a one-off. We have some state, we store it in a state file. Like we store each image in its own file.

Quoted 4 lines

Quoted 2 lines

Have a tool expose it. Part of the range is unspecified anyway.

I guess we need to agree to disagree then.

We can just extend our existing disagreement agreement.

Quoted 5 lines

Quoted 3 lines

Using a block format driver means that we don't have to care about a crash during a write, that we can snapshot it, etc.

Why?

Because block format drivers are already prepared for it.

Quoted 8 lines

We always need to care about a crash during write. What I've been thinking for a config file is the class approach of using a ~ and .# file to make sure that we write out the new file and then atomically rename it to get the new contents. Yeah, it's a bit heavy weight but this shouldn't be a very common thing to update.

Quoted 1 line

Quoted 6 lines

Quoted 3 lines

Suppose it has information about ide1-cd0's media tray. Now we restart qemu and cold-plug the cdrom into ide0-cd0. What happens to the information?

Whether media is present is not a property of a blockdev, it's a property of a device.

You're making it the property of a device path (I guess you could make it the property of a device's id).

Quoted 2 lines

What does it even mean to use your media-tray format with something like a CMOS device?

Nothing.

Quoted 15 lines

Quoted 14 lines

Quoted 8 lines

Technically, mac address is stored on eeprom and we store that as a device property today. We can't persist device properties even though you can change the mac address of a network card and it does persist across reboots. Are you advocating that we introduce an eeprom for every network card (all in a slightly different format) and have special tools to manipulate the eeprom to store and view the mac address?

Yes -- if we really want to support it. Obviously we have to store the entire eeprom, not just the portion containing the MAC address, so it's not just a key/value store. A card may even have its firmware in flash.

Quoted 3 lines

I think that's overengineering. I think we can go very far by just persisting small amounts of information in a central location. We're not building a cycle-accurate simulator here afterall.

Well, the whole thing is rather theoretical.

What about a simpler approach were QMP events will be written to a event-log-file (or even named pipe).

Quoted 2 lines · Dor Laor

What about a simpler approach were QMP events will be written to a event-log-file (or even named pipe).

The management tool can just use a small daemon that does nothing other than write QMP events to a log. There's no real need for this code to live in QEMU.

Since events are posted, even if we wrote it in QEMU, the event wouldn't be guaranteed on disk by the time the event invocation returns.

Regards,

Anthony Liguori

Quoted 8 lines · Anthony Liguori

Quoted 2 lines · Dor Laor

What about a simpler approach were QMP events will be written to a event-log-file (or even named pipe).

The management tool can just use a small daemon that does nothing other than write QMP events to a log. There's no real need for this code to live in QEMU.

IIUC in case the management daemon will run qemu using named pipe for qmp it will happen automatically.

If you agree to this approach it will simplify the more complex config file option (although it is nice to have as independent option for single hosts managed by simpler mgmts)

Quoted 7 lines

Since events are posted, even if we wrote it in QEMU, the event wouldn't be guaranteed on disk by the time the event invocation returns.

Regards,

Anthony Liguori

Quoted 10 lines · Dor Laor

Quoted 8 lines · Anthony Liguori

Quoted 2 lines · Dor Laor

What about a simpler approach were QMP events will be written to a event-log-file (or even named pipe).

The management tool can just use a small daemon that does nothing other than write QMP events to a log. There's no real need for this code to live in QEMU.

Quoted 2 lines

IIUC in case the management daemon will run qemu using named pipe for qmp it will happen automatically.

No, the event model is changing (it was always intended to change though). Events will need explicit registration so it's necessary to have bidirectional communication.

So you can't just do -qmp foo -chardev file:event.log,id=foo. You won't actually see most of the events.

Regards,

Anthony Liguori

Quoted 13 lines

If you agree to this approach it will simplify the more complex config file option (although it is nice to have as independent option for single hosts managed by simpler mgmts)

Quoted 7 lines

Since events are posted, even if we wrote it in QEMU, the event wouldn't be guaranteed on disk by the time the event invocation returns.

Regards,

Anthony Liguori

Quoted 7 lines · Anthony Liguori

Quoted 7 lines

If you move the cdrom to a different IDE channel, you have to update the stateful non-config file.

Whereas if you do

   $ qemu-img create -f cd-tray -b ~/foo.img ~/foo-media-tray.img
   $ qemu -cdrom ~/foo-media-tray.img

Quoted 4 lines

Quoted 1 line

the cd-rom tray state will be tracked in the image file.

Yeah, but how do you move it?

There is no need to move the file at all. Simply point the new drive at the media tray.

Quoted 2 lines

If you do a remove/add through QMP, then the config file will reflect things just fine.

If all access to the state file is through QMP then it becomes more palatable. A bit on that later.

Quoted 3 lines

If you want to do it outside of QEMU, then you can just ignore the config file and manage all of the state yourself. But it's never going to work as well (it will be racy) and you're pushing a tremendous amount of knowledge that ultimately belongs in QEMU (what state needs to persist) to something that isn't QEMU which means it's probably not going to be done correctly.

I know you're a big fan of the omnipotent management tool but my experience has been that we need to help the management tooling folks more by expecting less from them.

I thought that's what I'm doing by separating the state out. It's easy for management to assemble configuration from their database and convert it into a centralized representation (like a qemu command line). It's a lot harder to disassemble a central state representation and move it back to the database.

Using QMP is better than directly accessing the state file since qemu does the disassembly for you (provided the command references the device using its normal path, not some random key). The file just becomes a way to survive a crash, and all management needs to know about is to make it available and back it up. But it means that everything must be done via QMP, including assembly of the machine, otherwise the state file can become stale.

Separating the state out to the device is even easier, since management is already expected to take care of disk images. All that's needed is to create the media tray image once, then you can forget about it completely.

Quoted 6 lines

Quoted 2 lines

Again the question is who is the authoritative source of the configuration. Is it the management tool or is it qemu?

QEMU. No question about it. At any point in time, we are the authoritative source of what the guest's configuration is. There's no doubt about it. A management tool can try to keep up with us, but ultimately we are the only ones that know for sure.

We have all of this information internally. Just persisting it is not a major architectural change. It's something we should have been doing (arguably) from the very beginning.

That's a huge divergence from how management tools are written.

Currently they contain the required guest configuration, a
representation of what's the current live configuration, and they issue monitor commands to move the live configuration towards the required configuration (or just generate a qemu command line). What you're describing is completely different, I'm not even sure what it is.

Quoted 5 lines

Quoted 3 lines

The management tool already has to keep track of (the optional parts of) the guest device tree. It cannot start reading the stateful non-config file at random points in time. So all that is left is the guest controlled portions of the device tree, which are pretty rare, and random events like live-copy migration. I think that introducing a new authoritative source of information will create a lot of problems.

QEMU has always been the authoritative source. Nothing new has been introduced. We never persisted the machine's configuration which meant management tools had to try to aggressively keep up with us which is intrinsically error prone. Fixing this will only improve existing management tools.

If you look at management tools, they believe they are the authoritative source of configuration information (not guest state, which is more or less ignored).

Quoted 4 lines

Quoted 2 lines

Right, but we should make it easy, not hard.

Yeah, I fail to see how this makes it hard. We conveniently are saying, hey, this is all the state that needs to be persisted. We'll persist it for you if you want, otherwise, we'll expose it in a central location.

The state-in-a-file is just a blob. Don't expect the tool to parse it and reassociate the various bits to its own representation. Exposing it via QMP commands is a lot better though.

Quoted 8 lines · Avi Kivity

Quoted 7 lines · Anthony Liguori

Quoted 7 lines

If you move the cdrom to a different IDE channel, you have to update the stateful non-config file.

Whereas if you do

   $ qemu-img create -f cd-tray -b ~/foo.img ~/foo-media-tray.img
   $ qemu -cdrom ~/foo-media-tray.img

Quoted 6 lines

Quoted 4 lines

Quoted 1 line

the cd-rom tray state will be tracked in the image file.

Yeah, but how do you move it?

There is no need to move the file at all. Simply point the new drive at the media tray.

No, I was asking, how do you move the cdrom to a different IDE channel. Are you using QMP? Are you changing the command line arguments?

Quoted 5 lines

Quoted 3 lines

If you do a remove/add through QMP, then the config file will reflect things just fine.

If all access to the state file is through QMP then it becomes more palatable. A bit on that later.

As I think I've mentioned before, I hadn't really thought about an opaque state file but I'm not necessary opposed to it. I don't see an obvious advantage to making it opaque but I agree it should be accessible via QMP.

Quoted 6 lines

I thought that's what I'm doing by separating the state out. It's easy for management to assemble configuration from their database and convert it into a centralized representation (like a qemu command line). It's a lot harder to disassemble a central state representation and move it back to the database.

Using QMP is better than directly accessing the state file since qemu does the disassembly for you (provided the command references the device using its normal path, not some random key). The file just becomes a way to survive a crash, and all management needs to know about is to make it available and back it up. But it means that everything must be done via QMP, including assembly of the machine, otherwise the state file can become stale.

Separating the state out to the device is even easier, since management is already expected to take care of disk images. All that's needed is to create the media tray image once, then you can forget about it completely.

Except that instead of having one state file, we might have a dozen additional "device state" files.

Quoted 5 lines

Quoted 3 lines

QEMU. No question about it. At any point in time, we are the authoritative source of what the guest's configuration is. There's no doubt about it. A management tool can try to keep up with us, but ultimately we are the only ones that know for sure.

We have all of this information internally. Just persisting it is not a major architectural change. It's something we should have been doing (arguably) from the very beginning.

That's a huge divergence from how management tools are written.

This is one of the reasons why management tooling around QEMU needs quite a bit of improving.

There is simply no way a management tool can do a good job of being an authoritative source of configuration. The races we're discussion is a good example of why.

But beyond those races, QEMU is the only entity that knows with certainty what bits of information are important to persist in order to preserve a guest across shutdown/restart. The fact that we've punted this problem for so long has only ensured that management tools are either intrinsically broken or only support the most minimal subset of functionality we actually support.

Quoted 2 lines

Currently they contain the required guest configuration, a
representation of what's the current live configuration, and they issue monitor commands to move the live configuration towards the required configuration (or just generate a qemu command line). What you're describing is completely different, I'm not even sure what it is.

Management tools shouldn't have to think about how the monitor commands they issue impact the invocation options of QEMU.

Quoted 2 lines

If you look at management tools, they believe they are the authoritative source of configuration information (not guest state, which is more or less ignored).

It's because we've given them no other option.

Quoted 7 lines

Quoted 5 lines

Quoted 3 lines

Right, but we should make it easy, not hard.

Yeah, I fail to see how this makes it hard. We conveniently are saying, hey, this is all the state that needs to be persisted. We'll persist it for you if you want, otherwise, we'll expose it in a central location.

The state-in-a-file is just a blob. Don't expect the tool to parse it and reassociate the various bits to its own representation. Exposing it via QMP commands is a lot better though.

I don't really see this as being a major issue. There's no such thing as a "blob". If someone wants to manipulate the state, they will. We need to keep compatibility to support migrating from version-to-version.

I agree that we want to provide QMP interfaces to work with the state file. But I don't think we should be hostile to manual manipulation.

Quoted 9 lines · Anthony Liguori

Quoted 8 lines · Avi Kivity

Quoted 7 lines · Anthony Liguori

Quoted 7 lines

If you move the cdrom to a different IDE channel, you have to update the stateful non-config file.

Whereas if you do

   $ qemu-img create -f cd-tray -b ~/foo.img ~/foo-media-tray.img
   $ qemu -cdrom ~/foo-media-tray.img

Quoted 8 lines

Quoted 6 lines

Quoted 4 lines

Quoted 1 line

the cd-rom tray state will be tracked in the image file.

Yeah, but how do you move it?

There is no need to move the file at all. Simply point the new drive at the media tray.

No, I was asking, how do you move the cdrom to a different IDE channel. Are you using QMP? Are you changing the command line arguments?

Yes.

If we're doing hot-move (not really relevant to ide-cd) then you'd use QMP. If you're editing a virtual machine that is down, or scheduling a change for the next reboot, then you're using command line arguments (or cold-plugging into a stopped guest).

Requiring management to remember the old configuration and issue delta commands to move the device for the cold-plug case is increased complexity IMO.

Quoted 7 lines

Quoted 5 lines

Quoted 3 lines

If you do a remove/add through QMP, then the config file will reflect things just fine.

If all access to the state file is through QMP then it becomes more palatable. A bit on that later.

As I think I've mentioned before, I hadn't really thought about an opaque state file but I'm not necessary opposed to it. I don't see an obvious advantage to making it opaque but I agree it should be accessible via QMP.

The advantage is that we keep the management tool talking to one interface (I don't think we should prevent users from interpreting it, just make it unnecessary).

Quoted 8 lines

Quoted 6 lines

I thought that's what I'm doing by separating the state out. It's easy for management to assemble configuration from their database and convert it into a centralized representation (like a qemu command line). It's a lot harder to disassemble a central state representation and move it back to the database.

Using QMP is better than directly accessing the state file since qemu does the disassembly for you (provided the command references the device using its normal path, not some random key). The file just becomes a way to survive a crash, and all management needs to know about is to make it available and back it up. But it means that everything must be done via QMP, including assembly of the machine, otherwise the state file can become stale.

Separating the state out to the device is even easier, since management is already expected to take care of disk images. All that's needed is to create the media tray image once, then you can forget about it completely.

Except that instead of having one state file, we might have a dozen additional "device state" files.

That is fine. We already have one state file per block device.

Quoted 9 lines

Quoted 5 lines

Quoted 3 lines

QEMU. No question about it. At any point in time, we are the authoritative source of what the guest's configuration is. There's no doubt about it. A management tool can try to keep up with us, but ultimately we are the only ones that know for sure.

We have all of this information internally. Just persisting it is not a major architectural change. It's something we should have been doing (arguably) from the very beginning.

That's a huge divergence from how management tools are written.

This is one of the reasons why management tooling around QEMU needs quite a bit of improving.

There is simply no way a management tool can do a good job of being an authoritative source of configuration. The races we're discussion is a good example of why.

What we're discussing is not configuration. It is non-volatile state. Configuration comes from the user; state comes from the guest (the management tool may edit state; but the guest cannot edit the configuration).

I agree 100% the management tool cannot be the authoritative source of state.

My position is:
- the management tool should be 100% in control of configuration (how
the guest is put together from its components)
- qemu should be 100% in control of state (memory, disk state, NVRAM
in various components, cd-rom eject state, explosive bolts for payload separation, self-destruct mechanism, etc.)

Quoted 2 lines

But beyond those races, QEMU is the only entity that knows with certainty what bits of information are important to persist in order to preserve a guest across shutdown/restart. The fact that we've punted this problem for so long has only ensured that management tools are either intrinsically broken or only support the most minimal subset of functionality we actually support.

I'm not arguing about that. I just want to stress again the difference between state and configuration. Qemu has no authority, in my mind, as to configuration. Only state.

Quoted 4 lines

Quoted 2 lines

Currently they contain the required guest configuration, a
representation of what's the current live configuration, and they issue monitor commands to move the live configuration towards the required configuration (or just generate a qemu command line). What you're describing is completely different, I'm not even sure what it is.

Management tools shouldn't have to think about how the monitor commands they issue impact the invocation options of QEMU.

They have to, when creating a guest from scratch.

But I admit, this throws a new light (for me) on things. What's the implications?
- must have a qemu instance running when editing configuration, even when the guest is down

- cannot add additional information to configuration; must store it in an external database and cross-reference it with the qemu data using the device ID

- when editing non-hotpluggable configuration for the next boot, must
maintain old config somewhere, so we can issue delta commands later (might be needed for current way of doing things)

- no transactions/queries/etc except on non-authoritative source
- issues with shared-nothing design (well, can store the configuration file using DRBD).

Quoted 3 lines

Quoted 1 line

If you look at management tools, they believe they are the authoritative source of configuration information (not guest state, which is more or less ignored).

It's because we've given them no other option.

It's the natural way of doing it. You have a web interface that talks to a database. When you want to list all VMs that have network cards on the production subnet, you issue a database query and get a recordset. How do you do that when the authoritative source of information is spread across a cluster?

Quoted 8 lines

I agree 100% the management tool cannot be the authoritative source of state.

My position is:
- the management tool should be 100% in control of configuration (how
the guest is put together from its components)
- qemu should be 100% in control of state (memory, disk state, NVRAM
in various components, cd-rom eject state, explosive bolts for payload separation, self-destruct mechanism, etc.)

There simply is not such a clean separation between the two because things that the guest does affects the configuration of the guest.

Hot plug,

removable media eject,

persistent device settings (whether it's CMOS or EEPROM) all disrupt this model.

If you really wanted to have this separation, you'd have to be very strict about making all guest settings not be specified in config. You would need to do:

qemu-img create -f e1000-eprom -o macaddr=12:23:45:67:78:90 e1000.0.rom qemu-img create -f e1000-eprom -o macaddr=12:23:45:67:78:91 e1000.1.rom

qemu -device e1000,id=e1000.0,eeprom=e1000.0.rom -device e1000,id=e1000.1,eeprom=e1000.1.rom

And now I need a tool that lets me modify e1000-eprom images if I want to change the mac address dynamically (say I'm trying to clone a VM).

This type of model can be workable but as I said earlier, I think it's overengineering the problem.

We don't separate configuration from guest state today. Instead of setting ourselves up for failure by setting an unrealistic standard that we try to achieve and never do, let's embrace the system that is working for us today. We are authoritative for everything and guest state is intimately tied to the virtual machine configuration.

Quoted 5 lines

Quoted 3 lines

But beyond those races, QEMU is the only entity that knows with certainty what bits of information are important to persist in order to preserve a guest across shutdown/restart. The fact that we've punted this problem for so long has only ensured that management tools are either intrinsically broken or only support the most minimal subset of functionality we actually support.

I'm not arguing about that. I just want to stress again the difference between state and configuration. Qemu has no authority, in my mind, as to configuration. Only state.

Being the one that creates a guest based on configuration, I would say that we most certainly do.

Quoted 6 lines

Quoted 5 lines

Quoted 3 lines

Currently they contain the required guest configuration, a
representation of what's the current live configuration, and they issue monitor commands to move the live configuration towards the required configuration (or just generate a qemu command line). What you're describing is completely different, I'm not even sure what it is.

Management tools shouldn't have to think about how the monitor commands they issue impact the invocation options of QEMU.

Quoted 5 lines

They have to, when creating a guest from scratch.

But I admit, this throws a new light (for me) on things. What's the implications?
- must have a qemu instance running when editing configuration, even when the guest is down

QMP is an API. Whether a qemu instance is launched is an implementation detail. This could all be hidden completely with libqmp.

Quoted 3 lines

- cannot add additional information to configuration; must store it
in an external database and cross-reference it with the qemu data using the device ID

Don't confuse a management tool's notion of configuration with QEMU's configuration.

A management tools config is used to initially create and then manipulate an existing guest. If the management tool supports out-of-band manipulation of a configuration file, then it needs to determine how the configuration file changed and execute the appropriate commands.

Yes, it is. libvirt kind of cheats here and just deletes the old VM and creates a new one when editing the XML IIUC.

Quoted 3 lines

- no transactions/queries/etc except on non-authoritative source
- issues with shared-nothing design (well, can store the configuration file using DRBD).

In both cases, today a management tool races with QEMU so both of these points are currently true.

Quoted 6 lines

Quoted 4 lines

Quoted 2 lines

If you look at management tools, they believe they are the authoritative source of configuration information (not guest state, which is more or less ignored).

It's because we've given them no other option.

It's the natural way of doing it. You have a web interface that talks to a database. When you want to list all VMs that have network cards on the production subnet, you issue a database query and get a recordset. How do you do that when the authoritative source of information is spread across a cluster?

This problem still exists today. A guest can eject a network card on it's own (without the management tool issuing a device_del command). QEMU will delete the NIC when this happens.

The same is true with CDROM eject.

Management tools are simply not authoritative today.

Regards,

Anthony Liguori

Quoted 2 lines

We don't separate configuration from guest state today. Instead of setting ourselves up for failure by setting an unrealistic standard that we try to achieve and never do, let's embrace the system that is working for us today. We are authoritative for everything and guest state is intimately tied to the virtual machine configuration.

"we are authoritative for everything" is a clean break from everything that's being done today. It's also a clean break from the model of central management plus database. We can't force it on people.

Quoted 4 lines · Avi Kivity

Quoted 2 lines

We don't separate configuration from guest state today. Instead of setting ourselves up for failure by setting an unrealistic standard that we try to achieve and never do, let's embrace the system that is working for us today. We are authoritative for everything and guest state is intimately tied to the virtual machine configuration.

"we are authoritative for everything" is a clean break from everything that's being done today. It's also a clean break from the model of central management plus database. We can't force it on people.

No, it isn't. This has been the way QEMU has always been.

QEMU has always and will always continue to be useful in the absence of a management layer. That means that it will mix modifications to configuration with guest actions.

To avoid races, a management tool needs to either poll QEMU or listen for events to know when QEMU initiates a configuration change. This is how tools are written today.

The only thing being discussed is how to handle a very small and rare race condition whereas QEMU may send an notification to a crashed management tool *and* QEMU crashes before the management tool restarts.

The only two options to solve this problem are synchronous notifications or a QEMU global state file. Since the former is a radical change to our existing API, the later is a much more reasonable option.

If a management tool doesn't care about this race, it can simply not use the global state file.

Quoted 5 lines · Anthony Liguori

Quoted 3 lines · Avi Kivity

Quoted 3 lines

We don't separate configuration from guest state today. Instead of

Quoted 3 lines

Quoted 3 lines

"we are authoritative for everything" is a clean break from everything

Quoted 5 lines

No, it isn't. This has been the way QEMU has always been.

QEMU has always and will always continue to be useful in the absence of a

Quoted 2 lines

To avoid races, a management tool needs to either poll QEMU or listen for

Quoted 2 lines

The only thing being discussed is how to handle a very small and rare

Quoted 2 lines

The only two options to solve this problem are synchronous notifications

Quoted 2 lines

If a management tool doesn't care about this race, it can simply not use

You're just ignoring what I've written.

--
error compiling committee.c: too many arguments to function

Quoted 3 lines

You're just ignoring what I've written.

No, you're just impervious to my subtle attempt to refocus the discussion on solving a practical problem.

There's a lot of good, reasonably straight forward changes we can make that have a high return on investment.

The only suggestion I'm making beyond Marcelo's original patch is that we use a structured format and that we make it possible to use the same file to solve this problem in multiple places.

I don't think this creates a fundamental break in how management tools interact with QEMU. I don't think introducing RAID support in the block layer is a reasonable alternative.

Quoted 6 lines · Anthony Liguori

Quoted 4 lines

You're just ignoring what I've written.

No, you're just impervious to my subtle attempt to refocus the discussion

Quoted 2 lines

There's a lot of good, reasonably straight forward changes we can make

Quoted 1 line

Is making qemu the authoritative source of configuration information a

"No" to all three (ignoring for the moment whether it is good or not,

Quoted 3 lines

The only suggestion I'm making beyond Marcelo's original patch is that we

Quoted 1 line

No, you're suggesting a lot more than that.

Quoted 1 line

I don't think this creates a fundamental break in how management tools

Quoted 2 lines

Why not?

Something that avoids the whole state thing altogether:

- instead of atomically switching when live copy is done, keep on issuing

- issue a notification to management
- management receives the notification, and issues an atomic blockdev

this is really the RAID-1 solution but without the state file (credit

--
error compiling committee.c: too many arguments to function

Quoted 8 lines · On Feb 28, 2011 11:47 AM, "Avi Kivity" <[email protected]…

Quoted 6 lines · Anthony Liguori

Quoted 4 lines

You're just ignoring what I've written.

No, you're just impervious to my subtle attempt to refocus the

discussion on solving a practical problem.

Quoted 2 lines

Quoted 2 lines

There's a lot of good, reasonably straight forward changes we can

make that have a high return on investment.

Quoted 3 lines

Quoted 1 line

Is making qemu the authoritative source of configuration information

a straightforward change? Is the return on it high? Is the investment low?

I think this is where we fundamentally disagree. My position is that QEMU is already the authoritative source. Having a state file doesn't change anything.

Do a hot unplug of a network device with upstream libvirt with acpiphp unloaded, consult libvirt and then consult the monitor to see who has the right view of the guests config.

To me, that's the definition of authoritative.

Quoted 1 line

"No" to all three (ignoring for the moment whether it is good or not,

which we were debating).

Quoted 3 lines

Quoted 3 lines

The only suggestion I'm making beyond Marcelo's original patch is

that we use a structured format and that we make it possible to use the same file to solve this problem in multiple places.

Quoted 3 lines

Quoted 1 line

No, you're suggesting a lot more than that.

That's exactly what I'm suggesting from a technical perspective.

Quoted 1 line

Quoted 1 line

I don't think this creates a fundamental break in how management

tools interact with QEMU. I don't think introducing RAID support in the block layer is a reasonable alternative.

Quoted 4 lines

Quoted 2 lines

Why not?

Because its a lot of complexity and code that can go wrong while only solving the race for one specific case. Not to mention that we double the iop rate.

Quoted 3 lines

Something that avoids the whole state thing altogether:

- instead of atomically switching when live copy is done, keep on

issuing writes to both the origin and the live copy

Quoted 2 lines

- issue a notification to management
- management receives the notification, and issues an atomic blockdev

switch command

Quoted 1 line

this is really the RAID-1 solution but without the state file (credit

Dor). An advantage is that there is no additional latency when trying to catch up to the dirty bitmap.

It still suffers from the two generals problem. You cannot solve this without making one node reliable and that takes us back to it being either QEMU (posted event and state file) or the management tool (sync event).

Regards,

Anthony Liguori

Quoted 4 lines

--
error compiling committee.c: too many arguments to function

Quoted 63 lines · Anthony Liguori

Quoted 8 lines · On Feb 28, 2011 11:47 AM, "Avi Kivity" <[email protected]…

Quoted 6 lines · Anthony Liguori

Quoted 4 lines

You're just ignoring what I've written.

No, you're just impervious to my subtle attempt to refocus the

discussion on solving a practical problem.

Quoted 2 lines

Quoted 2 lines

There's a lot of good, reasonably straight forward changes we can

make that have a high return on investment.

Quoted 3 lines

Quoted 1 line

Is making qemu the authoritative source of configuration information

a straightforward change? Is the return on it high? Is the investment low?

I think this is where we fundamentally disagree. My position is that QEMU is already the authoritative source. Having a state file doesn't change anything.

Do a hot unplug of a network device with upstream libvirt with acpiphp unloaded, consult libvirt and then consult the monitor to see who has the right view of the guests config.

To me, that's the definition of authoritative.

Quoted 1 line

"No" to all three (ignoring for the moment whether it is good or not,

which we were debating).

Quoted 3 lines

Quoted 3 lines

The only suggestion I'm making beyond Marcelo's original patch is

that we use a structured format and that we make it possible to use the same file to solve this problem in multiple places.

Quoted 3 lines

Quoted 1 line

No, you're suggesting a lot more than that.

That's exactly what I'm suggesting from a technical perspective.

Quoted 1 line

Quoted 1 line

I don't think this creates a fundamental break in how management

tools interact with QEMU. I don't think introducing RAID support in the block layer is a reasonable alternative.

Quoted 4 lines

Quoted 2 lines

Why not?

Because its a lot of complexity and code that can go wrong while only solving the race for one specific case. Not to mention that we double the iop rate.

Quoted 3 lines

Something that avoids the whole state thing altogether:

- instead of atomically switching when live copy is done, keep on

issuing writes to both the origin and the live copy

Quoted 2 lines

- issue a notification to management
- management receives the notification, and issues an atomic blockdev

switch command

Quoted 8 lines

Quoted 1 line

this is really the RAID-1 solution but without the state file (credit

Dor). An advantage is that there is no additional latency when trying to catch up to the dirty bitmap.

It still suffers from the two generals problem. You cannot solve this without making one node reliable and that takes us back to it being either QEMU (posted event and state file) or the management tool (sync event).

It is safe w/o a state file by changing the basic live copy algorithm:

1. Live copy in progress stage
Once live copy command is issued, a dirty bit map is created for tracking. There is a single pass over the entire image where we copy blocks from the src to the dst.

Write commands for blocks that were already copied will be done twice for the src and dst.

Once the full copy single pass ends, we trigger a QMP event that this stage can end.

The live copy stage keeps running till the management issue a switch command. When it will happen, the switch is immediate and no need to copy additional blocks (but flush pending IOs).

2. Management sends a switch command.
Qemu stops the doubling the IO and switches to the destination. End.

Here is where your race is:

2. Management sends a switch command

3. QEMU receives switch command

4. QEMU stops doubling IO and switches to the destination

5. QEMU sends acknowledgement of switch command

6. Management receives acknowledge of switch command

7. Management changes internal state definition to reflect the new destination

If QEMU or the management tool crashes after step 4 and before step 6, when the management tool restarts QEMU with the source image, data loss will have occurred (and potentially corruption if a flush had happened).

This all boils down to the Two Generals Problem[1]. It's simply not fixable without making one end reliable and that means that someone needs to fsync() something *after* the switchover happens but before the first write happens. That can be QEMU (Avi's RAID proposal and my state file proposal) or it can be the management tool (if we introduce synchronous events).

Quoted 11 lines · Anthony Liguori

Here is where your race is:

2. Management sends a switch command

3. QEMU receives switch command

4. QEMU stops doubling IO and switches to the destination

5. QEMU sends acknowledgement of switch command

Quoted 5 lines

6. Management receives acknowledge of switch command

7. Management changes internal state definition to reflect the new destination

If QEMU or the management tool crashes after step 4 and before step 6, when the management tool restarts QEMU with the source image, data loss will have occurred (and potentially corruption if a flush had happened).

No. After step 2, any qemu restart will be with the destination image. If the management tool restarts, it can query the state (or just re-issue the switch command, which is idempotent).

Quoted 8 lines · On Feb 28, 2011 11:47 AM, "Avi Kivity" <[email protected]…

Quoted 6 lines · Anthony Liguori

Quoted 4 lines

You're just ignoring what I've written.

No, you're just impervious to my subtle attempt to refocus the

discussion on solving a practical problem.

Quoted 2 lines

Quoted 2 lines

There's a lot of good, reasonably straight forward changes we can

make that have a high return on investment.

Quoted 2 lines

Quoted 1 line

Quoted 1 line

Is making qemu the authoritative source of configuration information

a straightforward change? Is the return on it high? Is the investment low?

I think this is where we fundamentally disagree. My position is that QEMU is already the authoritative source. Having a state file doesn't change anything.

Do a hot unplug of a network device with upstream libvirt with acpiphp unloaded, consult libvirt and then consult the monitor to see who has the right view of the guests config.

To me, that's the definition of authoritative.

Quoted 1 line

"No" to all three (ignoring for the moment whether it is good or

not, which we were debating).

Quoted 2 lines

Quoted 1 line

Quoted 1 line

The only suggestion I'm making beyond Marcelo's original patch is

that we use a structured format and that we make it possible to use the same file to solve this problem in multiple places.

Quoted 3 lines

Quoted 1 line

No, you're suggesting a lot more than that.

That's exactly what I'm suggesting from a technical perspective.

Quoted 1 line

Quoted 1 line

I don't think this creates a fundamental break in how management

tools interact with QEMU. I don't think introducing RAID support in the block layer is a reasonable alternative.

Quoted 4 lines

Quoted 2 lines

Why not?

Because its a lot of complexity and code that can go wrong while only solving the race for one specific case. Not to mention that we double the iop rate.

Quoted 3 lines

Something that avoids the whole state thing altogether:

- instead of atomically switching when live copy is done, keep on

issuing writes to both the origin and the live copy

Quoted 2 lines

- issue a notification to management
- management receives the notification, and issues an atomic

blockdev switch command

Quoted 1 line

this is really the RAID-1 solution but without the state file

(credit Dor). An advantage is that there is no additional latency when trying to catch up to the dirty bitmap.

It still suffers from the two generals problem. You cannot solve this without making one node reliable and that takes us back to it being either QEMU (posted event and state file) or the management tool (sync event).

Quoted 20 lines · Anthony Liguori

Quoted 8 lines · On Feb 28, 2011 11:47 AM, "Avi Kivity" <[email protected]…

Quoted 6 lines · Anthony Liguori

Quoted 4 lines

You're just ignoring what I've written.

No, you're just impervious to my subtle attempt to refocus the

discussion on solving a practical problem.

Quoted 2 lines

Quoted 2 lines

There's a lot of good, reasonably straight forward changes we can

make that have a high return on investment.

Quoted 3 lines

Quoted 1 line

Is making qemu the authoritative source of configuration

information a straightforward change? Is the return on it high? Is the investment low?

Quoted 4 lines

I think this is where we fundamentally disagree. My position is that QEMU is already the authoritative source. Having a state file doesn't change anything.

Do a hot unplug of a network device with upstream libvirt with acpiphp unloaded, consult libvirt and then consult the monitor to see who has the right view of the guests config.

libvirt is right and the monitor is wrong.

On real hardware, calling _EJ0 doesn't affect the configuration one little bit (if I understand it correctly). It just turns off power to the slot. If you power-cycle, the card will be there.

Quoted 10 lines

To me, that's the definition of authoritative.

Quoted 1 line

"No" to all three (ignoring for the moment whether it is good or

not, which we were debating).

Quoted 3 lines

Quoted 3 lines

The only suggestion I'm making beyond Marcelo's original patch is

that we use a structured format and that we make it possible to use the same file to solve this problem in multiple places.

Quoted 2 lines

Quoted 1 line

Quoted 4 lines

Quoted 1 line

No, you're suggesting a lot more than that.

That's exactly what I'm suggesting from a technical perspective.

Unless I'm hallucinating, you're suggesting quite a bit more. A revolution in how qemu is to be managed.

Quoted 32 lines · Avi Kivity

Quoted 22 lines · Anthony Liguori

Quoted 8 lines · On Feb 28, 2011 11:47 AM, "Avi Kivity" <[email protected]…

Quoted 6 lines · Anthony Liguori

Quoted 4 lines

You're just ignoring what I've written.

No, you're just impervious to my subtle attempt to refocus the

discussion on solving a practical problem.

Quoted 2 lines

Quoted 2 lines

There's a lot of good, reasonably straight forward changes we can

make that have a high return on investment.

Quoted 3 lines

Quoted 1 line

Is making qemu the authoritative source of configuration

information a straightforward change? Is the return on it high? Is the investment low?

Quoted 8 lines

I think this is where we fundamentally disagree. My position is that QEMU is already the authoritative source. Having a state file doesn't change anything.

Do a hot unplug of a network device with upstream libvirt with acpiphp unloaded, consult libvirt and then consult the monitor to see who has the right view of the guests config.

Quoted 5 lines

libvirt is right and the monitor is wrong.

On real hardware, calling _EJ0 doesn't affect the configuration one little bit (if I understand it correctly). It just turns off power to the slot. If you power-cycle, the card will be there.

It's up to the hardware vendor. Since it's ACPI, it can result in any number of operations. Usually, there's some logic to flip on an LED or something.

There's nothing that prevents a vendor from ejecting the card. My point is that there aren't cleanly separated lines in the real world.

Quoted 16 lines

Quoted 15 lines

To me, that's the definition of authoritative.

Quoted 1 line

"No" to all three (ignoring for the moment whether it is good or

not, which we were debating).

Quoted 3 lines

Quoted 3 lines

The only suggestion I'm making beyond Marcelo's original patch is

that we use a structured format and that we make it possible to use the same file to solve this problem in multiple places.

Quoted 3 lines

Quoted 1 line

No, you're suggesting a lot more than that.

That's exactly what I'm suggesting from a technical perspective.

Quoted 2 lines

Unless I'm hallucinating, you're suggesting quite a bit more. A revolution in how qemu is to be managed.

Let me take another route to see if I can't persuade you.

First, let's clarify your proposal. You want to introduce a new block format

that references to block devices. It may also store a dirty bitmap to keep track of which blocks are out of sync. Hopefully, it goes without saying that the dirty bitmap is strictly optional (it's a performance optimization) so
let's ignore it.

Your format, as a text file, looks like:

[raid1]
primary=diska.img
secondary=diskb.img
active=primary

To use it, here's the sequence:

0) qemu uses disk A for a block device

1) create a raid1 block device pointing to disk A and disk B.

2) management tool asks qemu to us the new raid1 block device.

3) qemu acks (2)

4) at some point, the mirror completes, writes are going to both disks

5) qemu sends out an event indicating that the disks are in sync

6) management tool then sends a command to fail over to disk B

7) qemu acks (6)

We're making the management tool the "authoritative" source of how to launch
QEMU. That means that the management tool ultimately determines which command
line to relaunch QEMU with.

Here are the races:

A) If QEMU crashes between (2) and (3), it may have issues a write to the new
raid1 block device before the management tool sees (3). If this happens, when the management tool restarts QEMU with disk A, we're left with a dangling raid1 block device. Not a critical failure, but not ideal.

B) If QEMU crashes between (6) and (7), QEMU may have started writing to disk
B before the management tool sees (7). This means that the management tool will create the guest with the raid1 block device which no longer is the correct disk. This could fail in subtly bad ways. Depending on how read is implemented (if you try to do striping for instance), bad data could be returned. You could try to implement a policy of always reading from B if the block has been copied but this gets harry really quickly. It's definitely not RAID1 anymore.

You may observe that the problem is not the RAID1 mechanism, but changing from
using a normal device and the RAID1 mechanism. It would then be wise to say,
let's always use this image format. Since that eliminates the race, we don't
really need the copy bitmap anymore.

Now we're left with a simple format that just refers to two filenames.

However,
block devices are more than just a filename. It needs a format, cache settings, etc. So let's put this all in the RAID1 block format. We also need
a way to indicate which block device is selected.

Let's make it a text file for purposes of discussion. It will look something
like:

[primary]
filename=diska.img
cache=none
format=raw

[secondary]
filename=diskb.img
cache=writethrough
format=qcow2

[global]
active=primary

Since we might want to mirror multiple drives at once, we should probablyn support having multiple drives configured which means we need to not just have
a single active entry, but an entry associated with a particular device.

[drive "diskA"]
filename=diska.img
cache=none
format=raw

[drive "diskB"]
filename=diskb.img
cache=writethrough
format=qcow2

[device "vda"]
drive=diskB

And this is exactly what I'm proposing. It's really the natural generalization
of what you're proposing.

So basically, the only differences are:

1) always use the new RAID1 format
2) drop the progress bitmap
3) support multiple devices per file
4) let drive properties be specified beyond filename

All reasonable things to do.

Regards,

Anthony Liguori

Quoted 6 lines

Quoted 2 lines

Do a hot unplug of a network device with upstream libvirt with acpiphp unloaded, consult libvirt and then consult the monitor to see who has the right view of the guests config.

libvirt is right and the monitor is wrong.

On real hardware, calling _EJ0 doesn't affect the configuration one little bit (if I understand it correctly). It just turns off power to the slot. If you power-cycle, the card will be there.

It's up to the hardware vendor. Since it's ACPI, it can result in any number of operations. Usually, there's some logic to flip on an LED or something.

There's nothing that prevents a vendor from ejecting the card. My point is that there aren't cleanly separated lines in the real world.

Quoted 3 lines

Quoted 1 line

Unless I'm hallucinating, you're suggesting quite a bit more. A revolution in how qemu is to be managed.

Let me take another route to see if I can't persuade you.

First, let's clarify your proposal. You want to introduce a new block format that references to block devices. It may also store a dirty bitmap to keep track of which blocks are out of sync. Hopefully, it goes without saying that the dirty bitmap is strictly optional (it's a performance optimization) so let's ignore it.

Your format, as a text file, looks like:

[raid1]
primary=diska.img
secondary=diskb.img
active=primary

To use it, here's the sequence:

0) qemu uses disk A for a block device

1) create a raid1 block device pointing to disk A and disk B.

2) management tool asks qemu to us the new raid1 block device.

3) qemu acks (2)

4) at some point, the mirror completes, writes are going to both disks

5) qemu sends out an event indicating that the disks are in sync

6) management tool then sends a command to fail over to disk B

7) qemu acks (6)

We're making the management tool the "authoritative" source of how to launch QEMU. That means that the management tool ultimately determines which command line to relaunch QEMU with.

Here are the races:

A) If QEMU crashes between (2) and (3), it may have issues a write to the new raid1 block device before the management tool sees (3). If this
happens,
when the management tool restarts QEMU with disk A, we're left with a dangling raid1 block device. Not a critical failure, but not ideal.

B) If QEMU crashes between (6) and (7), QEMU may have started writing to disk B before the management tool sees (7). This means that the
management tool
will create the guest with the raid1 block device which no longer
is the
correct disk. This could fail in subtly bad ways. Depending on
how read
is implemented (if you try to do striping for instance), bad data
could be
returned. You could try to implement a policy of always reading
from B if
the block has been copied but this gets harry really quickly. It's definitely not RAID1 anymore.

You may observe that the problem is not the RAID1 mechanism, but changing from using a normal device and the RAID1 mechanism. It would then be wise to say, let's always use this image format. Since that eliminates the race, we don't really need the copy bitmap anymore.

Now we're left with a simple format that just refers to two filenames. However, block devices are more than just a filename. It needs a format, cache settings, etc. So let's put this all in the RAID1 block format. We also need a way to indicate which block device is selected.

Let's make it a text file for purposes of discussion. It will look something like:

[primary]
filename=diska.img
cache=none
format=raw

[secondary]
filename=diskb.img
cache=writethrough
format=qcow2

[global]
active=primary

Since we might want to mirror multiple drives at once, we should probablyn support having multiple drives configured which means we need to not just have a single active entry, but an entry associated with a particular device.

[drive "diskA"]
filename=diska.img
cache=none
format=raw

[drive "diskB"]
filename=diskb.img
cache=writethrough
format=qcow2

[device "vda"]
drive=diskB

And this is exactly what I'm proposing.

It's really the natural generalization
of what you're proposing.

So basically, the only differences are:

1) always use the new RAID1 format
2) drop the progress bitmap
3) support multiple devices per file
4) let drive properties be specified beyond filename

All reasonable things to do.

Quoted 5 lines

It's really the natural generalization
of what you're proposing.

So basically, the only differences are:

Quoted 6 lines

1) always use the new RAID1 format
2) drop the progress bitmap
3) support multiple devices per file
4) let drive properties be specified beyond filename

All reasonable things to do.

Well, I dislike 3, and the whole "qemu is authoritative source of configuration" thing.

Quoted 10 lines · Anthony Liguori

Quoted 3 lines

You're just ignoring what I've written.

No, you're just impervious to my subtle attempt to refocus the discussion on solving a practical problem.

There's a lot of good, reasonably straight forward changes we can make that have a high return on investment.

Is making qemu the authoritative source of configuration information a straightforward change? Is the return on it high? Is the investment low?

"No" to all three (ignoring for the moment whether it is good or not, which we were debating).

Quoted 4 lines

The only suggestion I'm making beyond Marcelo's original patch is that we use a structured format and that we make it possible to use the same file to solve this problem in multiple places.

No, you're suggesting a lot more than that.

Quoted 5 lines

I don't think this creates a fundamental break in how management tools interact with QEMU. I don't think introducing RAID support in the block layer is a reasonable alternative.

Why not?

Something that avoids the whole state thing altogether:

- instead of atomically switching when live copy is done, keep on issuing writes to both the origin and the live copy
- issue a notification to management
- management receives the notification, and issues an atomic blockdev switch command

this is really the RAID-1 solution but without the state file (credit Dor). An advantage is that there is no additional latency when trying to catch up to the dirty bitmap.

Quoted 8 lines

Something that avoids the whole state thing altogether:

- instead of atomically switching when live copy is done, keep on issuing writes to both the origin and the live copy
- issue a notification to management
- management receives the notification, and issues an atomic blockdev switch command

How do you know if QEMU performed the switch or not? That is, how can the switch command be atomic?

Quoted 3 lines

this is really the RAID-1 solution but without the state file (credit Dor). An advantage is that there is no additional latency when trying to catch up to the dirty bitmap.

So you're implementing a virtual block driver not visible to the user as an image format. The image format would allow storage of persistent copy progress, etc. so you lose all that.

All of that to avoid introducing a commit file which is not part of global qemu state.

Yes.   I view the command line as optional.  To me, this is the ideal interaction:

1) start qemu with an empty stateful config file

2) issue monitor commands to create all devices and backends

3) the stateful config file totally captures the state of all of the issued QMP commands.  The management tool can relaunch the guest just by passing the stateful config file to QEMU.

4) when the management tool needs to "extract" a config file, it can read the stateful config (through the monitor) and generate it's own config.

5) the management tool should treat the stateful config file as more or less opaque.  It shouldn't be visible to end user.

In the non-managed case, users should interact directly with the config file.

Quoted 17 lines · Anthony Liguori

Yes. I view the command line as optional. To me, this is the ideal
interaction:

1) start qemu with an empty stateful config file

2) issue monitor commands to create all devices and backends

3) the stateful config file totally captures the state of all of the issued QMP commands. The management tool can relaunch the guest just by passing the stateful config file to QEMU.

4) when the management tool needs to "extract" a config file, it can read the stateful config (through the monitor) and generate it's own config.

5) the management tool should treat the stateful config file as more or less opaque. It shouldn't be visible to end user.

Quoted 3 lines

In the non-managed case, users should interact directly with the config file.

For the typical TCG use case, that sounds a whole lot worse than the current situation of "just run qemu with a suitable command line"...

-- PMM

Who I can do it?

Quoted 16 lines · Anthony Liguori

Quoted 9 lines · Avi Kivity

Quoted 4 lines · Anthony Liguori

-drive already ties into the qemuopts infrastructure and we have readconfig and writeconfig. I don't think we're missing any major pieces to do this in a more proper fashion.

The problem with qemu config files is that it splits the authoritative source of where images are stored into two. Is it in the management tool's database or is it in qemu's config file?

I like to use the phrase "stateful config file". To me, it's just a database for QEMU to persist data about the VM. It's the only way for QEMU to make certain transactions atomic in the face of QEMU crashing.

Quoted 4 lines

The user visible config file is a totally different concept. A management tool launches QEMU and tells it where to keep it's state database. The management application may prepopulate the state database or it may just use an empty file.

In that case the word 'config' is misleading. To me, it implies that the user configures something, and qemu reads it, not something mostly internal to qemu.

Qemu does keep state. Currently only images, but in theory also the on-board NVRAM.

Quoted 9 lines

QEMU uses the state database to store information that is created dynamically. For instance, devices added through device_add. A device added via -device wouldn't necessary get added to the state database.

Practically speaking, it let's you invoke QEMU with a fixed command line, while still using the monitor to make changes that would otherwise require the command line being updated.

Then the invoker quickly loses track of what the actual state is. It can't just remember which commands it issued (presumably in response to the user updating user visible state). It has to parse the stateful config file qemu outputs. But at which points should it parse it?

I don't think it's reasonable to have three different ways to interact with qemu, all needed: the command line, reading and writing the stateful config file, and the monitor. I'd rather push for starting qemu with a blank guest and assembling (cold-plugging) all the hardware via the monitor before starting the guest.

Quoted 6 lines

I don't think it's reasonable to have three different ways to interact with qemu, all needed: the command line, reading and writing the stateful config file, and the monitor. I'd rather push for starting qemu with a blank guest and assembling (cold-plugging) all the hardware via the monitor before starting the guest.

Exactly. And qdev has brought his within our reach.

[...]

Quoted 8 lines · Markus Armbruster

Quoted 6 lines

I don't think it's reasonable to have three different ways to interact with qemu, all needed: the command line, reading and writing the stateful config file, and the monitor. I'd rather push for starting qemu with a blank guest and assembling (cold-plugging) all the hardware via the monitor before starting the guest.

Exactly. And qdev has brought his within our reach.

Actually, QMP is the star of the show here, not qdev.

The way we get here is by incrementally converting the option handling to be qmp calls. For instance, instead of:

case QEMU_OPTION_name:

      qemu_name = optarg;
      break;

We do:

case QEMU_OPTION_name:

       qmp_set_name(optarg, NULL);
       break;

When we can compile vl.c with nothing more than QMP dependencies, we've achieved the goals here. This will mean adding a lot of new QMP commands.

There are some command line options that must be handled before the machine is initialized and QMP is normally run. For 0.16, we can introduce a new QMP mode whereas the event loop gets to run before doing machine init and explicit create_machine() command is needed. This is the final bit that will be needed to realize this goal.

A lot of good things come out of this. Quite a lot of these new commands don't strictly need to run before machine init (like -name) which means we'll get the ability to change a lot of parameters without rebooting the guest which couldn't be changed before.

And this is all incremental stuff that can be done in parallel of the QAPI work. We just need to do the work of adding the function calls (or function call wrappers where appropriate).

Quoted 8 lines · Anthony Liguori

QEMU uses the state database to store information that is created dynamically.  For instance, devices added through device_add.  A device added via -device wouldn't necessary get added to the state database.

Practically speaking, it let's you invoke QEMU with a fixed command line, while still using the monitor to make changes that would otherwise require the command line being updated.

Then the invoker quickly loses track of what the actual state is.  It can't just remember which commands it issued (presumably in response to the user updating user visible state).  It has to parse the stateful config file qemu outputs.  But at which points should it parse it?

I don't think it's reasonable to have three different ways to interact with qemu, all needed: the command line, reading and writing the stateful config file, and the monitor.  I'd rather push for starting qemu with a blank guest and assembling (cold-plugging) all the hardware via the monitor before starting the guest.

Quoted 4 lines · Anthony Liguori

-drive already ties into the qemuopts infrastructure and we have readconfig and writeconfig. I don't think we're missing any major pieces to do this in a more proper fashion.

The problem with qemu config files is that it splits the authoritative source of where images are stored into two. Is it in the management tool's database or is it in qemu's config file?

For the problem at hand, one solution is to make qemu stop after the copy, and then management can issue an additional command to rearrange the disk and resume the guest. A drawback here is that if management dies, the guest is stopped until it restarts. We also make management latency guest visible, even if it doesn't die at an inconvenient place.

An alternative approach is to have the copy be performed by a new layered block format driver:

- create a new image, type = live-copy, containing three pieces of information
- source image
- destination image
- copy state (initially nothing is copied)
- tell qemu switch to the new image
- qemu starts copying, updates copy state as needed
- copy finishes, event is emitted; reads and writes still serviced
- management receives event, switches qemu to destination image
- management removes live-copy image

If management dies while this is happening, it can simply query the state of the copy.
Similarly, if qemu dies, the copy state is persistent (could be 0/1 or real range of blocks).

Quoted 31 lines · Avi Kivity

Quoted 4 lines · Anthony Liguori

-drive already ties into the qemuopts infrastructure and we have readconfig and writeconfig. I don't think we're missing any major pieces to do this in a more proper fashion.

The problem with qemu config files is that it splits the authoritative source of where images are stored into two. Is it in the management tool's database or is it in qemu's config file?

For the problem at hand, one solution is to make qemu stop after the copy, and then management can issue an additional command to rearrange the disk and resume the guest. A drawback here is that if management dies, the guest is stopped until it restarts. We also make management latency guest visible, even if it doesn't die at an inconvenient place.

An alternative approach is to have the copy be performed by a new layered block format driver:

- create a new image, type = live-copy, containing three pieces of information
- source image
- destination image
- copy state (initially nothing is copied)
- tell qemu switch to the new image
- qemu starts copying, updates copy state as needed
- copy finishes, event is emitted; reads and writes still serviced
- management receives event, switches qemu to destination image
- management removes live-copy image

Quoted 4 lines

If management dies while this is happening, it can simply query the state of the copy.
Similarly, if qemu dies, the copy state is persistent (could be 0/1 or real range of blocks).

You don't know if a given block is uptodate or not without the dirty bitmap. So unless you also keep track of dirty log somehow, this is meaningless.

So a commit file as proposed indicates copy state (in 0/1 fashion). The difference in your proposal is that such information is stored inside a special purpose image format?

Quoted 21 lines · Marcelo Tosatti

Quoted 20 lines · Avi Kivity

Quoted 4 lines · Anthony Liguori

-drive already ties into the qemuopts infrastructure and we have readconfig and writeconfig. I don't think we're missing any major pieces to do this in a more proper fashion.

The problem with qemu config files is that it splits the authoritative source of where images are stored into two. Is it in the management tool's database or is it in qemu's config file?

For the problem at hand, one solution is to make qemu stop after the copy, and then management can issue an additional command to rearrange the disk and resume the guest. A drawback here is that if management dies, the guest is stopped until it restarts. We also make management latency guest visible, even if it doesn't die at an inconvenient place.

An alternative approach is to have the copy be performed by a new layered block format driver:

Quoted 6 lines

Quoted 6 lines

- create a new image, type = live-copy, containing three pieces of information
- source image
- destination image
- copy state (initially nothing is copied)
- tell qemu switch to the new image