Block live migration
PATCH v5 2/3 Block live migration
This patch introduces block migration called during live migration. Block are being copied to the destination in an async way. First the code will transfer the whole disk and then transfer all dirty blocks accumulted during the migration.
Still need to improve transition from the iterative phase of migration to the end phase. For now transition will take place when all blocks transfered once, all the dirty blocks will be transfered during the end phase (guest is suspended).
Changes from v4:
- Global variabels moved to a global state structure allocated dynamically.
- Minor coding style issues.
- Poll block.c for tracking of dirty blocks instead of manage it here.
Makefile +1 -0
@@ -136,6 +136,7 @@ obj-y += qemu-char.o aio.o savevm.o obj-y += msmouse.o ps2.o obj-y += qdev.o qdev-properties.o obj-y += qint.o qstring.o qdict.o qlist.o qemu-config.o +obj-y += block-migration.o obj-$(CONFIG_BRLAPI) += baum.o obj-$(CONFIG_POSIX) += migration-exec.o migration-unix.o migration-fd.o
block-migration.c new file +558 -0
558 lines not on this page · raw message
block-migration.h new file +29 -0
@@ -0,0 +1,29 @@ +/* +* QEMU live block migration +* +* Copyright IBM, Corp. 2009 +* +* Authors: +* Liran Schour <[email protected]> +* +* This work is licensed under the terms of the GNU GPL, version 2. See +* the COPYING file in the top-level directory. +* +*/ + +#ifndef BLOCK_MIGRATION_H +#define BLOCK_MIGRATION_H + +typedef struct BlkMigDevState { +BlockDriverState *bs; +int bulk_completed; +int shared_base; +struct BlkMigDevState *next; +int64_t cur_sector; +int64_t total_sectors; +int64_t dirty; +}BlkMigDevState ; + +void blk_mig_init (void ); +void blk_mig_info (void ); +#endif /* BLOCK_MIGRATION_H */
buffered_file.c +9 -1
@@ -211,6 +211,13 @@ out:return s->xfer_limit; } +static size_t buffered_get_rate_limit (void *opaque) +{ +QEMUFileBuffered *s = opaque; + +return s->xfer_limit; +} +static void buffered_rate_tick (void *opaque) {QEMUFileBuffered *s = opaque;
@@ -251,7 +258,8 @@ QEMUFile *qemu_fopen_ops_buffered(void *opaque, s->file =qemu_fopen_ops (s, buffered_put_buffer,NULL , buffered_close, buffered_rate_limit, - buffered_set_rate_limit); + buffered_set_rate_limit, + buffered_get_rate_limit); s->timer =qemu_new_timer (rt_clock, buffered_rate_tick, s);
hw/hw.h +7 -2
@@ -43,12 +43,14 @@ typedef int (QEMUFileRateLimit)(void *opaque);* the old rate otherwise */ typedef size_t (QEMUFileSetRateLimit )(void *opaque,size_t new_rate); +typedef size_t (QEMUFileGetRateLimit )(void *opaque);QEMUFile *qemu_fopen_ops (void *opaque,QEMUFilePutBufferFunc *put_buffer,QEMUFileGetBufferFunc *get_buffer,QEMUFileCloseFunc *close,QEMUFileRateLimit *rate_limit, -QEMUFileSetRateLimit *set_rate_limit); +QEMUFileSetRateLimit *set_rate_limit, +QEMUFileGetRateLimit *get_rate_limit);QEMUFile *qemu_fopen (const char *filename,const char *mode);QEMUFile *qemu_fdopen (int fd,const char *mode);QEMUFile *qemu_fopen_socket (int fd);
@@ -85,6 +87,7 @@ unsigned int qemu_get_be32(QEMUFile *f);uint64_t qemu_get_be64 (QEMUFile *f);int qemu_file_rate_limit (QEMUFile *f);size_t qemu_file_set_rate_limit (QEMUFile *f,size_t new_rate); +size_t qemu_file_get_rate_limit (QEMUFile *f);int qemu_file_has_error (QEMUFile *f);void qemu_file_set_error (QEMUFile *f);
@@ -239,6 +242,7 @@ static inline void qemu_get_sbe64s(QEMUFile *f, int64_t *pv)int64_t qemu_ftell (QEMUFile *f);int64_t qemu_fseek (QEMUFile *f,int64_t pos,int whence); +typedef void SaveSetParamsHandler (int blk_enable,int shared,void * opaque);typedef void SaveStateHandler (QEMUFile *f,void *opaque);typedef int SaveLiveStateHandler (QEMUFile *f,int stage,void *opaque);typedef int LoadStateHandler (QEMUFile *f,void *opaque,int version_id);
@@ -253,7 +257,8 @@ int register_savevm(const char *idstr,int register_savevm_live (const char *idstr,int instance_id,int version_id, -SaveLiveStateHandler *save_live_state, +SaveSetParamsHandler *set_params, +SaveLiveStateHandler *save_live_state,SaveStateHandler *save_state,LoadStateHandler *load_state,void *opaque);
migration-exec.c +7 -2
@@ -53,8 +53,10 @@ static int exec_close(FdMigrationState *s) }MigrationState *exec_start_outgoing_migration (const char *command, -int64_t bandwidth_limit, -int detach) +int64_t bandwidth_limit, +int detach, +int blk, +int inc) {FdMigrationState *s;FILE *f;
@@ -84,6 +86,9 @@ MigrationState *exec_start_outgoing_migration(const char *command, s->mig_state.get_status = migrate_fd_get_status; s->mig_state.release = migrate_fd_release; + s->mig_state.blk = blk; + s->mig_state.shared = inc; + s->state =MIG_STATE_ACTIVE ; s->mon_resume =NULL ; s->bandwidth_limit = bandwidth_limit;
migration-fd.c +6 -1
@@ -54,7 +54,9 @@ static int fd_close(FdMigrationState *s)MigrationState *fd_start_outgoing_migration (Monitor *mon,const char *fdname,int64_t bandwidth_limit, -int detach) +int detach, +int blk, +int inc) {FdMigrationState *s;
@@ -78,6 +80,9 @@ MigrationState *fd_start_outgoing_migration(Monitor *mon, s->mig_state.get_status = migrate_fd_get_status; s->mig_state.release = migrate_fd_release; + s->mig_state.blk = blk; + s->mig_state.shared = inc; + s->state =MIG_STATE_ACTIVE ; s->mon_resume =NULL ; s->bandwidth_limit = bandwidth_limit;
migration-tcp.c +6 -1
@@ -78,7 +78,9 @@ static void tcp_wait_for_connect(void *opaque)MigrationState *tcp_start_outgoing_migration (const char *host_port,int64_t bandwidth_limit, -int detach) +int detach, +int blk, +int inc) {struct sockaddr_in addr;FdMigrationState *s;
@@ -96,6 +98,9 @@ MigrationState *tcp_start_outgoing_migration(const char *host_port, s->mig_state.get_status = migrate_fd_get_status; s->mig_state.release = migrate_fd_release; + s->mig_state.blk = blk; + s->mig_state.shared = inc; + s->state =MIG_STATE_ACTIVE ; s->mon_resume =NULL ; s->bandwidth_limit = bandwidth_limit;
migration-unix.c +6 -1
@@ -77,7 +77,9 @@ static void unix_wait_for_connect(void *opaque)MigrationState *unix_start_outgoing_migration (const char *path,int64_t bandwidth_limit, -int detach) +int detach, +int blk, +int inc) {FdMigrationState *s;struct sockaddr_un addr;
@@ -95,6 +97,9 @@ MigrationState *unix_start_outgoing_migration(const char *path, s->mig_state.get_status = migrate_fd_get_status; s->mig_state.release = migrate_fd_release; + s->mig_state.blk = blk; + s->mig_state.shared = inc; + s->state =MIG_STATE_ACTIVE ; s->mon_resume =NULL ; s->bandwidth_limit = bandwidth_limit;
migration.c +16 -7
@@ -58,16 +58,24 @@ void do_migrate(Monitor *mon, const QDict *qdict, QObject **ret_data)const char *p;int detach =qdict_get_int (qdict,"detach" );const char *uri =qdict_get_str (qdict,"uri" ); - +if (strstart (uri,"tcp:" , &p)) - s =tcp_start_outgoing_migration (p, max_throttle, detach); + s =tcp_start_outgoing_migration (p, max_throttle, detach, + (int )qdict_get_int (qdict,"blk" ), + (int )qdict_get_int (qdict,"inc" ));#if !defined(WIN32 ) else if (strstart(uri, "exec:", &p)) - s =exec_start_outgoing_migration (p, max_throttle, detach); + s =exec_start_outgoing_migration (p, max_throttle, detach, + (int )qdict_get_int (qdict,"blk" ), + (int )qdict_get_int (qdict,"inc" )); else if (strstart(uri, "unix:", &p)) - s =unix_start_outgoing_migration (p, max_throttle, detach); + s =unix_start_outgoing_migration (p, max_throttle, detach, + (int )qdict_get_int (qdict,"blk" ), + (int )qdict_get_int (qdict,"inc" )); else if (strstart(uri, "fd:", &p)) - s =fd_start_outgoing_migration (mon, p, max_throttle, detach); + s =fd_start_outgoing_migration (mon, p, max_throttle, detach, + (int )qdict_get_int (qdict,"blk" ), + (int )qdict_get_int (qdict,"inc" ));#endif else monitor_printf(mon, "unknown migration protocol: %s\n", uri);
@@ -251,13 +259,14 @@ void migrate_fd_connect(FdMigrationState *s) migrate_fd_close);dprintf ("beginning savevm\n" ); - ret =qemu_savevm_state_begin (s->file); + ret =qemu_savevm_state_begin (s->file, s->mig_state.blk, + s->mig_state.shared);if (ret <0 ) {dprintf ("failed, %d\n" , ret);migrate_fd_error (s);return ; } - +migrate_fd_put_ready (s); }
migration.h +15 -5
@@ -30,6 +30,8 @@ struct MigrationStatevoid (*cancel)(MigrationState *s);int (*get_status)(MigrationState *s);void (*release)(MigrationState *s); +int blk; +int shared; };typedef struct FdMigrationState FdMigrationState ;
@@ -65,27 +67,35 @@ void do_info_migrate(Monitor *mon);int exec_start_incoming_migration (const char *host_port);MigrationState *exec_start_outgoing_migration (const char *host_port, -int64_t bandwidth_limit, -int detach); +int64_t bandwidth_limit, +int detach, +int blk, +int inc);int tcp_start_incoming_migration (const char *host_port);MigrationState *tcp_start_outgoing_migration (const char *host_port,int64_t bandwidth_limit, -int detach); +int detach, +int blk, +int inc);int unix_start_incoming_migration (const char *path);MigrationState *unix_start_outgoing_migration (const char *path,int64_t bandwidth_limit, -int detach); +int detach, +int blk, +int inc);int fd_start_incoming_migration (const char *path);MigrationState *fd_start_outgoing_migration (Monitor *mon,const char *fdname,int64_t bandwidth_limit, -int detach); +int detach, +int blk, +int inc);void migrate_fd_monitor_suspend (FdMigrationState *s);
savevm.c +46 -16
174 lines not on this page · raw message
sysemu.h +1 -1
@@ -61,7 +61,7 @@ void qemu_announce_self(void);void main_loop_wait (int timeout); -int qemu_savevm_state_begin (QEMUFile *f); +int qemu_savevm_state_begin (QEMUFile *f,int blk_enable,int shared);int qemu_savevm_state_iterate (QEMUFile *f);int qemu_savevm_state_complete (QEMUFile *f);int qemu_savevm_state (QEMUFile *f);
vl.c +6 -3
@@ -148,6 +148,8 @@ int main(int argc, char **argv)#include "qemu-char.h" #include "cache-utils.h" #include "block.h" +#include "block_int.h" +#include "block-migration.h" #include "dma.h" #include "audio/audio.h" #include "migration.h"
@@ -2962,9 +2964,7 @@ static int ram_save_live(QEMUFile *f, int stage, void *opaque) bwidth =0.000001 ;/* try transferring iterative blocks of memory */ -if (stage ==3 ) { -/* flush all remaining blocks regardless of rate limiting */ while (ram_save_block (f) !=0 ) { bytes_transferred +=TARGET_PAGE_SIZE ;
@@ -5513,6 +5513,8 @@ int main(int argc, char **argv, char **envp)bdrv_init (); +blk_mig_init (); +/* we always create the cdrom drive, even if no disk is there */ drive_add (NULL ,CDROM_ALIAS );
@@ -5529,7 +5531,8 @@ int main(int argc, char **argv, char **envp)exit (1 );vmstate_register (0 , &vmstate_timers ,&timers_state); -register_savevm_live ("ram" ,0 ,3 , ram_save_live,NULL , ram_load,NULL ); +register_savevm_live ("ram" ,0 ,3 ,NULL , ram_save_live,NULL , + ram_load,NULL );/* Maintain compatibility with multiple stdio monitors */ if (!strcmp (monitor_devices[0 ],"stdio" )) {
Hi Liran,
trying to understand the code and fixing some cosmetic issues around progress reporting, one potentially performance-relevant question popped up:
Quoted 5 lines · [email protected] wrote:
block-migration.c new file +0
-0
...
block-migration.c line 107
Quoted 12 lines
block-migration.c new file +12
-0+static int mig_read_device_bulk (QEMUFile *f,BlkMigDevState *bms) +{ +int nr_sectors; +int64_t total_sectors, cur_sector =0 ; +BlockDriverState *bs = bms->bs; +BlkMigBlock *blk; + + blk =qemu_malloc (sizeof (BlkMigBlock)); + blk->buf =qemu_malloc (BLOCK_SIZE ); + + cur_sector = bms->cur_sector; + total_sectors =bdrv_getlength (bs) >>SECTOR_BITS ;
Why re-calculating total_sectors here? Specifically bdrv_getlength() can be a non-trivial I/O operation.
And what is the difference to bms->total_sectors which looks a lot like...
[...]
block-migration.c line 272
Quoted 12 lines
block-migration.c new file +12
-0+ +static void init_blk_migration (QEMUFile *f) +{ +BlkMigDevState **pbmds, *bmds; +BlockDriverState *bs; + +for (bs = bdrv_first; bs !=NULL ; bs = bs->next) { +if (bs->type ==BDRV_TYPE_HD ) { + bmds =qemu_mallocz (sizeof (BlkMigDevState)); + bmds->bs = bs; + bmds->bulk_completed =0 ; + bmds->total_sectors =bdrv_getlength (bs) >>SECTOR_BITS ;
...it already contains all the information we need?
Jan
[email protected] wrote on 25/11/2009 00:55:57:
Quoted 2 lines
trying to understand the code and fixing some cosmetic issues around progress reporting, one potentially performance-relevant question popped
up:
Quoted 26 lines
Quoted 5 lines · [email protected] wrote:
block-migration.c new file +0
-0...
block-migration.c line 107
Quoted 12 lines
block-migration.c new file +12
-0+static int mig_read_device_bulk (QEMUFile *f,BlkMigDevState *bms) +{ +int nr_sectors; +int64_t total_sectors, cur_sector =0 ; +BlockDriverState *bs = bms->bs; +BlkMigBlock *blk; + + blk =qemu_malloc (sizeof (BlkMigBlock)); + blk->buf =qemu_malloc (BLOCK_SIZE ); + + cur_sector = bms->cur_sector; + total_sectors =bdrv_getlength (bs) >>SECTOR_BITS ;Why re-calculating
total_sectorshere? Specificallybdrv_getlength()can be a non-trivial I/O operation.
Quoted 1 line
And what is the difference to
bms->total_sectorswhich looks a lot
like...
Quoted 17 lines
[...]
block-migration.c line 272
Quoted 12 lines
block-migration.c new file +12
-0+ +static void init_blk_migration (QEMUFile *f) +{ +BlkMigDevState **pbmds, *bmds; +BlockDriverState *bs; + +for (bs = bdrv_first; bs !=NULL ; bs = bs->next) { +if (bs->type ==BDRV_TYPE_HD ) { + bmds =qemu_mallocz (sizeof (BlkMigDevState)); + bmds->bs = bs; + bmds->bulk_completed =0 ; + bmds->total_sectors =bdrv_getlength (bs) >>SECTOR_BITS ;...it already contains all the information we need?
You are right no need to figure it out once again. We can use the old total_sectors value.
Thanks,
Liran
Quoted 47 lines · Liran Schour
[email protected] wrote on 25/11/2009 00:55:57:
Quoted 2 lines
trying to understand the code and fixing some cosmetic issues around progress reporting, one potentially performance-relevant question popped
up:
Quoted 23 lines
Quoted 5 lines · [email protected] wrote:
block-migration.c new file +0
-0...
block-migration.c line 107
Quoted 12 lines
block-migration.c new file +12
-0+static int mig_read_device_bulk (QEMUFile *f,BlkMigDevState *bms) +{ +int nr_sectors; +int64_t total_sectors, cur_sector =0 ; +BlockDriverState *bs = bms->bs; +BlkMigBlock *blk; + + blk =qemu_malloc (sizeof (BlkMigBlock)); + blk->buf =qemu_malloc (BLOCK_SIZE ); + + cur_sector = bms->cur_sector; + total_sectors =bdrv_getlength (bs) >>SECTOR_BITS ;Why re-calculating
total_sectorshere? Specificallybdrv_getlength()can be a non-trivial I/O operation.Quoted 1 line
And what is the difference to
bms->total_sectorswhich looks a lotlike...
Quoted 15 lines
[...]
block-migration.c line 272
Quoted 12 lines
block-migration.c new file +12
-0+ +static void init_blk_migration (QEMUFile *f) +{ +BlkMigDevState **pbmds, *bmds; +BlockDriverState *bs; + +for (bs = bdrv_first; bs !=NULL ; bs = bs->next) { +if (bs->type ==BDRV_TYPE_HD ) { + bmds =qemu_mallocz (sizeof (BlkMigDevState)); + bmds->bs = bs; + bmds->bulk_completed =0 ; + bmds->total_sectors =bdrv_getlength (bs) >>SECTOR_BITS ;...it already contains all the information we need?
Quoted 2 lines
You are right no need to figure it out once again. We can use the old
total_sectorsvalue.
Perfect.
Meanwhile I've quite a few patches here, including that change, will hopefully manage to send them out today.
Jan
Quoted 24 lines · [email protected] wrote:
block-migration.c new file +24
-0+static int block_load (QEMUFile *f,void *opaque,int version_id) +{ +int len, flags; +char device_name[256 ]; +int64_t addr; +BlockDriverState *bs; +uint8_t *buf; + + block_mig_state->sectors_per_block =bdrv_get_sectors_per_chunk (); + buf =qemu_malloc (BLOCK_SIZE ); + + do { + + addr = qemu_get_be64(f); + + flags = addr & ~SECTOR_MASK; + addr &= SECTOR_MASK; + + if(flags & BLK_MIG_FLAG_DEVICE_BLOCK) { + + /* get device name */ + len = qemu_get_byte(f); + + qemu_get_buffer(f, (uint8_t *)device_name, len);
block-migration.c line 507
Quoted 10 lines
block-migration.c new file +10
-0+ device_name[len] ='\0' ; + + bs =bdrv_find (device_name); + +qemu_get_buffer (f, buf, +BLOCK_SIZE ); +if (bs !=NULL ) { + +bdrv_write (bs, (addr >>SECTOR_BITS ), + buf, block_mig_state->sectors_per_block);
This synchronous write-back translates appears to be the reason for an unusable migration (or restore from snapshot) speed: about 100 KB/s here (ie. 22h for a rather small 8G guest :( ). Did you already try to improve this situation?
Jan
Signature · 2 lines
Siemens AG, Corporate Technology, CT T DE IT 1 Corporate Competence Center Embedded Linux
Jan Kiszka <[email protected]> wrote on 26/11/2009 15:53:49:
Quoted 11 lines
block-migration.c line 507
Quoted 6 lines
block-migration.c new file +6
-0+qemu_get_buffer (f, buf, +BLOCK_SIZE ); +if (bs !=NULL ) { + +bdrv_write (bs, (addr >>SECTOR_BITS ), + buf, block_mig_state->sectors_per_block);This synchronous write-back translates appears to be the reason for an unusable migration (or restore from snapshot) speed: about 100 KB/s here (ie. 22h for a rather small 8G guest :( ). Did you already try to improve this situation?
I have seen this behavior, but it seems that there is a very big difference in the performance if the new block device is based on an allocated file already (try the same migration to an already allocated file in the requested size) I am trying to figure out why we see this behavior.(any ideas?)
Anyway we can turn the writes to be async but we have to synchronize all destinations writes before completing the migration and moving the guest to the destination. When the guest starts to run on the destination all writes should be finished, so anyhow we need to wait synchronously to the writes. I will look on this further next week.
- Liran
Quoted 14 lines · Liran Schour
Jan Kiszka <[email protected]> wrote on 26/11/2009 15:53:49:
Quoted 10 lines
block-migration.c line 507
Quoted 6 lines
block-migration.c new file +6
-0+qemu_get_buffer (f, buf, +BLOCK_SIZE ); +if (bs !=NULL ) { + +bdrv_write (bs, (addr >>SECTOR_BITS ), + buf, block_mig_state->sectors_per_block);This synchronous write-back translates appears to be the reason for an unusable migration (or restore from snapshot) speed: about 100 KB/s here (ie. 22h for a rather small 8G guest :( ). Did you already try to improve this situation?
Quoted 5 lines
I have seen this behavior, but it seems that there is a very big difference in the performance if the new block device is based on an allocated file already (try the same migration to an already allocated file in the requested size) I am trying to figure out why we see this behavior.(any ideas?)
Yes, much faster, more than 6 MB/s. Not really impressing, but that's now likely due to the tiny block size.
That 4K also made the unallocated write so awfully slow as the image file had to be continuously extended by this amount.
Quoted 5 lines
Anyway we can turn the writes to be async but we have to synchronize all destinations writes before completing the migration and moving the guest to the destination. When the guest starts to run on the destination all writes should be finished, so anyhow we need to wait synchronously to the writes. I will look on this further next week.
Well, we actually need these changes:
1. Reasonable block size (if we cannot overcome the cluster size, we need to coalesce blocks on write out)
2. Async write, throttled by the sync read, or both asynchronously. Unfortunately, qemu's aio does not work yet at the point we need it...
I will also continue to dig into this.
Jan
Signature · 2 lines
Siemens AG, Corporate Technology, CT T DE IT 1 Corporate Competence Center Embedded Linux
Quoted 32 lines · Jan Kiszka
Quoted 13 lines · Liran Schour
Jan Kiszka <[email protected]> wrote on 26/11/2009 15:53:49:
Quoted 10 lines
block-migration.c line 507
Quoted 6 lines
block-migration.c new file +6
-0+qemu_get_buffer (f, buf, +BLOCK_SIZE ); +if (bs !=NULL ) { + +bdrv_write (bs, (addr >>SECTOR_BITS ), + buf, block_mig_state->sectors_per_block);This synchronous write-back translates appears to be the reason for an unusable migration (or restore from snapshot) speed: about 100 KB/s here (ie. 22h for a rather small 8G guest :( ). Did you already try to improve this situation?
Quoted 5 lines
I have seen this behavior, but it seems that there is a very big difference in the performance if the new block device is based on an allocated file already (try the same migration to an already allocated file in the requested size) I am trying to figure out why we see this behavior.(any ideas?)
Yes, much faster, more than 6 MB/s. Not really impressing, but that's now likely due to the tiny block size.
That 4K also made the unallocated write so awfully slow as the image file had to be continuously extended by this amount.
Quoted 5 lines
Anyway we can turn the writes to be async but we have to synchronize all destinations writes before completing the migration and moving the guest to the destination. When the guest starts to run on the destination all writes should be finished, so anyhow we need to wait synchronously to the writes. I will look on this further next week.
Quoted 3 lines
Well, we actually need these changes:
1. Reasonable block size (if we cannot overcome the cluster size, we need to coalesce blocks on write out)
If I understood the logic correctly we can easily increase it during phase 2, ie. while in async mode. We just need to save the block size in its header if we vary it.
Quoted 8 lines
2. Async write, throttled by the sync read, or both asynchronously. Unfortunately, qemu's aio does not work yet at the point we need it...
I will also continue to dig into this.
Jan
At this chance I realized that the length of the synchronous phase 3 depends on the number of dirty blocks collected during phase 2. Thus, the guest downtime will increase when it is under load, ie. typically at the point when the down-time must be minimal. We likely need an approach that takes the number of dirty sectors into account.
Jan
Signature · 2 lines
Siemens AG, Corporate Technology, CT T DE IT 1 Corporate Competence Center Embedded Linux
Jan Kiszka <[email protected]> wrote on 26/11/2009 19:24:40:
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Review
Quoted 2 lines
(quoted diff) +2
-0+ bdrv_write(bs, (addr >> SECTOR_BITS), + buf, block_mig_state->
sectors_per_block);
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
This synchronous write-back translates appears to be the reason for
an
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
unusable migration (or restore from snapshot) speed: about 100 KB/s
here
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 2 lines
(ie. 22h for a rather small 8G guest :( ). Did you already try to improve this situation?
I have seen this behavior, but it seems that there is a very big
difference
Quoted 1 line
Quoted 1 line
Quoted 1 line
in the performance if the new block device is based on an allocated
file
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
already (try the same migration to an already allocated file in the requested size) I am trying to figure out why we see this
behavior.(any
Quoted 9 lines
Quoted 9 lines
Quoted 1 line
ideas?)
Yes, much faster, more than 6 MB/s. Not really impressing, but that's now likely due to the tiny block size.
That 4K also made the unallocated write so awfully slow as the image file had to be continuously extended by this amount.
Quoted 1 line
Anyway we can turn the writes to be async but we have to synchronize
all
Quoted 1 line
Quoted 1 line
Quoted 1 line
destinations writes before completing the migration and moving the
guest to
Quoted 1 line
Quoted 1 line
Quoted 1 line
the destination. When the guest starts to run on the destination all
writes
Quoted 1 line
Quoted 1 line
Quoted 1 line
should be finished, so anyhow we need to wait synchronously to the
writes.
Quoted 9 lines
Quoted 5 lines
Quoted 1 line
I will look on this further next week.
Well, we actually need these changes:
1. Reasonable block size (if we cannot overcome the cluster size, we need to coalesce blocks on write out)If I understood the logic correctly we can easily increase it during phase 2, ie. while in async mode. We just need to save the block size in its header if we vary it.
Right, no problem to increase the block size or to use changed block size.
Quoted 6 lines
Quoted 6 lines
2. Async write, throttled by the sync read, or both asynchronously. Unfortunately, qemu's aio does not work yet at the point we need it...
I will also continue to dig into this.
Quoted 8 lines
Quoted 2 lines
Jan
At this chance I realized that the length of the synchronous phase 3 depends on the number of dirty blocks collected during phase 2. Thus, the guest downtime will increase when it is under load, ie. typically at the point when the down-time must be minimal. We likely need an approach that takes the number of dirty sectors into account.
True. It is in my todo list. We can do something similar to what is done while migrating RAM.
- Liran
Quoted 34 lines · Liran Schour
Jan Kiszka <[email protected]> wrote on 26/11/2009 19:24:40:
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Review
Quoted 2 lines
(quoted diff) +2
-0+ bdrv_write(bs, (addr >> SECTOR_BITS), + buf, block_mig_state->
sectors_per_block);Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
This synchronous write-back translates appears to be the reason for
an
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
unusable migration (or restore from snapshot) speed: about 100 KB/s
here
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 2 lines
(ie. 22h for a rather small 8G guest :( ). Did you already try to improve this situation?
I have seen this behavior, but it seems that there is a very big
difference
Quoted 1 line
Quoted 1 line
Quoted 1 line
in the performance if the new block device is based on an allocated
file
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
already (try the same migration to an already allocated file in the requested size) I am trying to figure out why we see this
behavior.(any
Quoted 8 lines
Quoted 8 lines
Quoted 1 line
ideas?)
Yes, much faster, more than 6 MB/s. Not really impressing, but that's now likely due to the tiny block size.
That 4K also made the unallocated write so awfully slow as the image file had to be continuously extended by this amount.
Quoted 1 line
Anyway we can turn the writes to be async but we have to synchronize
all
Quoted 1 line
Quoted 1 line
Quoted 1 line
destinations writes before completing the migration and moving the
guest to
Quoted 1 line
Quoted 1 line
Quoted 1 line
the destination. When the guest starts to run on the destination all
writes
Quoted 1 line
Quoted 1 line
Quoted 1 line
should be finished, so anyhow we need to wait synchronously to the
writes.
Quoted 1 line
Quoted 1 line
Quoted 1 line
I will look on this further next week.
Quoted 9 lines
Quoted 6 lines
Quoted 3 lines
Well, we actually need these changes:
1. Reasonable block size (if we cannot overcome the cluster size, we need to coalesce blocks on write out)If I understood the logic correctly we can easily increase it during phase 2, ie. while in async mode. We just need to save the block size in its header if we vary it.
Right, no problem to increase the block size or to use changed block size.
Yep, I'm using such a patch now: 1 MB block size already helps a lot.
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
2. Async write, throttled by the sync read, or both asynchronously. Unfortunately, qemu's aio does not work yet at the point we need it...
I will also continue to dig into this.
Jan
Quoted 8 lines
Quoted 5 lines
At this chance I realized that the length of the synchronous phase 3 depends on the number of dirty blocks collected during phase 2. Thus, the guest downtime will increase when it is under load, ie. typically at the point when the down-time must be minimal. We likely need an approach that takes the number of dirty sectors into account.
True. It is in my todo list. We can do something similar to what is done while migrating RAM.
FYI, git://git.kiszka.org/qemu.git queues/migration contains my current patch queue.
I was about to post, but now I'm back into debugging. The issue (EAGAIN when reading a snapshot via "-incoming exec:'cat snapshot'") may or may not be related to my changes. Once it's resolved, I will post officially, but maybe you already want to have a look.
Jan
Signature · 2 lines
Siemens AG, Corporate Technology, CT T DE IT 1 Corporate Competence Center Embedded Linux
Quoted 16 lines · Liran Schour
Jan Kiszka <[email protected]> wrote on 26/11/2009 15:53:49:
Quoted 11 lines
block-migration.c line 507
Quoted 6 lines
block-migration.c new file +6
-0+qemu_get_buffer (f, buf, +BLOCK_SIZE ); +if (bs !=NULL ) { + +bdrv_write (bs, (addr >>SECTOR_BITS ), + buf, block_mig_state->sectors_per_block);This synchronous write-back translates appears to be the reason for an unusable migration (or restore from snapshot) speed: about 100 KB/s here (ie. 22h for a rather small 8G guest :( ). Did you already try to improve this situation?
Quoted 10 lines
I have seen this behavior, but it seems that there is a very big difference in the performance if the new block device is based on an allocated file already (try the same migration to an already allocated file in the requested size) I am trying to figure out why we see this behavior.(any ideas?)
Anyway we can turn the writes to be async but we have to synchronize all destinations writes before completing the migration and moving the guest to the destination. When the guest starts to run on the destination all writes should be finished, so anyhow we need to wait synchronously to the writes. I will look on this further next week.
You second patchset for this feature (around September 10) was much faster than the others. Do you remember what was different?
Signature · 1 line
Pierre Riteau -- http://perso.univ-rennes1.fr/pierre.riteau/