Add support for fixed ram offsets during migration
cover Add support for fixed ram offsets during migration
This series implements what I call "fixed-ram" capability (I'm happy to change the name if people deem it necessary). The idea is that while performing ram migration each dirty page will have a specific offset in the resulting file. The main benefit this has is that now when migrating to a file it will have an upper limit to its size since a page that has been dirtied multiple times won't be saved multiple times in the stream. More details about the actual stream changes necessary to accommodate this can be found in patch 8.
Furthermore I intend on using this code as a foundation for implementing DIO-based save directly from qemu. The current iteration also introduces the "file:" URI to perform the save/restore to a file, however in the past there was a discussion that instead of a file an externally passed FD can be used.
What's missing/pending improvements:
- No tests, I've yet to spend some time with qtest to introduce some tests, will work on this based on the interest the series garners
- I feel the restore is somewhat suboptimal, namely that each and every page is read individually, probably there are some effects of read-ahead caching from linux that we are gaining but I feel it'd be more efficient if it'll be possible to read multiple pages provided they are sequential ( I have to check what support is in there in the bitmap ops to scan for contiguous ranges of sets bits)
- Finish support in analyze-migration.py
- Possibly implement the "external fd" support for "fixed-ram" capability.
All feedback is welcome.
Nikolay Borisov (11):
migration: support file: uri for source migration
migration: Add support for 'file:' uri for incoming migration
migration: Make migration json writer part of MigrationState struct
io: add pwritev support to QIOChannelFile
io: Add support for seekable channels
io: Add preadv support to QIOChannelFile
migration: add qemu_get_buffer_at
migration/ram: Introduce 'fixed-ram' migration stream capability
migration: Refactor precopy ram loading code
migration: Add support for 'fixed-ram' migration restore analyze-migration.py: add initial support for fixed ram streams
include/exec/ramblock.h | 7 + include/io/channel-file.h | 10 + include/io/channel.h | 1 + include/migration/qemu-file-types.h | 2 + io/channel-file.c | 55 +++++ migration/file.c | 38 ++++ migration/file.h | 10 + migration/meson.build | 1 + migration/migration.c | 61 +++++- migration/migration.h | 6 + migration/qemu-file.c | 82 +++++++ migration/qemu-file.h | 4 + migration/ram.c | 322 +++++++++++++++++++++------- migration/savevm.c | 39 ++-- qapi/migration.json | 2 +- scripts/analyze-migration.py | 49 ++++- 16 files changed, 587 insertions(+), 102 deletions(-) create mode 100644 migration/file.c create mode 100644 migration/file.h
--
2.34.1
PATCH 1/11 migration: support file: uri for source migration
Implement support for a "file:" uri so that a migration can be initiated directly to a file from QEMU.
--- migration/file.c | 23 +++++++++++++++++++++++ migration/file.h | 9 +++++++++ migration/meson.build | 1 + migration/migration.c | 3 +++ 4 files changed, 36 insertions(+) create mode 100644 migration/file.c create mode 100644 migration/file.h
migration/file.c new file +23 -0
@@ -0,0 +1,23 @@ +#include "qemu/osdep.h" +#include "channel.h" +#include "io/channel-file.h" +#include "file.h" +#include "qemu/error-report.h" + + +void file_start_outgoing_migration (MigrationState *s,const char *fname,Error **errp) +{ +QIOChannelFile *ioc; + + ioc =qio_channel_file_new_path (fname,O_CREAT |O_TRUNC |O_WRONLY ,0660 , errp); +if (!ioc) { +error_report ("Error creating a channel" ); +return ; + } + +qio_channel_set_name (QIO_CHANNEL (ioc),"migration-file-outgoing" ); +migration_channel_connect (s,QIO_CHANNEL (ioc),NULL ,NULL ); +object_unref (OBJECT (ioc)); +} + +
migration/file.h new file +9 -0
@@ -0,0 +1,9 @@ +#ifndef QEMU_MIGRATION_FILE_H +#define QEMU_MIGRATION_FILE_H + +void file_start_outgoing_migration (MigrationState *s, +const char *filename, +Error **errp); + +#endif +
migration/meson.build +1 -0
@@ -17,6 +17,7 @@ softmmu_ss.add(files( 'colo.c', 'exec.c', 'fd.c', + 'file.c', 'global_state.c', 'migration.c', 'multifd.c',
migration/migration.c +3 -0
@@ -20,6 +20,7 @@#include "migration/blocker.h" #include "exec.h" #include "fd.h" +#include "file.h" #include "socket.h" #include "sysemu/runstate.h" #include "sysemu/sysemu.h"
@@ -2414,6 +2415,8 @@ void qmp_migrate(const char *uri, bool has_blk, bool blk,exec_start_outgoing_migration (s, p, &local_err); }else if (strstart (uri,"fd:" , &p)) {fd_start_outgoing_migration (s, p, &local_err); + }else if (strstart (uri,"file:" , &p)) { +file_start_outgoing_migration (s, p, &local_err); }else {if (!(has_resume && resume)) {yank_unregister_instance (MIGRATION_YANK_INSTANCE );
Signature · 1 line
2.34.1
PATCH 2/11 migration: Add support for 'file:' uri for incoming migration
In reply to Nikolay BorisovThis is a counterpart to the 'file:' uri support for source migration, now a file can also serve as the source of an incoming migration.
--- migration/file.c | 15 +++++++++++++++ migration/file.h | 1 + migration/migration.c | 2 ++ 3 files changed, 18 insertions(+)
migration/file.c +15 -0
@@ -21,3 +21,18 @@ void file_start_outgoing_migration(MigrationState *s, const char *fname, Error * } +void file_start_incoming_migration (const char *fname,Error **errp) +{ +QIOChannelFile *ioc; + + ioc =qio_channel_file_new_path (fname,O_RDONLY ,0 , errp); +if (!ioc) { +error_report ("Error creating a channel" ); +return ; + } + +qio_channel_set_name (QIO_CHANNEL (ioc),"migration-file-incoming" ); +migration_channel_process_incoming (QIO_CHANNEL (ioc)); +object_unref (OBJECT (ioc)); +} +
migration/file.h +1 -0
@@ -5,5 +5,6 @@ void file_start_outgoing_migration(MigrationState *s,const char *filename, Error **errp); +void file_start_incoming_migration (const char *fname,Error **errp);#endif
migration/migration.c +2 -0
@@ -506,6 +506,8 @@ static void qemu_start_incoming_migration(const char *uri, Error **errp)exec_start_incoming_migration (p, errp); }else if (strstart (uri,"fd:" , &p)) {fd_start_incoming_migration (p, errp); + }else if (strstart (uri,"file:" , &p)) { +file_start_incoming_migration (p, errp); }else {error_setg (errp,"unknown migration protocol: %s" , uri); }
Signature · 1 line
2.34.1
PATCH 3/11 migration: Make migration json writer part of MigrationState struct
In reply to Nikolay BorisovThis is required so that migration stream configuration is written to the migration stream. This would allow analyze-migration to parse enabled capabilities for the migration and adjust its behavior accordingly. This is in preparation for analyze-migration.py to support 'fixed-ram' capability format changes.
--- migration/migration.c | 5 +++++ migration/migration.h | 3 +++ migration/savevm.c | 38 ++++++++++++++++++++++---------------- 3 files changed, 30 insertions(+), 16 deletions(-)
migration/migration.c +5 -0
@@ -1896,6 +1896,8 @@ static void migrate_fd_cleanup(MigrationState *s)g_free (s->hostname); s->hostname =NULL ; +json_writer_free (s->vmdesc); +qemu_savevm_state_cleanup ();if (s->to_dst_file) {
@@ -2154,6 +2156,7 @@ void migrate_init(MigrationState *s)error_free (s->error); s->error =NULL ; s->hostname =NULL ; + s->vmdesc =NULL ;migrate_set_state (&s->state,MIGRATION_STATUS_NONE ,MIGRATION_STATUS_SETUP );
@@ -4269,6 +4272,8 @@ void migrate_fd_connect(MigrationState *s, Error *error_in)return ; } + s->vmdesc =json_writer_new (false); +if (multifd_save_setup (&local_err) !=0 ) {error_report_err (local_err); migrate_set_state(&s->state, MIGRATION_STATUS_SETUP,
migration/migration.h +3 -0
@@ -17,6 +17,7 @@#include "exec/cpu-common.h" #include "hw/qdev-core.h" #include "qapi/qapi-types-migration.h" +#include "qapi/qmp/json-writer.h" #include "qemu/thread.h" #include "qemu/coroutine_int.h" #include "io/channel.h"
@@ -261,6 +262,8 @@ struct MigrationState {int state; +JSONWriter *vmdesc; +/* State related to return path */ struct {/* Protected by qemu_file_lock */
migration/savevm.c +22 -16
@@ -1137,13 +1137,18 @@ void qemu_savevm_non_migratable_list(strList **reasons)void qemu_savevm_state_header (QEMUFile *f) { +MigrationState *s =migrate_get_current ();trace_savevm_state_header ();qemu_put_be32 (f,QEMU_VM_FILE_MAGIC );qemu_put_be32 (f,QEMU_VM_FILE_VERSION ); -if (migrate_get_current ()->send_configuration) { +if (s->send_configuration) {qemu_put_byte (f,QEMU_VM_CONFIGURATION ); -vmstate_save_state (f, &vmstate_configuration, &savevm_state,0 ); +json_writer_start_object (s->vmdesc,NULL ); +json_writer_start_object (s->vmdesc,"configuration" ); +vmstate_save_state (f, &vmstate_configuration, &savevm_state, s->vmdesc); +json_writer_end_object (s->vmdesc); + } }
@@ -1364,15 +1369,16 @@ int qemu_savevm_state_complete_precopy_non_iterable(QEMUFile *f, bool in_postcopy, bool inactivate_disks) { -g_autoptr (JSONWriter ) vmdesc =NULL ; +MigrationState *s =migrate_get_current ();int vmdesc_len;SaveStateEntry *se;int ret; - vmdesc =json_writer_new (false); -json_writer_start_object (vmdesc,NULL ); -json_writer_int64 (vmdesc,"page_size" ,qemu_target_page_size ()); -json_writer_start_array (vmdesc,"devices" ); +if (!s->send_configuration) { +json_writer_start_object (s->vmdesc,NULL ); + } +json_writer_int64 (s->vmdesc,"page_size" ,qemu_target_page_size ()); +json_writer_start_array (s->vmdesc,"devices" ); QTAILQ_FOREACH(se, &savevm_state.handlers, entry) {if ((!se->ops || !se->ops->save_state) && !se->vmsd) {
@@ -1385,12 +1391,12 @@ int qemu_savevm_state_complete_precopy_non_iterable(QEMUFile *f,trace_savevm_section_start (se->idstr, se->section_id); -json_writer_start_object (vmdesc,NULL ); -json_writer_str (vmdesc,"name" , se->idstr); -json_writer_int64 (vmdesc,"instance_id" , se->instance_id); +json_writer_start_object (s->vmdesc,NULL ); +json_writer_str (s->vmdesc,"name" , se->idstr); +json_writer_int64 (s->vmdesc,"instance_id" , se->instance_id);save_section_header (f, se,QEMU_VM_SECTION_FULL ); - ret =vmstate_save (f, se, vmdesc); + ret =vmstate_save (f, se, s->vmdesc);if (ret) {qemu_file_set_error (f, ret);return ret;
@@ -1398,7 +1404,7 @@ int qemu_savevm_state_complete_precopy_non_iterable(QEMUFile *f,trace_savevm_section_end (se->idstr, se->section_id,0 );save_section_footer (f, se); -json_writer_end_object (vmdesc); +json_writer_end_object (s->vmdesc); }if (inactivate_disks) {
@@ -1417,14 +1423,14 @@ int qemu_savevm_state_complete_precopy_non_iterable(QEMUFile *f,qemu_put_byte (f,QEMU_VM_EOF ); } -json_writer_end_array (vmdesc); -json_writer_end_object (vmdesc); - vmdesc_len =strlen (json_writer_get (vmdesc)); +json_writer_end_array (s->vmdesc); +json_writer_end_object (s->vmdesc); + vmdesc_len =strlen (json_writer_get (s->vmdesc));if (should_send_vmdesc ()) {qemu_put_byte (f,QEMU_VM_VMDESCRIPTION );qemu_put_be32 (f, vmdesc_len); -qemu_put_buffer (f, (uint8_t *)json_writer_get (vmdesc), vmdesc_len); +qemu_put_buffer (f, (uint8_t *)json_writer_get (s->vmdesc), vmdesc_len); }return 0 ;
Signature · 1 line
2.34.1
PATCH 4/11 io: add pwritev support to QIOChannelFile
In reply to Nikolay BorisovThe upcoming 'fixed-ram' feature would require qemu to write data at specific offsets of the file. This is currently missing so this patch adds it. I've chosen to implement it as a type-specific function rather than plumb it through the generic channel interface as it relies on being able to do seeks.
--- include/io/channel-file.h | 5 +++++ io/channel-file.c | 24 ++++++++++++++++++++++++ 2 files changed, 29 insertions(+)
include/io/channel-file.h +5 -0
@@ -89,4 +89,9 @@ qio_channel_file_new_path(const char *path,mode_t mode, Error **errp); +ssize_t qio_channel_file_pwritev (QIOChannel *ioc, +const struct iovec *iov, +size_t niov, +off_t offset, +Error **errp);#endif /* QIO_CHANNEL_FILE_H */
io/channel-file.c +24 -0
@@ -136,6 +136,30 @@ static ssize_t qio_channel_file_writev(QIOChannel *ioc,return ret; } +ssize_t qio_channel_file_pwritev (QIOChannel *ioc, +const struct iovec *iov, +size_t niov, +off_t offset, +Error **errp) +{ +QIOChannelFile *fioc =QIO_CHANNEL_FILE (ioc); +ssize_t ret; + + retry: + ret =pwritev (fioc->fd, iov, niov, offset); +if (ret <=0 ) { +if (errno ==EAGAIN ) { +return QIO_CHANNEL_ERR_BLOCK ; + } +if (errno ==EINTR ) { + goto retry; + } +error_setg_errno (errp, errno,"Unable to write to file" ); +return -1 ; + } +return ret; +} +static int qio_channel_file_set_blocking (QIOChannel *ioc,bool enabled,Error **errp)
Signature · 1 line
2.34.1
PATCH 5/11 io: Add support for seekable channels
In reply to Nikolay BorisovAdd a bunch of auxiliarry methods and a feature flag to work with SEEKABLE channels. Currently the only channel considered seekable is QIOChannelFile. Also add a bunch of helper functions to QEMUFile that can make use of this channel feature. All of this is in prepration for supporting 'fixed-ram' migration stream feature.
--- include/io/channel.h | 1 + include/migration/qemu-file-types.h | 2 + io/channel-file.c | 5 +++ migration/qemu-file.c | 59 +++++++++++++++++++++++++++++ migration/qemu-file.h | 3 ++ 5 files changed, 70 insertions(+)
include/io/channel.h +1 -0
@@ -41,6 +41,7 @@ enum QIOChannelFeature {QIO_CHANNEL_FEATURE_SHUTDOWN ,QIO_CHANNEL_FEATURE_LISTEN ,QIO_CHANNEL_FEATURE_WRITE_ZERO_COPY , + QIO_CHANNEL_FEATURE_SEEKABLE, };
include/migration/qemu-file-types.h +2 -0
@@ -50,6 +50,8 @@ unsigned int qemu_get_be16(QEMUFile *f);unsigned int qemu_get_be32 (QEMUFile *f);uint64_t qemu_get_be64 (QEMUFile *f); +bool qemu_file_is_seekable (QEMUFile *f); +static inline void qemu_put_be64s (QEMUFile *f,const uint64_t *pv) {qemu_put_be64 (f, *pv);
io/channel-file.c +5 -0
@@ -35,6 +35,7 @@ qio_channel_file_new_fd(int fd) ioc->fd = fd; +qio_channel_set_feature (QIO_CHANNEL (ioc),QIO_CHANNEL_FEATURE_SEEKABLE );trace_qio_channel_file_new_fd (ioc, fd);return ioc;
@@ -59,6 +60,10 @@ qio_channel_file_new_path(const char *path,return NULL ; } +if (lseek (ioc->fd,0 ,SEEK_CUR ) != (off_t)-1 ) { +qio_channel_set_feature (QIO_CHANNEL (ioc),QIO_CHANNEL_FEATURE_SEEKABLE ); + } +trace_qio_channel_file_new_path (ioc, path, flags, mode, ioc->fd);return ioc;
migration/qemu-file.c +59 -0
77 lines not on this page · raw message
migration/qemu-file.h +3 -0
@@ -148,6 +148,9 @@ int qemu_file_shutdown(QEMUFile *f);QEMUFile *qemu_file_get_return_path (QEMUFile *f);void qemu_fflush (QEMUFile *f);void qemu_file_set_blocking (QEMUFile *f,bool block); +void qemu_set_offset (QEMUFile *f,off_t off,int whence); +off_t qemu_get_offset (QEMUFile *f); +void qemu_put_buffer_at (QEMUFile *f,const uint8_t *buf,size_t buflen,off_t pos);void ram_control_before_iterate (QEMUFile *f,uint64_t flags);void ram_control_after_iterate (QEMUFile *f,uint64_t flags);
Signature · 1 line
2.34.1
PATCH 6/11 io: Add preadv support to QIOChannelFile
In reply to Nikolay Borisovpreadv is going to be needed when 'fixed-ram'-enabled stream are to be restored. Simply add a wrapper around preadv that's specific to QIOChannelFile.
--- include/io/channel-file.h | 5 +++++ io/channel-file.c | 26 ++++++++++++++++++++++++++ 2 files changed, 31 insertions(+)
include/io/channel-file.h +5 -0
@@ -89,6 +89,11 @@ qio_channel_file_new_path(const char *path,mode_t mode, Error **errp); +ssize_t qio_channel_file_preadv (QIOChannel *ioc, +const struct iovec *iov, +size_t niov, +off_t offset, +Error **errp); ssize_t qio_channel_file_pwritev(QIOChannel *ioc, const struct iovec *iov, size_t niov,
io/channel-file.c +26 -0
@@ -141,6 +141,32 @@ static ssize_t qio_channel_file_writev(QIOChannel *ioc,return ret; } +ssize_t qio_channel_file_preadv (QIOChannel *ioc, +const struct iovec *iov, +size_t niov, +off_t offset, +Error **errp) +{ +QIOChannelFile *fioc =QIO_CHANNEL_FILE (ioc); +ssize_t ret; + + retry: + ret =preadv (fioc->fd, iov, niov, offset); +if (ret <0 ) { +if (errno ==EAGAIN ) { +return QIO_CHANNEL_ERR_BLOCK ; + } +if (errno ==EINTR ) { + goto retry; + } + +error_setg_errno (errp, errno,"Unable to read from file" ); +return -1 ; + } + +return ret; +} + ssize_t qio_channel_file_pwritev(QIOChannel *ioc, const struct iovec *iov, size_t niov,
Signature · 1 line
2.34.1
PATCH 7/11 migration: add qemu_get_buffer_at
In reply to Nikolay BorisovRestoring a 'fixed-ram' enabled migration stream would require reading from specific offsets in the file so add a helper to QEMUFile that uses the newly introduced qio_channel_file_preadv.
--- migration/qemu-file.c | 23 +++++++++++++++++++++++ migration/qemu-file.h | 1 + 2 files changed, 24 insertions(+)
migration/qemu-file.c +23 -0
@@ -564,6 +564,29 @@ void qemu_put_buffer_at(QEMUFile *f, const uint8_t *buf, size_t buflen, off_t poreturn ; } + +size_t qemu_get_buffer_at (QEMUFile *f,const uint8_t *buf,size_t buflen,off_t pos) +{ +Error *err =NULL ; +struct iovec iov = { .iov_base = (char *)buf, .iov_len = buflen }; +ssize_t ret; + +if (f->last_error) { +return 0 ; + } + + ret =qio_channel_file_preadv (f->ioc, &iov,1 , pos, &err); +if (ret ==-1 ) { + goto error; + } + +return (size_t )ret; + + error: +qemu_file_set_error_obj (f, -EIO , err); +return 0 ; +} +void qemu_set_offset (QEMUFile *f,off_t off,int whence) {Error *err =NULL ;
migration/qemu-file.h +1 -0
@@ -151,6 +151,7 @@ void qemu_file_set_blocking(QEMUFile *f, bool block);void qemu_set_offset (QEMUFile *f,off_t off,int whence);off_t qemu_get_offset (QEMUFile *f);void qemu_put_buffer_at (QEMUFile *f,const uint8_t *buf,size_t buflen,off_t pos); +size_t qemu_get_buffer_at (QEMUFile *f,const uint8_t *buf,size_t buflen,off_t pos);void ram_control_before_iterate (QEMUFile *f,uint64_t flags);void ram_control_after_iterate (QEMUFile *f,uint64_t flags);
Signature · 1 line
2.34.1
PATCH 8/11 migration/ram: Introduce 'fixed-ram' migration stream capability
In reply to Nikolay BorisovImplement 'fixed-ram' feature. The core of the feature is to ensure that each ram page of the migration stream has a specific offset in the resulting migration stream. The reason why we'd want such behavior are two fold:
- When doing a 'fixed-ram' migration the resulting file will have a bounded size, since pages which are dirtied multiple times will always go to a fixed location in the file, rather than constantly being added to a sequential stream. This eliminates cases where a vm with, say, 1g of ram can result in a migration file that's 10s of Gbs, provided that the workload constantly redirties memory.
- It paves the way to implement DIO-enabled save/restore of the migration stream as the pages are ensured to be written at aligned offsets.
The features requires changing the format. First, a bitmap is introduced which tracks which pages have been written (i.e are dirtied) during migration and subsequently it's being written in the resultin file, again at a fixed location for every ramblock. Zero pages are ignored as they'd be zero in the destination migration as well. With the changed format data would look like the following:
|name len|name|used_len|pc*|bitmap_size|pages_offset|bitmap|pages|
* pc - refers to the page_size/mr->addr members, so newly added members begin from "bitmap_size".
This layout is initialized during ram_save_setup so instead of having a sequential stream of pages that follow the ramblock headers the dirty pages for a ramblock follow its header. Since all pages have a fixed location RAM_SAVE_FLAG_EOS is no longer generated on every migration iteration but there is effectively a single RAM_SAVE_FLAG_EOS right at the end.
--- include/exec/ramblock.h | 7 +++ migration/migration.c | 51 +++++++++++++++++++- migration/migration.h | 1 + migration/ram.c | 100 ++++++++++++++++++++++++++++++++-------- migration/savevm.c | 1 + qapi/migration.json | 2 +- 6 files changed, 140 insertions(+), 22 deletions(-)
include/exec/ramblock.h +7 -0
@@ -43,6 +43,13 @@ struct RAMBlock {size_t page_size;/* dirty bitmap used during migration */ unsigned long *bmap; +/* shadow dirty bitmap used when migrating to a file */ +unsigned long *shadow_bmap; +/* offset in the file pages belonging to this ramblock are saved, used +* only during migration to a file +*/ +off_t bitmap_offset; +uint64_t pages_offset;/* bitmap of already received pages in postcopy */ unsigned long *receivedmap;
migration/migration.c +50 -1
87 lines not on this page · raw message
migration/migration.h +1 -0
@@ -410,6 +410,7 @@ bool migrate_zero_blocks(void);bool migrate_dirty_bitmaps (void );bool migrate_ignore_shared (void );bool migrate_validate_uuid (void ); +int migrate_fixed_ram (void );bool migrate_auto_converge (void );bool migrate_use_multifd (void );
migration/ram.c +80 -20
159 lines not on this page · raw message
migration/savevm.c +1 -0
@@ -240,6 +240,7 @@ static bool should_validate_capability(int capability)/* Validate only new capabilities to keep compatibility. */ switch (capability) {case MIGRATION_CAPABILITY_X_IGNORE_SHARED : +case MIGRATION_CAPABILITY_FIXED_RAM :return true;default :return false;
qapi/migration.json +1 -1
@@ -485,7 +485,7 @@ ## { 'enum': 'MigrationCapability', 'data': ['xbzrle', 'rdma-pin-all', 'auto-converge', 'zero-blocks', - 'compress', 'events', 'postcopy-ram', + 'compress', 'events', 'postcopy-ram', 'fixed-ram', { 'name': 'x-colo', 'features': [ 'unstable' ] }, 'release-ram', 'block', 'return-path', 'pause-before-switchover', 'multifd',
Signature · 1 line
2.34.1
PATCH 9/11 migration: Refactor precopy ram loading code
In reply to Nikolay BorisovTo facilitate easier implementaiton of the 'fixed-ram' migration restore factor out the code responsible for parsing the ramblocks headers. This also makes ram_load_precopy easier to comprehend.
--- migration/ram.c | 142 +++++++++++++++++++++++++++--------------------- 1 file changed, 80 insertions(+), 62 deletions(-)
migration/ram.c +80 -62
@@ -4254,6 +4254,83 @@ void colo_flush_ram_cache(void)trace_colo_flush_ram_cache_end (); } +static int parse_ramblock (QEMUFile *f,RAMBlock *block,ram_addr_t length) +{ +int ret =0 ; +/* ADVISE is earlier, it shows the source has the postcopy capability on */ +bool postcopy_advised =postcopy_is_advised (); + +assert (block); + +if (!qemu_ram_is_migratable (block)) { +error_report ("block %s should not be migrated !" , block->idstr); + ret = -EINVAL ; + } + +if (length != block->used_length) { +Error *local_err =NULL ; + + ret =qemu_ram_resize (block, length, &local_err); +if (local_err) { +error_report_err (local_err); + } + } +/* For postcopy we need to check hugepage sizes match */ +if (postcopy_advised &&migrate_postcopy_ram () && + block->page_size != qemu_host_page_size) { +uint64_t remote_page_size =qemu_get_be64 (f); +if (remote_page_size != block->page_size) { +error_report ("Mismatched RAM page size %s " +"(local) %zd != %" PRId64, block->idstr, + block->page_size, remote_page_size); + ret = -EINVAL ; + } + } +if (migrate_ignore_shared ()) { +hwaddr addr =qemu_get_be64 (f); +if (ramblock_is_ignored (block) && + block->mr->addr != addr) { +error_report ("Mismatched GPAs for block %s " +"%" PRId64"!= %" PRId64, block->idstr, + (uint64_t )addr, + (uint64_t )block->mr->addr); + ret = -EINVAL ; + } + } +ram_control_load_hook (f,RAM_CONTROL_BLOCK_REG , block->idstr); + +return ret; +} + +static int parse_ramblocks (QEMUFile *f,ram_addr_t total_ram_bytes) +{ +int ret =0 ; + +/* Synchronize RAM block list */ +while (!ret && total_ram_bytes) { +char id[256 ]; +RAMBlock *block; +ram_addr_t length; +int len =qemu_get_byte (f); + +qemu_get_buffer (f, (uint8_t *)id, len); + id[len] =0 ; + length =qemu_get_be64 (f); + + block =qemu_ram_block_by_name (id); +if (block) { + ret =parse_ramblock (f, block, length); + }else { +error_report ("Unknown ramblock \"%s\", cannot accept " +"migration" , id); + ret = -EINVAL ; + } + total_ram_bytes -= length; + } + +return ret; +} +/** * ram_load_precopy: load pages in precopy case *
@@ -4268,14 +4345,13 @@ static int ram_load_precopy(QEMUFile *f) {MigrationIncomingState *mis =migration_incoming_get_current ();int flags =0 , ret =0 , invalid_flags =0 , len =0 , i =0 ; -/* ADVISE is earlier, it shows the source has the postcopy capability on */ -bool postcopy_advised =postcopy_is_advised (); +if (!migrate_use_compression ()) { invalid_flags |=RAM_SAVE_FLAG_COMPRESS_PAGE ; }while (!ret && !(flags &RAM_SAVE_FLAG_EOS )) { -ram_addr_t addr, total_ram_bytes; +ram_addr_t addr;void *host =NULL , *host_bak =NULL ;uint8_t ch;
@@ -4346,65 +4422,7 @@ static int ram_load_precopy(QEMUFile *f)switch (flags & ~RAM_SAVE_FLAG_CONTINUE ) {case RAM_SAVE_FLAG_MEM_SIZE : -/* Synchronize RAM block list */ - total_ram_bytes = addr; -while (!ret && total_ram_bytes) { -RAMBlock *block; -char id[256 ]; -ram_addr_t length; - - len =qemu_get_byte (f); -qemu_get_buffer (f, (uint8_t *)id, len); - id[len] =0 ; - length =qemu_get_be64 (f); - - block =qemu_ram_block_by_name (id); -if (block && !qemu_ram_is_migratable (block)) { -error_report ("block %s should not be migrated !" , id); - ret = -EINVAL ; - }else if (block) { -if (length != block->used_length) { -Error *local_err =NULL ; - - ret =qemu_ram_resize (block, length, - &local_err); -if (local_err) { -error_report_err (local_err); - } - } -/* For postcopy we need to check hugepage sizes match */ -if (postcopy_advised &&migrate_postcopy_ram () && - block->page_size != qemu_host_page_size) { -uint64_t remote_page_size =qemu_get_be64 (f); -if (remote_page_size != block->page_size) { -error_report ("Mismatched RAM page size %s " -"(local) %zd != %" PRId64, - id, block->page_size, - remote_page_size); - ret = -EINVAL ; - } - } -if (migrate_ignore_shared ()) { -hwaddr addr =qemu_get_be64 (f); -if (ramblock_is_ignored (block) && - block->mr->addr != addr) { -error_report ("Mismatched GPAs for block %s " -"%" PRId64"!= %" PRId64, - id, (uint64_t )addr, - (uint64_t )block->mr->addr); - ret = -EINVAL ; - } - } -ram_control_load_hook (f,RAM_CONTROL_BLOCK_REG , - block->idstr); - }else { -error_report ("Unknown ramblock \"%s\", cannot " -"accept migration" , id); - ret = -EINVAL ; - } - - total_ram_bytes -= length; - } + ret =parse_ramblocks (f, addr);break ;case RAM_SAVE_FLAG_ZERO :
Signature · 1 line
2.34.1
PATCH 10/11 migration: Add support for 'fixed-ram' migration restore
In reply to Nikolay BorisovAdd the necessary code to parse the format changes for 'fixed-ram' capability. One of the more notable changes in behavior is that in the 'fixed-ram' case ram pages are restored in one go rather than constantly looping through the migration stream. Also due to idiosyncrasies of the format I have added the 'ram_migrated' since it was easier to simply return directly from ->load_state rather than introducing more conditionals around the code to prevent ->load_state being called multiple times (from qemu_loadvm_section_start_full/qemu_loadvm_section_part_end i.e. from multiple QEMU_VM_SECTION_(PART|END) flags).
--- migration/migration.h | 2 + migration/ram.c | 86 ++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 86 insertions(+), 2 deletions(-)
migration/migration.h +2 -0
@@ -96,6 +96,8 @@ struct MigrationIncomingState {bool have_listen_thread;QemuThread listen_thread; +bool ram_migrated; +/* For the kernel to send us notifications */ int userfault_fd;/* To notify the fault_thread to wake, e.g., when need to quit */
migration/ram.c +84 -2
104 lines not on this page · raw message
Signature · 1 line
2.34.1
PATCH 11/11 analyze-migration.py: add initial support for fixed ram streams
In reply to Nikolay BorisovThis commit introduces the minimum code necessary to support parsing migration strems with 'fixed-ram' capability set. The only thing really missing is the implementation of write_or_dump_fixed_ram() which deals with '-x'/'-m' options.
--- scripts/analyze-migration.py | 49 +++++++++++++++++++++++++++++++++--- 1 file changed, 46 insertions(+), 3 deletions(-)
scripts/analyze-migration.py +46 -3
@@ -23,6 +23,7 @@ import collections import struct import sys +import math def mkdir_p(path):
@@ -119,11 +120,16 @@ def __init__(self, file, version_id, ramargs, section_key): self.file = file self.section_key = section_key self.TARGET_PAGE_SIZE = ramargs['page_size'] + self.TARGET_PAGE_BITS = math.log2(self.TARGET_PAGE_SIZE) self.dump_memory = ramargs['dump_memory'] self.write_memory = ramargs['write_memory'] + self.fixed_ram = ramargs['fixed-ram'] self.sizeinfo = collections.OrderedDict() + self.bitmap_offset = collections.OrderedDict() + self.pages_offset = collections.OrderedDict() self.data = collections.OrderedDict() self.data['section sizes'] = self.sizeinfo + self.ram_read = False self.name = '' if self.write_memory: self.files = { }
@@ -140,7 +146,13 @@ def __str__(self): def getDict(self): return self.data + def write_or_dump_fixed_ram(self): + pass + def read(self): + if self.fixed_ram and self.ram_read: + return + # Read all RAM sections while True: addr = self.file.read64()
@@ -167,7 +179,25 @@ def read(self): f.truncate(0) f.truncate(len) self.files[self.name] = f + + if self.fixed_ram: + bitmap_len = self.file.read32() + # skip the pages_offset which we don't need + offset = self.file.tell() + 8 + self.bitmap_offset[self.name] = offset + offset = ((offset + bitmap_len + self.TARGET_PAGE_SIZE - 1) // self.TARGET_PAGE_SIZE) * self.TARGET_PAGE_SIZE + self.pages_offset[self.name] = offset + self.file.file.seek(offset + len) + flags &= ~self.RAM_SAVE_FLAG_MEM_SIZE + if self.fixed_ram: + self.ram_read = True + # now we should rewind to the ram page offset of the first + # ram section + if self.fixed_ram: + if self.write_memory or self.dump_memory: + self.write_or_dump_fixed_ram() + return if flags & self.RAM_SAVE_FLAG_COMPRESS: if flags & self.RAM_SAVE_FLAG_CONTINUE:
@@ -208,7 +238,7 @@ def read(self): # End of RAM section if flags & self.RAM_SAVE_FLAG_EOS: - break + return if flags != 0: raise Exception("Unknown RAM flags: %x" % flags)
@@ -521,6 +551,7 @@ def read(self, desc_only = False, dump_memory = False, write_memory = False): ramargs['page_size'] = self.vmsd_desc['page_size'] ramargs['dump_memory'] = dump_memory ramargs['write_memory'] = write_memory + ramargs['fixed-ram'] = False self.section_classes[('ram',0)][1] = ramargs while True:
@@ -528,8 +559,20 @@ def read(self, desc_only = False, dump_memory = False, write_memory = False): if section_type == self.QEMU_VM_EOF: break elif section_type == self.QEMU_VM_CONFIGURATION: - section = ConfigurationSection(file) - section.read() + config_desc = self.vmsd_desc.get('configuration') + if config_desc is not None: + config = VMSDSection(file, 1, config_desc, 'configuration') + config.read() + caps = config.data.get("configuration/capabilities") + if caps is not None: + caps = caps.data["capabilities"] + if type(caps) != list: + caps = [caps] + for i in caps: + # chomp out string length + cap = i.data[1:].decode("utf8") + if cap == "fixed-ram": + ramargs['fixed-ram'] = True elif section_type == self.QEMU_VM_SECTION_START or section_type == self.QEMU_VM_SECTION_FULL: section_id = file.read32() name = file.readstr()
Signature · 1 line
2.34.1