Add support for fixed ram offsets during migration
v2 cover Add support for fixed ram offsets during migration
Here's a slightly updated version of the patchset. In its core it remains however I have addressed a couple of issue, namely :
* Don't initialize the shadow bitmap as all set - this was causing the full ram to be copied always.
* Fixed a memory leak in parse_ramblocks_fixed_ram by making the dirty bitmap variable an g_autofree one
* Slightly reworked how ram is being copied during restore and now the code is working in chunks of 4mb (might have to tweak this?)
For a more general overview of what this series is all about refer to the initial posting [0].
[0] https://lore.kernel.org/qemu-devel/[email protected]/
Nikolay Borisov (11):
migration: support file: uri for source migration
migration: Add support for 'file:' uri for incoming migration
migration: Make migration json writer part of MigrationState struct
io: add pwritev support to QIOChannelFile
io: Add support for seekable channels
io: Add preadv support to QIOChannelFile
migration: add qemu_get_buffer_at
migration/ram: Introduce 'fixed-ram' migration stream capability
migration: Refactor precopy ram loading code
migration: Add support for 'fixed-ram' migration restore analyze-migration.py: add initial support for fixed ram streams
include/exec/ramblock.h | 7 + include/io/channel-file.h | 10 + include/io/channel.h | 1 + include/migration/qemu-file-types.h | 2 + io/channel-file.c | 55 +++++ migration/file.c | 38 ++++ migration/file.h | 10 + migration/meson.build | 1 + migration/migration.c | 61 +++++- migration/migration.h | 6 + migration/qemu-file.c | 82 +++++++ migration/qemu-file.h | 4 + migration/ram.c | 328 +++++++++++++++++++++------- migration/savevm.c | 39 ++-- qapi/migration.json | 2 +- scripts/analyze-migration.py | 49 ++++- 16 files changed, 594 insertions(+), 101 deletions(-) create mode 100644 migration/file.c create mode 100644 migration/file.h
--
2.34.1
PATCH v2 1/11 migration: support file: uri for source migration
Implement support for a "file:" uri so that a migration can be initiated directly to a file from QEMU.
--- migration/file.c | 23 +++++++++++++++++++++++ migration/file.h | 9 +++++++++ migration/meson.build | 1 + migration/migration.c | 3 +++ 4 files changed, 36 insertions(+) create mode 100644 migration/file.c create mode 100644 migration/file.h
migration/file.c new file +23 -0
@@ -0,0 +1,23 @@ +#include "qemu/osdep.h" +#include "channel.h" +#include "io/channel-file.h" +#include "file.h" +#include "qemu/error-report.h" + + +void file_start_outgoing_migration (MigrationState *s,const char *fname,Error **errp) +{ +QIOChannelFile *ioc; + + ioc =qio_channel_file_new_path (fname,O_CREAT |O_TRUNC |O_WRONLY ,0660 , errp); +if (!ioc) { +error_report ("Error creating a channel" ); +return ; + } + +qio_channel_set_name (QIO_CHANNEL (ioc),"migration-file-outgoing" ); +migration_channel_connect (s,QIO_CHANNEL (ioc),NULL ,NULL ); +object_unref (OBJECT (ioc)); +} + +
migration/file.h new file +9 -0
@@ -0,0 +1,9 @@ +#ifndef QEMU_MIGRATION_FILE_H +#define QEMU_MIGRATION_FILE_H + +void file_start_outgoing_migration (MigrationState *s, +const char *filename, +Error **errp); + +#endif +
migration/meson.build +1 -0
@@ -17,6 +17,7 @@ softmmu_ss.add(files( 'colo.c', 'exec.c', 'fd.c', + 'file.c', 'global_state.c', 'migration.c', 'multifd.c',
migration/migration.c +3 -0
@@ -20,6 +20,7 @@#include "migration/blocker.h" #include "exec.h" #include "fd.h" +#include "file.h" #include "socket.h" #include "sysemu/runstate.h" #include "sysemu/sysemu.h"
@@ -2414,6 +2415,8 @@ void qmp_migrate(const char *uri, bool has_blk, bool blk,exec_start_outgoing_migration (s, p, &local_err); }else if (strstart (uri,"fd:" , &p)) {fd_start_outgoing_migration (s, p, &local_err); + }else if (strstart (uri,"file:" , &p)) { +file_start_outgoing_migration (s, p, &local_err); }else {if (!(has_resume && resume)) {yank_unregister_instance (MIGRATION_YANK_INSTANCE );
Signature · 1 line
2.34.1
Re: [PATCH v2 01/11] migration: support file: uri for source migration
Quoted 4 lines · Nikolay Borisov
Implement support for a "file:" uri so that a migration can be initiated directly to a file from QEMU.
Signed-off-by Nikolay Borisov <[email protected]>
Quoted 8 lines
--- migration/file.c | 23 +++++++++++++++++++++++ migration/file.h | 9 +++++++++ migration/meson.build | 1 + migration/migration.c | 3 +++ 4 files changed, 36 insertions(+) create mode 100644 migration/file.c create mode 100644 migration/file.h
With regards,
Daniel
Signature · 3 lines
|: https://berrange.com -o- https://www.flickr.com/photos/dberrange :| |: https://libvirt.org -o- https://fstop138.berrange.com :| |: https://entangle-photo.org -o- https://www.instagram.com/dberrange :|
Re: [PATCH v2 01/11] migration: support file: uri for source migration
In reply to Nikolay BorisovQuoted 2 lines · Nikolay Borisov
Implement support for a "file:" uri so that a migration can be initiated directly to a file from QEMU.
Can we add a reminder here
Unlike other migration protocol backends, the 'file' protocol cannot honour non-blocking mode. POSIX file/block storage will always report ready to read/write, regardless of how slow the underlying storage will be at servicing the request.
For outgoing migration this limitation is not a serious problem as the migration data transfer always happens in a dedicated thread. It may, however, result in delays in honouring a request to cancel the migration operation.
Quoted 91 lines
Signed-off-by Nikolay Borisov <[email protected]>--- migration/file.c | 23 +++++++++++++++++++++++ migration/file.h | 9 +++++++++ migration/meson.build | 1 + migration/migration.c | 3 +++ 4 files changed, 36 insertions(+) create mode 100644 migration/file.c create mode 100644 migration/file.hmigration/file.c new file +23
-0@@ -0,0 +1,23 @@ +#include "qemu/osdep.h" +#include "channel.h" +#include "io/channel-file.h" +#include "file.h" +#include "qemu/error-report.h" + + +void file_start_outgoing_migration (MigrationState *s,const char *fname,Error **errp) +{ +QIOChannelFile *ioc; + + ioc =qio_channel_file_new_path (fname,O_CREAT |O_TRUNC |O_WRONLY ,0660 , errp); +if (!ioc) { +error_report ("Error creating a channel" ); +return ; + } + +qio_channel_set_name (QIO_CHANNEL (ioc),"migration-file-outgoing" ); +migration_channel_connect (s,QIO_CHANNEL (ioc),NULL ,NULL ); +object_unref (OBJECT (ioc)); +} + +migration/file.h new file +9
-0@@ -0,0 +1,9 @@ +#ifndef QEMU_MIGRATION_FILE_H +#define QEMU_MIGRATION_FILE_H + +void file_start_outgoing_migration (MigrationState *s, +const char *filename, +Error **errp); + +#endif +migration/meson.build +1
-0@@ -17,6 +17,7 @@ softmmu_ss.add(files( 'colo.c', 'exec.c', 'fd.c', + 'file.c', 'global_state.c', 'migration.c', 'multifd.c',migration/migration.c +3
-0@@ -20,6 +20,7 @@#include "migration/blocker.h" #include "exec.h" #include "fd.h" +#include "file.h" #include "socket.h" #include "sysemu/runstate.h" #include "sysemu/sysemu.h" @@ -2414,6 +2415,8 @@ void qmp_migrate(const char *uri, bool has_blk, bool blk,exec_start_outgoing_migration (s, p, &local_err); }else if (strstart (uri,"fd:" , &p)) {fd_start_outgoing_migration (s, p, &local_err); + }else if (strstart (uri,"file:" , &p)) { +file_start_outgoing_migration (s, p, &local_err); }else {if (!(has_resume && resume)) {yank_unregister_instance (MIGRATION_YANK_INSTANCE );Signature · 1 line
2.34.1
With regards,
Daniel
Signature · 3 lines
|: https://berrange.com -o- https://www.flickr.com/photos/dberrange :| |: https://libvirt.org -o- https://fstop138.berrange.com :| |: https://entangle-photo.org -o- https://www.instagram.com/dberrange :|
Re: [PATCH v2 01/11] migration: support file: uri for source migration
Quoted 11 lines · Daniel P. Berrangé
Quoted 2 lines · Nikolay Borisov
Implement support for a "file:" uri so that a migration can be initiated directly to a file from QEMU.
Can we add a reminder here
Unlike other migration protocol backends, the 'file' protocol cannot honour non-blocking mode. POSIX file/block storage will always report ready to read/write, regardless of how slow the underlying storage will be at servicing the request.
Quoted 4 lines
For outgoing migration this limitation is not a serious problem as the migration data transfer always happens in a dedicated thread. It may, however, result in delays in honouring a request to cancel the migration operation.
Sure, I assume the same text should be added to the incoming migration patch as well and emphasize that it could be more problematic there?
In any case I'd wait to gather more feedback before sending v3
Quoted 95 lines
Quoted 92 lines
Signed-off-by Nikolay Borisov <[email protected]>--- migration/file.c | 23 +++++++++++++++++++++++ migration/file.h | 9 +++++++++ migration/meson.build | 1 + migration/migration.c | 3 +++ 4 files changed, 36 insertions(+)create mode 100644
migration/file.c
create mode 100644migration/file.hmigration/file.c new file +23
-0@@ -0,0 +1,23 @@ +#include "qemu/osdep.h" +#include "channel.h" +#include "io/channel-file.h" +#include "file.h" +#include "qemu/error-report.h" + + +void file_start_outgoing_migration (MigrationState *s,const char *fname,Error **errp) +{ +QIOChannelFile *ioc; + + ioc =qio_channel_file_new_path (fname,O_CREAT |O_TRUNC |O_WRONLY ,0660 , errp); +if (!ioc) { +error_report ("Error creating a channel" ); +return ; + } + +qio_channel_set_name (QIO_CHANNEL (ioc),"migration-file-outgoing" ); +migration_channel_connect (s,QIO_CHANNEL (ioc),NULL ,NULL ); +object_unref (OBJECT (ioc)); +} + +migration/file.h new file +9
-0@@ -0,0 +1,9 @@ +#ifndef QEMU_MIGRATION_FILE_H +#define QEMU_MIGRATION_FILE_H + +void file_start_outgoing_migration (MigrationState *s, +const char *filename, +Error **errp); + +#endif +migration/meson.build +1
-0@@ -17,6 +17,7 @@ softmmu_ss.add(files( 'colo.c', 'exec.c', 'fd.c', + 'file.c', 'global_state.c', 'migration.c', 'multifd.c',migration/migration.c +3
-0@@ -20,6 +20,7 @@#include "migration/blocker.h" #include "exec.h" #include "fd.h" +#include "file.h" #include "socket.h" #include "sysemu/runstate.h" #include "sysemu/sysemu.h" @@ -2414,6 +2415,8 @@ void qmp_migrate(const char *uri, bool has_blk, bool blk,exec_start_outgoing_migration (s, p, &local_err); }else if (strstart (uri,"fd:" , &p)) {fd_start_outgoing_migration (s, p, &local_err); + }else if (strstart (uri,"file:" , &p)) { +file_start_outgoing_migration (s, p, &local_err); }else {if (!(has_resume && resume)) {yank_unregister_instance (MIGRATION_YANK_INSTANCE );Signature · 1 line
2.34.1With regards,
Daniel
PATCH v2 2/11 migration: Add support for 'file:' uri for incoming migration
In reply to Nikolay BorisovThis is a counterpart to the 'file:' uri support for source migration, now a file can also serve as the source of an incoming migration.
--- migration/file.c | 15 +++++++++++++++ migration/file.h | 1 + migration/migration.c | 2 ++ 3 files changed, 18 insertions(+)
migration/file.c +15 -0
@@ -21,3 +21,18 @@ void file_start_outgoing_migration(MigrationState *s, const char *fname, Error * } +void file_start_incoming_migration (const char *fname,Error **errp) +{ +QIOChannelFile *ioc; + + ioc =qio_channel_file_new_path (fname,O_RDONLY ,0 , errp); +if (!ioc) { +error_report ("Error creating a channel" ); +return ; + } + +qio_channel_set_name (QIO_CHANNEL (ioc),"migration-file-incoming" ); +migration_channel_process_incoming (QIO_CHANNEL (ioc)); +object_unref (OBJECT (ioc)); +} +
migration/file.h +1 -0
@@ -5,5 +5,6 @@ void file_start_outgoing_migration(MigrationState *s,const char *filename, Error **errp); +void file_start_incoming_migration (const char *fname,Error **errp);#endif
migration/migration.c +2 -0
@@ -506,6 +506,8 @@ static void qemu_start_incoming_migration(const char *uri, Error **errp)exec_start_incoming_migration (p, errp); }else if (strstart (uri,"fd:" , &p)) {fd_start_incoming_migration (p, errp); + }else if (strstart (uri,"file:" , &p)) { +file_start_incoming_migration (p, errp); }else {error_setg (errp,"unknown migration protocol: %s" , uri); }
Signature · 1 line
2.34.1
Re: [PATCH v2 02/11] migration: Add support for 'file:' uri for incoming migration
Quoted 2 lines · Nikolay Borisov
This is a counterpart to the 'file:' uri support for source migration, now a file can also serve as the source of an incoming migration.
As with the prvious patch, can we add a reminder:
Unlike other migration protocol backends, the 'file' protocol cannot honour non-blocking mode. POSIX file/block storage will always report ready to read/write, regardless of how slow the underlying storage will be at servicing the request.
For incoming migration this limitation may result in the main event loop not being fully responsive while loading the VM state. This won't impact the VM since it is not running at this phase, however, it may impact management applications.
Quoted 14 lines
Signed-off-by Nikolay Borisov <[email protected]>--- migration/file.c | 15 +++++++++++++++ migration/file.h | 1 + migration/migration.c | 2 ++ 3 files changed, 18 insertions(+)migration/migration.c +0
-0
migration/migration.c static void qemu_start_incoming_migration(const char *uri, Error **errp) · line 510
Quoted 5 lines
migration/migration.c +2
-0@@ -506,6 +506,8 @@ static void qemu_start_incoming_migration(const char *uri, Error **errp)exec_start_incoming_migration (p, errp); }else if (strstart (uri,"fd:" , &p)) {fd_start_incoming_migration (p, errp); + }else if (strstart (uri,"file:" , &p)) { +file_start_incoming_migration (p, errp);
A <tab> crept in there by mistake.
migration/migration.c static void qemu_start_incoming_migration(const char *uri, Error **errp) · line 513
Quoted 3 lines
migration/migration.c +0
-0}else {error_setg (errp,"unknown migration protocol: %s" , uri); }
With the above fixed
With regards,
Daniel
Signature · 3 lines
|: https://berrange.com -o- https://www.flickr.com/photos/dberrange :| |: https://libvirt.org -o- https://fstop138.berrange.com :| |: https://entangle-photo.org -o- https://www.instagram.com/dberrange :|
PATCH v2 3/11 migration: Make migration json writer part of MigrationState struct
In reply to Nikolay BorisovThis is required so that migration stream configuration is written to the migration stream. This would allow analyze-migration to parse enabled capabilities for the migration and adjust its behavior accordingly. This is in preparation for analyze-migration.py to support 'fixed-ram' capability format changes.
--- migration/migration.c | 5 +++++ migration/migration.h | 3 +++ migration/savevm.c | 38 ++++++++++++++++++++++---------------- 3 files changed, 30 insertions(+), 16 deletions(-)
migration/migration.c +5 -0
@@ -1896,6 +1896,8 @@ static void migrate_fd_cleanup(MigrationState *s)g_free (s->hostname); s->hostname =NULL ; +json_writer_free (s->vmdesc); +qemu_savevm_state_cleanup ();if (s->to_dst_file) {
@@ -2154,6 +2156,7 @@ void migrate_init(MigrationState *s)error_free (s->error); s->error =NULL ; s->hostname =NULL ; + s->vmdesc =NULL ;migrate_set_state (&s->state,MIGRATION_STATUS_NONE ,MIGRATION_STATUS_SETUP );
@@ -4269,6 +4272,8 @@ void migrate_fd_connect(MigrationState *s, Error *error_in)return ; } + s->vmdesc =json_writer_new (false); +if (multifd_save_setup (&local_err) !=0 ) {error_report_err (local_err); migrate_set_state(&s->state, MIGRATION_STATUS_SETUP,
migration/migration.h +3 -0
@@ -17,6 +17,7 @@#include "exec/cpu-common.h" #include "hw/qdev-core.h" #include "qapi/qapi-types-migration.h" +#include "qapi/qmp/json-writer.h" #include "qemu/thread.h" #include "qemu/coroutine_int.h" #include "io/channel.h"
@@ -261,6 +262,8 @@ struct MigrationState {int state; +JSONWriter *vmdesc; +/* State related to return path */ struct {/* Protected by qemu_file_lock */
migration/savevm.c +22 -16
@@ -1137,13 +1137,18 @@ void qemu_savevm_non_migratable_list(strList **reasons)void qemu_savevm_state_header (QEMUFile *f) { +MigrationState *s =migrate_get_current ();trace_savevm_state_header ();qemu_put_be32 (f,QEMU_VM_FILE_MAGIC );qemu_put_be32 (f,QEMU_VM_FILE_VERSION ); -if (migrate_get_current ()->send_configuration) { +if (s->send_configuration) {qemu_put_byte (f,QEMU_VM_CONFIGURATION ); -vmstate_save_state (f, &vmstate_configuration, &savevm_state,0 ); +json_writer_start_object (s->vmdesc,NULL ); +json_writer_start_object (s->vmdesc,"configuration" ); +vmstate_save_state (f, &vmstate_configuration, &savevm_state, s->vmdesc); +json_writer_end_object (s->vmdesc); + } }
@@ -1364,15 +1369,16 @@ int qemu_savevm_state_complete_precopy_non_iterable(QEMUFile *f, bool in_postcopy, bool inactivate_disks) { -g_autoptr (JSONWriter ) vmdesc =NULL ; +MigrationState *s =migrate_get_current ();int vmdesc_len;SaveStateEntry *se;int ret; - vmdesc =json_writer_new (false); -json_writer_start_object (vmdesc,NULL ); -json_writer_int64 (vmdesc,"page_size" ,qemu_target_page_size ()); -json_writer_start_array (vmdesc,"devices" ); +if (!s->send_configuration) { +json_writer_start_object (s->vmdesc,NULL ); + } +json_writer_int64 (s->vmdesc,"page_size" ,qemu_target_page_size ()); +json_writer_start_array (s->vmdesc,"devices" ); QTAILQ_FOREACH(se, &savevm_state.handlers, entry) {if ((!se->ops || !se->ops->save_state) && !se->vmsd) {
@@ -1385,12 +1391,12 @@ int qemu_savevm_state_complete_precopy_non_iterable(QEMUFile *f,trace_savevm_section_start (se->idstr, se->section_id); -json_writer_start_object (vmdesc,NULL ); -json_writer_str (vmdesc,"name" , se->idstr); -json_writer_int64 (vmdesc,"instance_id" , se->instance_id); +json_writer_start_object (s->vmdesc,NULL ); +json_writer_str (s->vmdesc,"name" , se->idstr); +json_writer_int64 (s->vmdesc,"instance_id" , se->instance_id);save_section_header (f, se,QEMU_VM_SECTION_FULL ); - ret =vmstate_save (f, se, vmdesc); + ret =vmstate_save (f, se, s->vmdesc);if (ret) {qemu_file_set_error (f, ret);return ret;
@@ -1398,7 +1404,7 @@ int qemu_savevm_state_complete_precopy_non_iterable(QEMUFile *f,trace_savevm_section_end (se->idstr, se->section_id,0 );save_section_footer (f, se); -json_writer_end_object (vmdesc); +json_writer_end_object (s->vmdesc); }if (inactivate_disks) {
@@ -1417,14 +1423,14 @@ int qemu_savevm_state_complete_precopy_non_iterable(QEMUFile *f,qemu_put_byte (f,QEMU_VM_EOF ); } -json_writer_end_array (vmdesc); -json_writer_end_object (vmdesc); - vmdesc_len =strlen (json_writer_get (vmdesc)); +json_writer_end_array (s->vmdesc); +json_writer_end_object (s->vmdesc); + vmdesc_len =strlen (json_writer_get (s->vmdesc));if (should_send_vmdesc ()) {qemu_put_byte (f,QEMU_VM_VMDESCRIPTION );qemu_put_be32 (f, vmdesc_len); -qemu_put_buffer (f, (uint8_t *)json_writer_get (vmdesc), vmdesc_len); +qemu_put_buffer (f, (uint8_t *)json_writer_get (s->vmdesc), vmdesc_len); }return 0 ;
Signature · 1 line
2.34.1
Re: [PATCH v2 03/11] migration: Make migration json writer part of MigrationState struct
Quoted 76 lines · Nikolay Borisov
This is required so that migration stream configuration is written to the migration stream. This would allow analyze-migration to parse enabled capabilities for the migration and adjust its behavior accordingly. This is in preparation for
analyze-migration.pyto support 'fixed-ram' capability format changes.Signed-off-by Nikolay Borisov <[email protected]>--- migration/migration.c | 5 +++++ migration/migration.h | 3 +++ migration/savevm.c | 38 ++++++++++++++++++++++---------------- 3 files changed, 30 insertions(+), 16 deletions(-)migration/migration.c +5
-0@@ -1896,6 +1896,8 @@ static void migrate_fd_cleanup(MigrationState *s)g_free (s->hostname); s->hostname =NULL ; +json_writer_free (s->vmdesc); +qemu_savevm_state_cleanup ();if (s->to_dst_file) {@@ -2154,6 +2156,7 @@ void migrate_init(MigrationState *s)error_free (s->error); s->error =NULL ; s->hostname =NULL ; + s->vmdesc =NULL ;migrate_set_state (&s->state,MIGRATION_STATUS_NONE ,MIGRATION_STATUS_SETUP );@@ -4269,6 +4272,8 @@ void migrate_fd_connect(MigrationState *s, Error *error_in)return ; } + s->vmdesc =json_writer_new (false); +if (multifd_save_setup (&local_err) !=0 ) {error_report_err (local_err); migrate_set_state(&s->state, MIGRATION_STATUS_SETUP,migration/migration.h +3
-0@@ -17,6 +17,7 @@#include "exec/cpu-common.h" #include "hw/qdev-core.h" #include "qapi/qapi-types-migration.h" +#include "qapi/qmp/json-writer.h" #include "qemu/thread.h" #include "qemu/coroutine_int.h" #include "io/channel.h" @@ -261,6 +262,8 @@ struct MigrationState {int state; +JSONWriter *vmdesc; +/* State related to return path */ struct {/* Protected by qemu_file_lock */ migration/savevm.c +1
-0@@ -1137,13 +1137,18 @@ void qemu_savevm_non_migratable_list(strList **reasons)void qemu_savevm_state_header (QEMUFile *f) { +MigrationState *s =migrate_get_current ();trace_savevm_state_header ();qemu_put_be32 (f,QEMU_VM_FILE_MAGIC );qemu_put_be32 (f,QEMU_VM_FILE_VERSION );
migration/savevm.c void qemu_savevm_non_migratable_list(strList **reasons) · line 1151
Quoted 10 lines
migration/savevm.c +6
-2-if (migrate_get_current ()->send_configuration) { +if (s->send_configuration) {qemu_put_byte (f,QEMU_VM_CONFIGURATION ); -vmstate_save_state (f, &vmstate_configuration, &savevm_state,0 ); +json_writer_start_object (s->vmdesc,NULL ); +json_writer_start_object (s->vmdesc,"configuration" ); +vmstate_save_state (f, &vmstate_configuration, &savevm_state, s->vmdesc); +json_writer_end_object (s->vmdesc); +
IIUC, this is changing the info that is written in the VM configuration section, by adding an extra level of nesting to the object.
Isn't this going to cause backwards compatibility problems ?
Nothing in the patch seems to take account of the exctra 'configuiration' object that has been started
Also, there's two json_writer_start_object calls, but only one json_writer_end_object.
BTW, some <tab> crept into this patch.
Quoted 73 lines
migration/savevm.c +15
-14} }@@ -1364,15 +1369,16 @@ int qemu_savevm_state_complete_precopy_non_iterable(QEMUFile *f, bool in_postcopy, bool inactivate_disks) { -g_autoptr (JSONWriter ) vmdesc =NULL ; +MigrationState *s =migrate_get_current ();int vmdesc_len;SaveStateEntry *se;int ret; - vmdesc =json_writer_new (false); -json_writer_start_object (vmdesc,NULL ); -json_writer_int64 (vmdesc,"page_size" ,qemu_target_page_size ()); -json_writer_start_array (vmdesc,"devices" ); +if (!s->send_configuration) { +json_writer_start_object (s->vmdesc,NULL ); + } +json_writer_int64 (s->vmdesc,"page_size" ,qemu_target_page_size ()); +json_writer_start_array (s->vmdesc,"devices" ); QTAILQ_FOREACH(se, &savevm_state.handlers, entry) {if ((!se->ops || !se->ops->save_state) && !se->vmsd) {@@ -1385,12 +1391,12 @@ int qemu_savevm_state_complete_precopy_non_iterable(QEMUFile *f,trace_savevm_section_start (se->idstr, se->section_id); -json_writer_start_object (vmdesc,NULL ); -json_writer_str (vmdesc,"name" , se->idstr); -json_writer_int64 (vmdesc,"instance_id" , se->instance_id); +json_writer_start_object (s->vmdesc,NULL ); +json_writer_str (s->vmdesc,"name" , se->idstr); +json_writer_int64 (s->vmdesc,"instance_id" , se->instance_id);save_section_header (f, se,QEMU_VM_SECTION_FULL ); - ret =vmstate_save (f, se, vmdesc); + ret =vmstate_save (f, se, s->vmdesc);if (ret) {qemu_file_set_error (f, ret);return ret;@@ -1398,7 +1404,7 @@ int qemu_savevm_state_complete_precopy_non_iterable(QEMUFile *f,trace_savevm_section_end (se->idstr, se->section_id,0 );save_section_footer (f, se); -json_writer_end_object (vmdesc); +json_writer_end_object (s->vmdesc); }if (inactivate_disks) {@@ -1417,14 +1423,14 @@ int qemu_savevm_state_complete_precopy_non_iterable(QEMUFile *f,qemu_put_byte (f,QEMU_VM_EOF ); } -json_writer_end_array (vmdesc); -json_writer_end_object (vmdesc); - vmdesc_len =strlen (json_writer_get (vmdesc)); +json_writer_end_array (s->vmdesc); +json_writer_end_object (s->vmdesc); + vmdesc_len =strlen (json_writer_get (s->vmdesc));if (should_send_vmdesc ()) {qemu_put_byte (f,QEMU_VM_VMDESCRIPTION );qemu_put_be32 (f, vmdesc_len); -qemu_put_buffer (f, (uint8_t *)json_writer_get (vmdesc), vmdesc_len); +qemu_put_buffer (f, (uint8_t *)json_writer_get (s->vmdesc), vmdesc_len); }return 0 ;Signature · 1 line
2.34.1
With regards,
Daniel
Signature · 3 lines
|: https://berrange.com -o- https://www.flickr.com/photos/dberrange :| |: https://libvirt.org -o- https://fstop138.berrange.com :| |: https://entangle-photo.org -o- https://www.instagram.com/dberrange :|
Re: [PATCH v2 03/11] migration: Make migration json writer part of MigrationState struct
Quoted 92 lines · Daniel P. Berrangé
Quoted 76 lines · Nikolay Borisov
This is required so that migration stream configuration is written to the migration stream. This would allow analyze-migration to parse enabled capabilities for the migration and adjust its behavior accordingly. This is in preparation for
analyze-migration.pyto support 'fixed-ram' capability format changes.Signed-off-by Nikolay Borisov <[email protected]>--- migration/migration.c | 5 +++++ migration/migration.h | 3 +++ migration/savevm.c | 38 ++++++++++++++++++++++---------------- 3 files changed, 30 insertions(+), 16 deletions(-)migration/migration.c +5
-0@@ -1896,6 +1896,8 @@ static void migrate_fd_cleanup(MigrationState *s)g_free (s->hostname); s->hostname =NULL ; +json_writer_free (s->vmdesc); +qemu_savevm_state_cleanup ();if (s->to_dst_file) {@@ -2154,6 +2156,7 @@ void migrate_init(MigrationState *s)error_free (s->error); s->error =NULL ; s->hostname =NULL ; + s->vmdesc =NULL ;migrate_set_state (&s->state,MIGRATION_STATUS_NONE ,MIGRATION_STATUS_SETUP );@@ -4269,6 +4272,8 @@ void migrate_fd_connect(MigrationState *s, Error *error_in)return ; } + s->vmdesc =json_writer_new (false); +if (multifd_save_setup (&local_err) !=0 ) {error_report_err (local_err); migrate_set_state(&s->state, MIGRATION_STATUS_SETUP,migration/migration.h +3
-0@@ -17,6 +17,7 @@#include "exec/cpu-common.h" #include "hw/qdev-core.h" #include "qapi/qapi-types-migration.h" +#include "qapi/qmp/json-writer.h" #include "qemu/thread.h" #include "qemu/coroutine_int.h" #include "io/channel.h" @@ -261,6 +262,8 @@ struct MigrationState {int state; +JSONWriter *vmdesc; +/* State related to return path */ struct {/* Protected by qemu_file_lock */ migration/savevm.c +1
-0@@ -1137,13 +1137,18 @@ void qemu_savevm_non_migratable_list(strList **reasons)void qemu_savevm_state_header (QEMUFile *f) { +MigrationState *s =migrate_get_current ();trace_savevm_state_header ();qemu_put_be32 (f,QEMU_VM_FILE_MAGIC );qemu_put_be32 (f,QEMU_VM_FILE_VERSION );migration/savevm.c void qemu_savevm_non_migratable_list(strList **reasons) · line 1151
Quoted 10 lines
migration/savevm.c +6
-2-if (migrate_get_current ()->send_configuration) { +if (s->send_configuration) {qemu_put_byte (f,QEMU_VM_CONFIGURATION ); -vmstate_save_state (f, &vmstate_configuration, &savevm_state,0 ); +json_writer_start_object (s->vmdesc,NULL ); +json_writer_start_object (s->vmdesc,"configuration" ); +vmstate_save_state (f, &vmstate_configuration, &savevm_state, s->vmdesc); +json_writer_end_object (s->vmdesc); +IIUC, this is changing the info that is written in the VM configuration section, by adding an extra level of nesting to the object.
Quoted 4 lines
Isn't this going to cause backwards compatibility problems ?
Nothing in the patch seems to take account of the exctra 'configuiration' object that has been started
The resulting json looks like:
{
"configuration": {
"vmsd_name": "configuration",
"version": 1,
"fields": [
{
"name": "len",
"type": "uint32",
"size": 4
},
{
"name": "name",
"type": "buffer",
"size": 13
}
],
"subsections": [
{
"vmsd_name": "configuration/capabilities",
"version": 1,
"fields": [
{
"name": "caps_count",
"type": "uint32",
"size": 4
},
{
"name": "capabilities",
"type": "capability",
"size": 10
}
]
}
]
},
"page_size": 4096,
"devices": [
{
"name": "timer",
"instance_id": 0,//ommitted
So the "configuration" object is indeed added, but older versions of qemu can ignore it without any problem.
Quoted 3 lines
Also, there's two
json_writer_start_objectcalls, but only onejson_writer_end_object.
That's intentional, the first one begins the top-level object and it is actually paired with the final call to json_writer_end_object(s->vmdesc); in qemu_savevm_state_complete_precopy_non_iterable .
Quoted 2 lines
BTW, some <tab> crept into this patch.
Will fix this.
PS. I usually work on the linux kernel so vim used my linuxsty.vim settings. However, I eventually instsalled .editorconfig support so I guess those are leftovers.
Quoted 78 lines
Quoted 75 lines
migration/savevm.c +15
-14} }@@ -1364,15 +1369,16 @@ int qemu_savevm_state_complete_precopy_non_iterable(QEMUFile *f, bool in_postcopy, bool inactivate_disks) { -g_autoptr (JSONWriter ) vmdesc =NULL ; +MigrationState *s =migrate_get_current ();int vmdesc_len;SaveStateEntry *se;int ret; - vmdesc =json_writer_new (false); -json_writer_start_object (vmdesc,NULL ); -json_writer_int64 (vmdesc,"page_size" ,qemu_target_page_size ()); -json_writer_start_array (vmdesc,"devices" ); +if (!s->send_configuration) { +json_writer_start_object (s->vmdesc,NULL ); + } +json_writer_int64 (s->vmdesc,"page_size" ,qemu_target_page_size ()); +json_writer_start_array (s->vmdesc,"devices" ); QTAILQ_FOREACH(se, &savevm_state.handlers, entry) {if ((!se->ops || !se->ops->save_state) && !se->vmsd) {@@ -1385,12 +1391,12 @@ int qemu_savevm_state_complete_precopy_non_iterable(QEMUFile *f,trace_savevm_section_start (se->idstr, se->section_id); -json_writer_start_object (vmdesc,NULL ); -json_writer_str (vmdesc,"name" , se->idstr); -json_writer_int64 (vmdesc,"instance_id" , se->instance_id); +json_writer_start_object (s->vmdesc,NULL ); +json_writer_str (s->vmdesc,"name" , se->idstr); +json_writer_int64 (s->vmdesc,"instance_id" , se->instance_id);save_section_header (f, se,QEMU_VM_SECTION_FULL ); - ret =vmstate_save (f, se, vmdesc); + ret =vmstate_save (f, se, s->vmdesc);if (ret) {qemu_file_set_error (f, ret);return ret;@@ -1398,7 +1404,7 @@ int qemu_savevm_state_complete_precopy_non_iterable(QEMUFile *f,trace_savevm_section_end (se->idstr, se->section_id,0 );save_section_footer (f, se); -json_writer_end_object (vmdesc); +json_writer_end_object (s->vmdesc); }if (inactivate_disks) {@@ -1417,14 +1423,14 @@ int qemu_savevm_state_complete_precopy_non_iterable(QEMUFile *f,qemu_put_byte (f,QEMU_VM_EOF ); } -json_writer_end_array (vmdesc); -json_writer_end_object (vmdesc); - vmdesc_len =strlen (json_writer_get (vmdesc)); +json_writer_end_array (s->vmdesc); +json_writer_end_object (s->vmdesc); + vmdesc_len =strlen (json_writer_get (s->vmdesc));if (should_send_vmdesc ()) {qemu_put_byte (f,QEMU_VM_VMDESCRIPTION );qemu_put_be32 (f, vmdesc_len); -qemu_put_buffer (f, (uint8_t *)json_writer_get (vmdesc), vmdesc_len); +qemu_put_buffer (f, (uint8_t *)json_writer_get (s->vmdesc), vmdesc_len); }return 0 ;Signature · 1 line
2.34.1With regards,
Daniel
Re: [PATCH v2 03/11] migration: Make migration json writer part of MigrationState struct
Quoted 133 lines · Nikolay Borisov
Quoted 83 lines · Daniel P. Berrangé
Quoted 67 lines · Nikolay Borisov
This is required so that migration stream configuration is written to the migration stream. This would allow analyze-migration to parse enabled capabilities for the migration and adjust its behavior accordingly. This is in preparation for
analyze-migration.pyto support 'fixed-ram' capability format changes.Signed-off-by Nikolay Borisov <[email protected]>--- migration/migration.c | 5 +++++ migration/migration.h | 3 +++ migration/savevm.c | 38 ++++++++++++++++++++++---------------- 3 files changed, 30 insertions(+), 16 deletions(-)migration/migration.c +5
-0@@ -1896,6 +1896,8 @@ static void migrate_fd_cleanup(MigrationState *s)g_free (s->hostname); s->hostname =NULL ; +json_writer_free (s->vmdesc); +qemu_savevm_state_cleanup ();if (s->to_dst_file) {@@ -2154,6 +2156,7 @@ void migrate_init(MigrationState *s)error_free (s->error); s->error =NULL ; s->hostname =NULL ; + s->vmdesc =NULL ;migrate_set_state (&s->state,MIGRATION_STATUS_NONE ,MIGRATION_STATUS_SETUP );@@ -4269,6 +4272,8 @@ void migrate_fd_connect(MigrationState *s, Error *error_in)return ; } + s->vmdesc =json_writer_new (false); +if (multifd_save_setup (&local_err) !=0 ) {error_report_err (local_err); migrate_set_state(&s->state, MIGRATION_STATUS_SETUP,migration/migration.h +3
-0@@ -17,6 +17,7 @@#include "exec/cpu-common.h" #include "hw/qdev-core.h" #include "qapi/qapi-types-migration.h" +#include "qapi/qmp/json-writer.h" #include "qemu/thread.h" #include "qemu/coroutine_int.h" #include "io/channel.h" @@ -261,6 +262,8 @@ struct MigrationState {int state; +JSONWriter *vmdesc; +/* State related to return path */ struct {/* Protected by qemu_file_lock */ migration/savevm.c +1
-0@@ -1137,13 +1137,18 @@ void qemu_savevm_non_migratable_list(strList **reasons)void qemu_savevm_state_header (QEMUFile *f) { +MigrationState *s =migrate_get_current ();trace_savevm_state_header ();qemu_put_be32 (f,QEMU_VM_FILE_MAGIC );migration/savevm.c void qemu_savevm_non_migratable_list(strList **reasons) · line 1149
Quoted 10 lines
migration/savevm.c +6
-2qemu_put_be32 (f,QEMU_VM_FILE_VERSION ); -if (migrate_get_current ()->send_configuration) { +if (s->send_configuration) {qemu_put_byte (f,QEMU_VM_CONFIGURATION ); -vmstate_save_state (f, &vmstate_configuration, &savevm_state,0 ); +json_writer_start_object (s->vmdesc,NULL ); +json_writer_start_object (s->vmdesc,"configuration" ); +vmstate_save_state (f, &vmstate_configuration, &savevm_state, s->vmdesc); +json_writer_end_object (s->vmdesc); +IIUC, this is changing the info that is written in the VM configuration section, by adding an extra level of nesting to the object.
Quoted 4 lines
Isn't this going to cause backwards compatibility problems ?
Nothing in the patch seems to take account of the exctra 'configuiration' object that has been started
The resulting json looks like:
{ "configuration": { "vmsd_name": "configuration", "version": 1, "fields": [ { "name": "len", "type": "uint32", "size": 4 }, { "name": "name", "type": "buffer", "size": 13 } ], "subsections": [ { "vmsd_name": "configuration/capabilities", "version": 1, "fields": [ { "name": "caps_count", "type": "uint32", "size": 4 }, { "name": "capabilities", "type": "capability", "size": 10 } ] } ] }, "page_size": 4096, "devices": [ { "name": "timer", "instance_id": 0,
Quoted 4 lines
//ommitted
So the "configuration" object is indeed added, but older versions of qemu can ignore it without any problem.
IIUC, after looking further, this JSON is only used by the analyze-migration.py script ? If that's right, then we should have the change to analyze-migration.py that copes with the "configuration" option in the same patch.
Quoted 6 lines
Quoted 2 lines
Also, there's two
json_writer_start_objectcalls, but only onejson_writer_end_object.That's intentional, the first one begins the top-level object and it is actually paired with the final call to
json_writer_end_object(s->vmdesc); inqemu_savevm_state_complete_precopy_non_iterable.
Oh right, can you add a comment to both locations, mentioning where their respective pair lives.
With regards,
Daniel
Signature · 3 lines
|: https://berrange.com -o- https://www.flickr.com/photos/dberrange :| |: https://libvirt.org -o- https://fstop138.berrange.com :| |: https://entangle-photo.org -o- https://www.instagram.com/dberrange :|
PATCH v2 4/11 io: add pwritev support to QIOChannelFile
In reply to Nikolay BorisovThe upcoming 'fixed-ram' feature would require qemu to write data at specific offsets of the file. This is currently missing so this patch adds it. I've chosen to implement it as a type-specific function rather than plumb it through the generic channel interface as it relies on being able to do seeks.
--- include/io/channel-file.h | 5 +++++ io/channel-file.c | 24 ++++++++++++++++++++++++ 2 files changed, 29 insertions(+)
include/io/channel-file.h +5 -0
@@ -89,4 +89,9 @@ qio_channel_file_new_path(const char *path,mode_t mode, Error **errp); +ssize_t qio_channel_file_pwritev (QIOChannel *ioc, +const struct iovec *iov, +size_t niov, +off_t offset, +Error **errp);#endif /* QIO_CHANNEL_FILE_H */
io/channel-file.c +24 -0
@@ -136,6 +136,30 @@ static ssize_t qio_channel_file_writev(QIOChannel *ioc,return ret; } +ssize_t qio_channel_file_pwritev (QIOChannel *ioc, +const struct iovec *iov, +size_t niov, +off_t offset, +Error **errp) +{ +QIOChannelFile *fioc =QIO_CHANNEL_FILE (ioc); +ssize_t ret; + + retry: + ret =pwritev (fioc->fd, iov, niov, offset); +if (ret <=0 ) { +if (errno ==EAGAIN ) { +return QIO_CHANNEL_ERR_BLOCK ; + } +if (errno ==EINTR ) { + goto retry; + } +error_setg_errno (errp, errno,"Unable to write to file" ); +return -1 ; + } +return ret; +} +static int qio_channel_file_set_blocking (QIOChannel *ioc,bool enabled,Error **errp)
Signature · 1 line
2.34.1
Re: [PATCH v2 04/11] io: add pwritev support to QIOChannelFile
Quoted 5 lines · Nikolay Borisov
The upcoming 'fixed-ram' feature would require qemu to write data at specific offsets of the file. This is currently missing so this patch adds it. I've chosen to implement it as a type-specific function rather than plumb it through the generic channel interface as it relies on being able to do seeks.
Well the base QIOChannel does have an 'io_seek' callback, so it already has some support for seeking.
In addition to this QIOChannelFile class, the QIOChannelBuffer and QIOChannelNull classes could also allow seekable IO ops.
So I think it is OK to add this to the QIOChannel base class, especially since your next patch adds a "SEEKABLE" feature flag.
We have io_writev/io_readv callbacks. If we add io_pwritev/io_preadv callbacks too, we should validate that they exist when the SEEKABLE feature is marked present.
Quoted 59 lines
Signed-off-by Nikolay Borisov <[email protected]>--- include/io/channel-file.h | 5 +++++ io/channel-file.c | 24 ++++++++++++++++++++++++ 2 files changed, 29 insertions(+)include/io/channel-file.h +5
-0@@ -89,4 +89,9 @@ qio_channel_file_new_path(const char *path,mode_t mode, Error **errp); +ssize_t qio_channel_file_pwritev (QIOChannel *ioc, +const struct iovec *iov, +size_t niov, +off_t offset, +Error **errp);#endif /* QIO_CHANNEL_FILE_H */ io/channel-file.c +24
-0@@ -136,6 +136,30 @@ static ssize_t qio_channel_file_writev(QIOChannel *ioc,return ret; } +ssize_t qio_channel_file_pwritev (QIOChannel *ioc, +const struct iovec *iov, +size_t niov, +off_t offset, +Error **errp) +{ +QIOChannelFile *fioc =QIO_CHANNEL_FILE (ioc); +ssize_t ret; + + retry: + ret =pwritev (fioc->fd, iov, niov, offset); +if (ret <=0 ) { +if (errno ==EAGAIN ) { +return QIO_CHANNEL_ERR_BLOCK ; + } +if (errno ==EINTR ) { + goto retry; + } +error_setg_errno (errp, errno,"Unable to write to file" ); +return -1 ; + } +return ret; +} +static int qio_channel_file_set_blocking (QIOChannel *ioc,bool enabled,Error **errp)Signature · 1 line
2.34.1
With regards,
Daniel
Signature · 3 lines
|: https://berrange.com -o- https://www.flickr.com/photos/dberrange :| |: https://libvirt.org -o- https://fstop138.berrange.com :| |: https://entangle-photo.org -o- https://www.instagram.com/dberrange :|
PATCH v2 5/11 io: Add support for seekable channels
In reply to Nikolay BorisovAdd a bunch of auxiliarry methods and a feature flag to work with SEEKABLE channels. Currently the only channel considered seekable is QIOChannelFile. Also add a bunch of helper functions to QEMUFile that can make use of this channel feature. All of this is in prepration for supporting 'fixed-ram' migration stream feature.
--- include/io/channel.h | 1 + include/migration/qemu-file-types.h | 2 + io/channel-file.c | 5 +++ migration/qemu-file.c | 59 +++++++++++++++++++++++++++++ migration/qemu-file.h | 3 ++ 5 files changed, 70 insertions(+)
include/io/channel.h +1 -0
@@ -41,6 +41,7 @@ enum QIOChannelFeature {QIO_CHANNEL_FEATURE_SHUTDOWN ,QIO_CHANNEL_FEATURE_LISTEN ,QIO_CHANNEL_FEATURE_WRITE_ZERO_COPY , + QIO_CHANNEL_FEATURE_SEEKABLE, };
include/migration/qemu-file-types.h +2 -0
@@ -50,6 +50,8 @@ unsigned int qemu_get_be16(QEMUFile *f);unsigned int qemu_get_be32 (QEMUFile *f);uint64_t qemu_get_be64 (QEMUFile *f); +bool qemu_file_is_seekable (QEMUFile *f); +static inline void qemu_put_be64s (QEMUFile *f,const uint64_t *pv) {qemu_put_be64 (f, *pv);
io/channel-file.c +5 -0
@@ -35,6 +35,7 @@ qio_channel_file_new_fd(int fd) ioc->fd = fd; +qio_channel_set_feature (QIO_CHANNEL (ioc),QIO_CHANNEL_FEATURE_SEEKABLE );trace_qio_channel_file_new_fd (ioc, fd);return ioc;
@@ -59,6 +60,10 @@ qio_channel_file_new_path(const char *path,return NULL ; } +if (lseek (ioc->fd,0 ,SEEK_CUR ) != (off_t)-1 ) { +qio_channel_set_feature (QIO_CHANNEL (ioc),QIO_CHANNEL_FEATURE_SEEKABLE ); + } +trace_qio_channel_file_new_path (ioc, path, flags, mode, ioc->fd);return ioc;
migration/qemu-file.c +59 -0
77 lines not on this page · raw message
migration/qemu-file.h +3 -0
@@ -148,6 +148,9 @@ int qemu_file_shutdown(QEMUFile *f);QEMUFile *qemu_file_get_return_path (QEMUFile *f);void qemu_fflush (QEMUFile *f);void qemu_file_set_blocking (QEMUFile *f,bool block); +void qemu_set_offset (QEMUFile *f,off_t off,int whence); +off_t qemu_get_offset (QEMUFile *f); +void qemu_put_buffer_at (QEMUFile *f,const uint8_t *buf,size_t buflen,off_t pos);void ram_control_before_iterate (QEMUFile *f,uint64_t flags);void ram_control_after_iterate (QEMUFile *f,uint64_t flags);
Signature · 1 line
2.34.1
Re: [PATCH v2 05/11] io: Add support for seekable channels
Quoted 5 lines · Nikolay Borisov
Add a bunch of auxiliarry methods and a feature flag to work with SEEKABLE channels. Currently the only channel considered seekable is QIOChannelFile. Also add a bunch of helper functions to QEMUFile that can make use of this channel feature. All of this is in prepration for supporting 'fixed-ram' migration stream feature.
QIOChannelBuffer/Null are also seekable.
Quoted 9 lines
Signed-off-by Nikolay Borisov <[email protected]>--- include/io/channel.h | 1 + include/migration/qemu-file-types.h | 2 + io/channel-file.c | 5 +++ migration/qemu-file.c | 59 +++++++++++++++++++++++++++++ migration/qemu-file.h | 3 ++ 5 files changed, 70 insertions(+)
Can you separate the migration/ tree bits into a second patch that follows the io/ bits.
Quoted 39 lines
include/io/channel.h +1
-0@@ -41,6 +41,7 @@ enum QIOChannelFeature {QIO_CHANNEL_FEATURE_SHUTDOWN ,QIO_CHANNEL_FEATURE_LISTEN ,QIO_CHANNEL_FEATURE_WRITE_ZERO_COPY , + QIO_CHANNEL_FEATURE_SEEKABLE, };include/migration/qemu-file-types.h +2
-0@@ -50,6 +50,8 @@ unsigned int qemu_get_be16(QEMUFile *f);unsigned int qemu_get_be32 (QEMUFile *f);uint64_t qemu_get_be64 (QEMUFile *f); +bool qemu_file_is_seekable (QEMUFile *f); +static inline void qemu_put_be64s (QEMUFile *f,const uint64_t *pv) {qemu_put_be64 (f, *pv);io/channel-file.c +1
-0@@ -35,6 +35,7 @@ qio_channel_file_new_fd(int fd) ioc->fd = fd; +qio_channel_set_feature (QIO_CHANNEL (ioc),QIO_CHANNEL_FEATURE_SEEKABLE );trace_qio_channel_file_new_fd (ioc, fd);return ioc;
io/channel-file.c qio_channel_file_new_path(const char *path, · line 67
Quoted 8 lines
io/channel-file.c +4
-0@@ -59,6 +60,10 @@ qio_channel_file_new_path(const char *path,return NULL ; } +if (lseek (ioc->fd,0 ,SEEK_CUR ) != (off_t)-1 ) { +qio_channel_set_feature (QIO_CHANNEL (ioc),QIO_CHANNEL_FEATURE_SEEKABLE ); + } +trace_qio_channel_file_new_path (ioc, path, flags, mode, ioc->fd);
Wondering why you do the lseek() sanitytest for only one of the two constructors ? Shouldn't we do it for both ?
With regards,
Daniel
Signature · 3 lines
|: https://berrange.com -o- https://www.flickr.com/photos/dberrange :| |: https://libvirt.org -o- https://fstop138.berrange.com :| |: https://entangle-photo.org -o- https://www.instagram.com/dberrange :|
Re: [PATCH v2 05/11] io: Add support for seekable channels
Quoted 8 lines · Daniel P. Berrangé
Quoted 5 lines · Nikolay Borisov
Add a bunch of auxiliarry methods and a feature flag to work with SEEKABLE channels. Currently the only channel considered seekable is QIOChannelFile. Also add a bunch of helper functions to QEMUFile that can make use of this channel feature. All of this is in prepration for supporting 'fixed-ram' migration stream feature.
QIOChannelBuffer/Null are also seekable.
Right, however I think for seek we also want to rely on the feature of filesystem that when you seek beyond EOF you won't actually allocate/use up the space from (eof, CUR_SEEK), with ChannelBuffer we'd have to actually allocate the space or add this support on top.
Quoted 56 lines
Quoted 10 lines
Signed-off-by Nikolay Borisov <[email protected]>--- include/io/channel.h | 1 + include/migration/qemu-file-types.h | 2 + io/channel-file.c | 5 +++ migration/qemu-file.c | 59 +++++++++++++++++++++++++++++ migration/qemu-file.h | 3 ++ 5 files changed, 70 insertions(+)Can you separate the migration/ tree bits into a second patch that follows the io/ bits.
Quoted 42 lines
include/io/channel.h +1
-0@@ -41,6 +41,7 @@ enum QIOChannelFeature {QIO_CHANNEL_FEATURE_SHUTDOWN ,QIO_CHANNEL_FEATURE_LISTEN ,QIO_CHANNEL_FEATURE_WRITE_ZERO_COPY , + QIO_CHANNEL_FEATURE_SEEKABLE, };include/migration/qemu-file-types.h +2
-0@@ -50,6 +50,8 @@ unsigned int qemu_get_be16(QEMUFile *f);unsigned int qemu_get_be32 (QEMUFile *f);uint64_t qemu_get_be64 (QEMUFile *f); +bool qemu_file_is_seekable (QEMUFile *f); +static inline void qemu_put_be64s (QEMUFile *f,const uint64_t *pv) {qemu_put_be64 (f, *pv);io/channel-file.c +1
-0@@ -35,6 +35,7 @@ qio_channel_file_new_fd(int fd) ioc->fd = fd; +qio_channel_set_feature (QIO_CHANNEL (ioc),QIO_CHANNEL_FEATURE_SEEKABLE );trace_qio_channel_file_new_fd (ioc, fd);return ioc;@@ -59,6 +60,10 @@ qio_channel_file_new_path(const char *path,return NULL ; }
Quoted 8 lines
io/channel-file.c qio_channel_file_new_path(const char *path, · line 67
Quoted 5 lines
io/channel-file.c +4
-0+if (lseek (ioc->fd,0 ,SEEK_CUR ) != (off_t)-1 ) { +qio_channel_set_feature (QIO_CHANNEL (ioc),QIO_CHANNEL_FEATURE_SEEKABLE ); + } +trace_qio_channel_file_new_path (ioc, path, flags, mode, ioc->fd);Wondering why you do the
lseek()sanitytest for only one of the two constructors ? Shouldn't we do it for both ?
Good point, AFAIR my train of thought was something along the lines of "what if the path being passed actually lies on a remote fs, we definitely need to test for this support", however now that I think about it a bit more - nothing prevents for the passed in fd to also be lying on a remote fs, so good point. Will fix this in next iteration.
Quoted 6 lines
With regards,
Daniel
Re: [PATCH v2 05/11] io: Add support for seekable channels
Quoted 10 lines · Nikolay Borisov
Quoted 7 lines · Daniel P. Berrangé
Quoted 5 lines · Nikolay Borisov
Add a bunch of auxiliarry methods and a feature flag to work with SEEKABLE channels. Currently the only channel considered seekable is QIOChannelFile. Also add a bunch of helper functions to QEMUFile that can make use of this channel feature. All of this is in prepration for supporting 'fixed-ram' migration stream feature.
Quoted 6 lines
Quoted 1 line
QIOChannelBuffer/Null are also seekable.
Right, however I think for seek we also want to rely on the feature of filesystem that when you seek beyond EOF you won't actually allocate/use up the space from (eof,
CUR_SEEK), with ChannelBuffer we'd have to actually allocate the space or add this support on top.
I'm fine with not implementing this for ChannelBuffer. Mostly making the point that the APIs should be in QIOChannel base class, so that someone could implement it in any subclass in future where it makes sense.
With regards,
Daniel
Signature · 3 lines
|: https://berrange.com -o- https://www.flickr.com/photos/dberrange :| |: https://libvirt.org -o- https://fstop138.berrange.com :| |: https://entangle-photo.org -o- https://www.instagram.com/dberrange :|
PATCH v2 6/11 io: Add preadv support to QIOChannelFile
In reply to Nikolay Borisovpreadv is going to be needed when 'fixed-ram'-enabled stream are to be restored. Simply add a wrapper around preadv that's specific to QIOChannelFile.
--- include/io/channel-file.h | 5 +++++ io/channel-file.c | 26 ++++++++++++++++++++++++++ 2 files changed, 31 insertions(+)
include/io/channel-file.h +5 -0
@@ -89,6 +89,11 @@ qio_channel_file_new_path(const char *path,mode_t mode, Error **errp); +ssize_t qio_channel_file_preadv (QIOChannel *ioc, +const struct iovec *iov, +size_t niov, +off_t offset, +Error **errp); ssize_t qio_channel_file_pwritev(QIOChannel *ioc, const struct iovec *iov, size_t niov,
io/channel-file.c +26 -0
@@ -141,6 +141,32 @@ static ssize_t qio_channel_file_writev(QIOChannel *ioc,return ret; } +ssize_t qio_channel_file_preadv (QIOChannel *ioc, +const struct iovec *iov, +size_t niov, +off_t offset, +Error **errp) +{ +QIOChannelFile *fioc =QIO_CHANNEL_FILE (ioc); +ssize_t ret; + + retry: + ret =preadv (fioc->fd, iov, niov, offset); +if (ret <0 ) { +if (errno ==EAGAIN ) { +return QIO_CHANNEL_ERR_BLOCK ; + } +if (errno ==EINTR ) { + goto retry; + } + +error_setg_errno (errp, errno,"Unable to read from file" ); +return -1 ; + } + +return ret; +} + ssize_t qio_channel_file_pwritev(QIOChannel *ioc, const struct iovec *iov, size_t niov,
Signature · 1 line
2.34.1
PATCH v2 7/11 migration: add qemu_get_buffer_at
In reply to Nikolay BorisovRestoring a 'fixed-ram' enabled migration stream would require reading from specific offsets in the file so add a helper to QEMUFile that uses the newly introduced qio_channel_file_preadv.
--- migration/qemu-file.c | 23 +++++++++++++++++++++++ migration/qemu-file.h | 1 + 2 files changed, 24 insertions(+)
migration/qemu-file.c +23 -0
@@ -564,6 +564,29 @@ void qemu_put_buffer_at(QEMUFile *f, const uint8_t *buf, size_t buflen, off_t poreturn ; } + +size_t qemu_get_buffer_at (QEMUFile *f,const uint8_t *buf,size_t buflen,off_t pos) +{ +Error *err =NULL ; +struct iovec iov = { .iov_base = (char *)buf, .iov_len = buflen }; +ssize_t ret; + +if (f->last_error) { +return 0 ; + } + + ret =qio_channel_file_preadv (f->ioc, &iov,1 , pos, &err); +if (ret ==-1 ) { + goto error; + } + +return (size_t )ret; + + error: +qemu_file_set_error_obj (f, -EIO , err); +return 0 ; +} +void qemu_set_offset (QEMUFile *f,off_t off,int whence) {Error *err =NULL ;
migration/qemu-file.h +1 -0
@@ -151,6 +151,7 @@ void qemu_file_set_blocking(QEMUFile *f, bool block);void qemu_set_offset (QEMUFile *f,off_t off,int whence);off_t qemu_get_offset (QEMUFile *f);void qemu_put_buffer_at (QEMUFile *f,const uint8_t *buf,size_t buflen,off_t pos); +size_t qemu_get_buffer_at (QEMUFile *f,const uint8_t *buf,size_t buflen,off_t pos);void ram_control_before_iterate (QEMUFile *f,uint64_t flags);void ram_control_after_iterate (QEMUFile *f,uint64_t flags);
Signature · 1 line
2.34.1
PATCH v2 8/11 migration/ram: Introduce 'fixed-ram' migration stream capability
In reply to Nikolay BorisovImplement 'fixed-ram' feature. The core of the feature is to ensure that each ram page of the migration stream has a specific offset in the resulting migration stream. The reason why we'd want such behavior are two fold:
- When doing a 'fixed-ram' migration the resulting file will have a bounded size, since pages which are dirtied multiple times will always go to a fixed location in the file, rather than constantly being added to a sequential stream. This eliminates cases where a vm with, say, 1g of ram can result in a migration file that's 10s of Gbs, provided that the workload constantly redirties memory.
- It paves the way to implement DIO-enabled save/restore of the migration stream as the pages are ensured to be written at aligned offsets.
The features requires changing the format. First, a bitmap is introduced which tracks which pages have been written (i.e are dirtied) during migration and subsequently it's being written in the resultin file, again at a fixed location for every ramblock. Zero pages are ignored as they'd be zero in the destination migration as well. With the changed format data would look like the following:
|name len|name|used_len|pc*|bitmap_size|pages_offset|bitmap|pages|
* pc - refers to the page_size/mr->addr members, so newly added members begin from "bitmap_size".
This layout is initialized during ram_save_setup so instead of having a sequential stream of pages that follow the ramblock headers the dirty pages for a ramblock follow its header. Since all pages have a fixed location RAM_SAVE_FLAG_EOS is no longer generated on every migration iteration but there is effectively a single RAM_SAVE_FLAG_EOS right at the end.
--- include/exec/ramblock.h | 7 +++ migration/migration.c | 51 +++++++++++++++++++++- migration/migration.h | 1 + migration/ram.c | 97 +++++++++++++++++++++++++++++++++-------- migration/savevm.c | 1 + qapi/migration.json | 2 +- 6 files changed, 138 insertions(+), 21 deletions(-)
include/exec/ramblock.h +7 -0
@@ -43,6 +43,13 @@ struct RAMBlock {size_t page_size;/* dirty bitmap used during migration */ unsigned long *bmap; +/* shadow dirty bitmap used when migrating to a file */ +unsigned long *shadow_bmap; +/* offset in the file pages belonging to this ramblock are saved, used +* only during migration to a file +*/ +off_t bitmap_offset; +uint64_t pages_offset;/* bitmap of already received pages in postcopy */ unsigned long *receivedmap;
migration/migration.c +50 -1
87 lines not on this page · raw message
migration/migration.h +1 -0
@@ -410,6 +410,7 @@ bool migrate_zero_blocks(void);bool migrate_dirty_bitmaps (void );bool migrate_ignore_shared (void );bool migrate_validate_uuid (void ); +int migrate_fixed_ram (void );bool migrate_auto_converge (void );bool migrate_use_multifd (void );
migration/ram.c +78 -19
156 lines not on this page · raw message
migration/savevm.c +1 -0
@@ -240,6 +240,7 @@ static bool should_validate_capability(int capability)/* Validate only new capabilities to keep compatibility. */ switch (capability) {case MIGRATION_CAPABILITY_X_IGNORE_SHARED : +case MIGRATION_CAPABILITY_FIXED_RAM :return true;default :return false;
qapi/migration.json +1 -1
@@ -485,7 +485,7 @@ ## { 'enum': 'MigrationCapability', 'data': ['xbzrle', 'rdma-pin-all', 'auto-converge', 'zero-blocks', - 'compress', 'events', 'postcopy-ram', + 'compress', 'events', 'postcopy-ram', 'fixed-ram', { 'name': 'x-colo', 'features': [ 'unstable' ] }, 'release-ram', 'block', 'return-path', 'pause-before-switchover', 'multifd',
Signature · 1 line
2.34.1
PATCH v2 9/11 migration: Refactor precopy ram loading code
In reply to Nikolay BorisovTo facilitate easier implementaiton of the 'fixed-ram' migration restore factor out the code responsible for parsing the ramblocks headers. This also makes ram_load_precopy easier to comprehend.
--- migration/ram.c | 142 +++++++++++++++++++++++++++--------------------- 1 file changed, 80 insertions(+), 62 deletions(-)
migration/ram.c +80 -62
@@ -4253,6 +4253,83 @@ void colo_flush_ram_cache(void)trace_colo_flush_ram_cache_end (); } +static int parse_ramblock (QEMUFile *f,RAMBlock *block,ram_addr_t length) +{ +int ret =0 ; +/* ADVISE is earlier, it shows the source has the postcopy capability on */ +bool postcopy_advised =postcopy_is_advised (); + +assert (block); + +if (!qemu_ram_is_migratable (block)) { +error_report ("block %s should not be migrated !" , block->idstr); + ret = -EINVAL ; + } + +if (length != block->used_length) { +Error *local_err =NULL ; + + ret =qemu_ram_resize (block, length, &local_err); +if (local_err) { +error_report_err (local_err); + } + } +/* For postcopy we need to check hugepage sizes match */ +if (postcopy_advised &&migrate_postcopy_ram () && + block->page_size != qemu_host_page_size) { +uint64_t remote_page_size =qemu_get_be64 (f); +if (remote_page_size != block->page_size) { +error_report ("Mismatched RAM page size %s " +"(local) %zd != %" PRId64, block->idstr, + block->page_size, remote_page_size); + ret = -EINVAL ; + } + } +if (migrate_ignore_shared ()) { +hwaddr addr =qemu_get_be64 (f); +if (ramblock_is_ignored (block) && + block->mr->addr != addr) { +error_report ("Mismatched GPAs for block %s " +"%" PRId64"!= %" PRId64, block->idstr, + (uint64_t )addr, + (uint64_t )block->mr->addr); + ret = -EINVAL ; + } + } +ram_control_load_hook (f,RAM_CONTROL_BLOCK_REG , block->idstr); + +return ret; +} + +static int parse_ramblocks (QEMUFile *f,ram_addr_t total_ram_bytes) +{ +int ret =0 ; + +/* Synchronize RAM block list */ +while (!ret && total_ram_bytes) { +char id[256 ]; +RAMBlock *block; +ram_addr_t length; +int len =qemu_get_byte (f); + +qemu_get_buffer (f, (uint8_t *)id, len); + id[len] =0 ; + length =qemu_get_be64 (f); + + block =qemu_ram_block_by_name (id); +if (block) { + ret =parse_ramblock (f, block, length); + }else { +error_report ("Unknown ramblock \"%s\", cannot accept " +"migration" , id); + ret = -EINVAL ; + } + total_ram_bytes -= length; + } + +return ret; +} +/** * ram_load_precopy: load pages in precopy case *
@@ -4267,14 +4344,13 @@ static int ram_load_precopy(QEMUFile *f) {MigrationIncomingState *mis =migration_incoming_get_current ();int flags =0 , ret =0 , invalid_flags =0 , len =0 , i =0 ; -/* ADVISE is earlier, it shows the source has the postcopy capability on */ -bool postcopy_advised =postcopy_is_advised (); +if (!migrate_use_compression ()) { invalid_flags |=RAM_SAVE_FLAG_COMPRESS_PAGE ; }while (!ret && !(flags &RAM_SAVE_FLAG_EOS )) { -ram_addr_t addr, total_ram_bytes; +ram_addr_t addr;void *host =NULL , *host_bak =NULL ;uint8_t ch;
@@ -4345,65 +4421,7 @@ static int ram_load_precopy(QEMUFile *f)switch (flags & ~RAM_SAVE_FLAG_CONTINUE ) {case RAM_SAVE_FLAG_MEM_SIZE : -/* Synchronize RAM block list */ - total_ram_bytes = addr; -while (!ret && total_ram_bytes) { -RAMBlock *block; -char id[256 ]; -ram_addr_t length; - - len =qemu_get_byte (f); -qemu_get_buffer (f, (uint8_t *)id, len); - id[len] =0 ; - length =qemu_get_be64 (f); - - block =qemu_ram_block_by_name (id); -if (block && !qemu_ram_is_migratable (block)) { -error_report ("block %s should not be migrated !" , id); - ret = -EINVAL ; - }else if (block) { -if (length != block->used_length) { -Error *local_err =NULL ; - - ret =qemu_ram_resize (block, length, - &local_err); -if (local_err) { -error_report_err (local_err); - } - } -/* For postcopy we need to check hugepage sizes match */ -if (postcopy_advised &&migrate_postcopy_ram () && - block->page_size != qemu_host_page_size) { -uint64_t remote_page_size =qemu_get_be64 (f); -if (remote_page_size != block->page_size) { -error_report ("Mismatched RAM page size %s " -"(local) %zd != %" PRId64, - id, block->page_size, - remote_page_size); - ret = -EINVAL ; - } - } -if (migrate_ignore_shared ()) { -hwaddr addr =qemu_get_be64 (f); -if (ramblock_is_ignored (block) && - block->mr->addr != addr) { -error_report ("Mismatched GPAs for block %s " -"%" PRId64"!= %" PRId64, - id, (uint64_t )addr, - (uint64_t )block->mr->addr); - ret = -EINVAL ; - } - } -ram_control_load_hook (f,RAM_CONTROL_BLOCK_REG , - block->idstr); - }else { -error_report ("Unknown ramblock \"%s\", cannot " -"accept migration" , id); - ret = -EINVAL ; - } - - total_ram_bytes -= length; - } + ret =parse_ramblocks (f, addr);break ;case RAM_SAVE_FLAG_ZERO :
Signature · 1 line
2.34.1
PATCH v2 10/11 migration: Add support for 'fixed-ram' migration restore
In reply to Nikolay BorisovAdd the necessary code to parse the format changes for 'fixed-ram' capability. One of the more notable changes in behavior is that in the 'fixed-ram' case ram pages are restored in one go rather than constantly looping through the migration stream. Also due to idiosyncrasies of the format I have added the 'ram_migrated' since it was easier to simply return directly from ->load_state rather than introducing more conditionals around the code to prevent ->load_state being called multiple times (from qemu_loadvm_section_start_full/qemu_loadvm_section_part_end i.e. from multiple QEMU_VM_SECTION_(PART|END) flags).
--- migration/migration.h | 2 + migration/ram.c | 95 ++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 95 insertions(+), 2 deletions(-)
migration/migration.h +2 -0
@@ -96,6 +96,8 @@ struct MigrationIncomingState {bool have_listen_thread;QemuThread listen_thread; +bool ram_migrated; +/* For the kernel to send us notifications */ int userfault_fd;/* To notify the fault_thread to wake, e.g., when need to quit */
migration/ram.c +93 -2
113 lines not on this page · raw message
Signature · 1 line
2.34.1
PATCH v2 11/11 analyze-migration.py: add initial support for fixed ram streams
In reply to Nikolay BorisovThis commit introduces the minimum code necessary to support parsing migration strems with 'fixed-ram' capability set. The only thing really missing is the implementation of write_or_dump_fixed_ram() which deals with '-x'/'-m' options.
--- scripts/analyze-migration.py | 49 +++++++++++++++++++++++++++++++++--- 1 file changed, 46 insertions(+), 3 deletions(-)
scripts/analyze-migration.py +46 -3
@@ -23,6 +23,7 @@ import collections import struct import sys +import math def mkdir_p(path):
@@ -119,11 +120,16 @@ def __init__(self, file, version_id, ramargs, section_key): self.file = file self.section_key = section_key self.TARGET_PAGE_SIZE = ramargs['page_size'] + self.TARGET_PAGE_BITS = math.log2(self.TARGET_PAGE_SIZE) self.dump_memory = ramargs['dump_memory'] self.write_memory = ramargs['write_memory'] + self.fixed_ram = ramargs['fixed-ram'] self.sizeinfo = collections.OrderedDict() + self.bitmap_offset = collections.OrderedDict() + self.pages_offset = collections.OrderedDict() self.data = collections.OrderedDict() self.data['section sizes'] = self.sizeinfo + self.ram_read = False self.name = '' if self.write_memory: self.files = { }
@@ -140,7 +146,13 @@ def __str__(self): def getDict(self): return self.data + def write_or_dump_fixed_ram(self): + pass + def read(self): + if self.fixed_ram and self.ram_read: + return + # Read all RAM sections while True: addr = self.file.read64()
@@ -167,7 +179,25 @@ def read(self): f.truncate(0) f.truncate(len) self.files[self.name] = f + + if self.fixed_ram: + bitmap_len = self.file.read32() + # skip the pages_offset which we don't need + offset = self.file.tell() + 8 + self.bitmap_offset[self.name] = offset + offset = ((offset + bitmap_len + self.TARGET_PAGE_SIZE - 1) // self.TARGET_PAGE_SIZE) * self.TARGET_PAGE_SIZE + self.pages_offset[self.name] = offset + self.file.file.seek(offset + len) + flags &= ~self.RAM_SAVE_FLAG_MEM_SIZE + if self.fixed_ram: + self.ram_read = True + # now we should rewind to the ram page offset of the first + # ram section + if self.fixed_ram: + if self.write_memory or self.dump_memory: + self.write_or_dump_fixed_ram() + return if flags & self.RAM_SAVE_FLAG_COMPRESS: if flags & self.RAM_SAVE_FLAG_CONTINUE:
@@ -208,7 +238,7 @@ def read(self): # End of RAM section if flags & self.RAM_SAVE_FLAG_EOS: - break + return if flags != 0: raise Exception("Unknown RAM flags: %x" % flags)
@@ -521,6 +551,7 @@ def read(self, desc_only = False, dump_memory = False, write_memory = False): ramargs['page_size'] = self.vmsd_desc['page_size'] ramargs['dump_memory'] = dump_memory ramargs['write_memory'] = write_memory + ramargs['fixed-ram'] = False self.section_classes[('ram',0)][1] = ramargs while True:
@@ -528,8 +559,20 @@ def read(self, desc_only = False, dump_memory = False, write_memory = False): if section_type == self.QEMU_VM_EOF: break elif section_type == self.QEMU_VM_CONFIGURATION: - section = ConfigurationSection(file) - section.read() + config_desc = self.vmsd_desc.get('configuration') + if config_desc is not None: + config = VMSDSection(file, 1, config_desc, 'configuration') + config.read() + caps = config.data.get("configuration/capabilities") + if caps is not None: + caps = caps.data["capabilities"] + if type(caps) != list: + caps = [caps] + for i in caps: + # chomp out string length + cap = i.data[1:].decode("utf8") + if cap == "fixed-ram": + ramargs['fixed-ram'] = True elif section_type == self.QEMU_VM_SECTION_START or section_type == self.QEMU_VM_SECTION_FULL: section_id = file.read32() name = file.readstr()
Signature · 1 line
2.34.1