migration: Better error handling in return path thread
cover migration: Better error handling in return path thread
This is a small series that reworks error handling of postcopy return path threads.
We used to contain a bunch of error_report(), converting them into error_setg() properly and deliver any of those errors to migration generic error reports (via migrate_set_error()). Then these errors can also be observed in query-migrate after postcopy is paused.
Dropped the return-path specific error reporting: mark_source_rp_bad(), because it's a duplication if we can always use migrate_set_error().
Please have a look, thanks.
Peter Xu (7):
migration: Let migrate_set_error() take ownership
migration: Introduce migrate_has_error()
migration: Refactor error handling in source return path
migration: Deliver return path file error to migrate state too
migration: Display error in query-migrate irrelevant of status
qemufile: Always return a verbose error
migration: Provide explicit error message for file shutdowns
migration/migration.h | 8 +- migration/ram.h | 5 +- migration/channel.c | 1 - migration/migration.c | 168 +++++++++++++++++++++++---------------- migration/multifd.c | 10 +-- migration/postcopy-ram.c | 1 - migration/qemu-file.c | 20 ++++- migration/ram.c | 42 +++++----- migration/trace-events | 2 +- 9 files changed, 147 insertions(+), 110 deletions(-)
Signature · 1 line
2.41.0
PATCH 1/7 migration: Let migrate_set_error() take ownership
migrate_set_error() used one error_copy() so it always copy an error. However that's not the major use case - the major use case is one would like to pass the error to migrate_set_error() without further touching the error.
It can be proved if we see most of the callers are freeing the error explicitly right afterwards. There're a few outliers (only if when the caller) where we can use error_copy() explicitly there.
--- migration/migration.h | 6 +++--- migration/channel.c | 1 - migration/migration.c | 20 ++++++++++++++------ migration/multifd.c | 10 ++++------ migration/postcopy-ram.c | 1 - migration/ram.c | 1 - 6 files changed, 21 insertions(+), 18 deletions(-)
migration/migration.h +3 -3
@@ -468,8 +468,8 @@ bool migration_has_all_channels(void);uint64_t migrate_max_downtime (void ); -void migrate_set_error (MigrationState *s,const Error *error); -void migrate_fd_error (MigrationState *s,const Error *error); +void migrate_set_error (MigrationState *s,Error *error); +void migrate_fd_error (MigrationState *s,Error *error);void migrate_fd_connect (MigrationState *s,Error *error_in);
@@ -513,7 +513,7 @@ int foreach_not_ignored_block(RAMBlockIterFunc func, void *opaque);void migration_make_urgent_request (void );void migration_consume_urgent_request (void );bool migration_rate_limit (void ); -void migration_cancel (const Error *error); +void migration_cancel (Error *error);void populate_vfio_info (MigrationInfo *info);void postcopy_temp_page_reset (PostcopyTmpPage *tmp_page);
migration/channel.c +0 -1
@@ -90,7 +90,6 @@ void migration_channel_connect(MigrationState *s, } }migrate_fd_connect (s, error); -error_free (error); }
migration/migration.c +14 -6
@@ -160,7 +160,7 @@ void migration_object_init(void)dirty_bitmap_mig_init (); } -void migration_cancel (const Error *error) +void migration_cancel (Error *error) {if (error) {migrate_set_error (current_migration, error);
@@ -1197,11 +1197,20 @@ static void migrate_fd_cleanup_bh(void *opaque)object_unref (OBJECT (s)); } -void migrate_set_error (MigrationState *s,const Error *error) +/* +* Set error for current migration state. The `error' ownership will be +* moved from the caller to MigrationState, so the caller doesn't need to +* free the error. +* +* If the caller still needs to reference the `error' passed in, one should +* use error_copy() explicitly. +*/ +void migrate_set_error (MigrationState *s,Error *error) {QEMU_LOCK_GUARD (&s->error_mutex);if (!s->error) { - s->error =error_copy (error); +/* Record the first error triggered */ + s->error = error; } }
@@ -1214,7 +1223,7 @@ static void migrate_error_free(MigrationState *s) } } -void migrate_fd_error (MigrationState *s,const Error *error) +void migrate_fd_error (MigrationState *s,Error *error) {trace_migrate_fd_error (error_get_pretty (error));assert (s->to_dst_file ==NULL );
@@ -1678,7 +1687,7 @@ void qmp_migrate(const char *uri, bool has_blk, bool blk,if (!(has_resume && resume)) {yank_unregister_instance (MIGRATION_YANK_INSTANCE ); } -migrate_fd_error (s, local_err); +migrate_fd_error (s,error_copy (local_err));error_propagate (errp, local_err);return ; }
@@ -2595,7 +2604,6 @@ static MigThrError migration_detect_error(MigrationState *s)if (local_error) {migrate_set_error (s, local_error); -error_free (local_error); }if (state ==MIGRATION_STATUS_POSTCOPY_ACTIVE && ret) {
migration/multifd.c +4 -6
@@ -551,7 +551,6 @@ void multifd_save_cleanup(void) multifd_send_state->ops->send_cleanup (p, &local_err);if (local_err) {migrate_set_error (migrate_get_current (), local_err); -error_free (local_err); } }qemu_sem_destroy (&multifd_send_state->channels_ready);
@@ -750,7 +749,6 @@ out:if (local_err) {trace_multifd_send_error (p->id);multifd_send_terminate_threads (local_err); -error_free (local_err); }/*
@@ -883,7 +881,6 @@ static void multifd_new_send_channel_cleanup(MultiFDSendParams *p,*/ p->quit = true;object_unref (OBJECT (ioc)); -error_free (err); }static void multifd_new_send_channel_async (QIOTask *task,gpointer opaque)
@@ -1148,7 +1145,6 @@ static void *multifd_recv_thread(void *opaque)if (local_err) {multifd_recv_terminate_threads (local_err); -error_free (local_err); }qemu_mutex_lock (&p->mutex); p->running = false;
@@ -1240,7 +1236,8 @@ void multifd_recv_new_channel(QIOChannel *ioc, Error **errp) id =multifd_recv_initial_packet (ioc, &local_err);if (id <0 ) { -multifd_recv_terminate_threads (local_err); +/* Copy local error because we'll also return it to caller */ +multifd_recv_terminate_threads (error_copy (local_err)); error_propagate_prepend(errp, local_err, "failed to receive packet" " via multifd channel %d: ",
@@ -1253,7 +1250,8 @@ void multifd_recv_new_channel(QIOChannel *ioc, Error **errp)if (p->c !=NULL ) {error_setg (&local_err,"multifd: received id '%d' already setup'" , id); -multifd_recv_terminate_threads (local_err); +/* Copy local error because we'll also return it to caller */ +multifd_recv_terminate_threads (error_copy (local_err));error_propagate (errp, local_err);return ; }
migration/postcopy-ram.c +0 -1
@@ -1594,7 +1594,6 @@ postcopy_preempt_send_channel_done(MigrationState *s, {if (local_err) {migrate_set_error (s, local_err); -error_free (local_err); }else {migration_ioc_register_yank (ioc); s->postcopy_qemufile_src =qemu_file_new_output (ioc);
migration/ram.c +0 -1
@@ -4267,7 +4267,6 @@ static void ram_mig_ram_block_resized(RAMBlockNotifier *n, void *host,*/ error_setg (&err,"RAM block '%s' resized during precopy." , rb->idstr);migration_cancel (err); -error_free (err); }switch (ps) {
Signature · 1 line
2.41.0
PATCH 2/7 migration: Introduce migrate_has_error()
In reply to Peter XuIntroduce a helper to detect whether MigrationState.error is set for whatever reason. It is intended to not taking the error_mutex here because neither do we reference the pointer, nor do we modify the pointer. State why it's safe to do so.
This is preparation work for any thread (e.g. source return path thread) to setup errors in an unified way to MigrationState, rather than relying on its own way to set errors (mark_source_rp_bad()).
--- migration/migration.h | 1 + migration/migration.c | 15 +++++++++++++++ 2 files changed, 16 insertions(+)
migration/migration.h +1 -0
@@ -469,6 +469,7 @@ bool migration_has_all_channels(void);uint64_t migrate_max_downtime (void );void migrate_set_error (MigrationState *s,Error *error); +bool migrate_has_error (MigrationState *s);void migrate_fd_error (MigrationState *s,Error *error);void migrate_fd_connect (MigrationState *s,Error *error_in);
migration/migration.c +15 -0
@@ -1214,6 +1214,21 @@ void migrate_set_error(MigrationState *s, Error *error) } } +/* +* Whether the migration state has error set? +* +* Note this function explicitly didn't use error_mutex, because it only +* reads the error pointer for a boolean status. +* +* As long as the Error* is set, it shouldn't be freed before migration +* cleanup, so any thread can use this helper to safely detect whether +* there's anything wrong happened already. +*/ +bool migrate_has_error (MigrationState *s) +{ +return qatomic_read (&s->error); +} +static void migrate_error_free (MigrationState *s) {QEMU_LOCK_GUARD (&s->error_mutex);
Signature · 1 line
2.41.0
PATCH 3/7 migration: Refactor error handling in source return path
In reply to Peter Xurp_state.error was a boolean used to show error happened in return path thread. That's not only duplicating error reporting (migrate_set_error), but also not good enough in that we only do error_report() and set it to true, we never can keep a history of the exact error and show it in query-migrate.
To make this better, a few things done:
- Use error_setg() rather than error_report() across the whole lifecycle of return path thread, keeping the error in an Error*.
- Use migrate_set_error() to apply that captured error to the global migration object when error occured in this thread.
- With above, no need to have mark_source_rp_bad(), remove it, alongside with rp_state.error itself.
--- migration/migration.h | 1 - migration/ram.h | 5 +- migration/migration.c | 118 ++++++++++++++++++++--------------------- migration/ram.c | 41 +++++++------- migration/trace-events | 2 +- 5 files changed, 82 insertions(+), 85 deletions(-)
migration/migration.h +0 -1
@@ -290,7 +290,6 @@ struct MigrationState {/* Protected by qemu_file_lock */ QEMUFile *from_dst_file;QemuThread rp_thread; -bool error;/* * We can also check non-zero of rp_thread, but there's no "official" * way to do this, so this bool makes it slightly more elegant.
migration/ram.h +3 -2
@@ -52,7 +52,8 @@ uint64_t ram_bytes_total(void);void mig_throttle_counter_reset (void );uint64_t ram_pagesize_summary (void ); -int ram_save_queue_pages (const char *rbname,ram_addr_t start,ram_addr_t len); +int ram_save_queue_pages (const char *rbname,ram_addr_t start,ram_addr_t len, +Error **errp);void ram_postcopy_migrated_memory_release (MigrationState *ms);/* For outgoing discard bitmap */ void ram_postcopy_send_discard_bitmap (MigrationState *ms);
@@ -72,7 +73,7 @@ void ramblock_recv_bitmap_set(RAMBlock *rb, void *host_addr);void ramblock_recv_bitmap_set_range (RAMBlock *rb,void *host_addr,size_t nr);int64_t ramblock_recv_bitmap_send (QEMUFile *file,const char *block_name); -int ram_dirty_bitmap_reload (MigrationState *s,RAMBlock *rb); +int ram_dirty_bitmap_reload (MigrationState *s,RAMBlock *rb,Error **errp);bool ramblock_page_is_discarded (RAMBlock *rb,ram_addr_t start);void postcopy_preempt_shutdown_file (MigrationState *s);void *postcopy_preempt_thread (void *opaque);
migration/migration.c +57 -61
275 lines not on this page · raw message
migration/ram.c +21 -20
100 lines not on this page · raw message
migration/trace-events +1 -1
@@ -161,7 +161,7 @@ migration_completion_postcopy_end_after_complete(void) "" migration_rate_limit_pre(int ms) "%d ms" migration_rate_limit_post(int urgent) "urgent: %d" migration_return_path_end_before(void) "" -migration_return_path_end_after(int rp_error) "%d" +migration_return_path_end_after(void) "" migration_thread_after_loop(void) "" migration_thread_file_err(void) "" migration_thread_setup_complete(void) ""
Signature · 1 line
2.41.0
Re: [PATCH 3/7] migration: Refactor error handling in source return path
Quoted 11 lines · Peter Xu
rp_state.error was a boolean used to show error happened in return path thread. That's not only duplicating error reporting (
migrate_set_error), but also not good enough in that we only doerror_report()and set it to true, we never can keep a history of the exact error and show it in query-migrate.To make this better, a few things done:
- Use
error_setg()rather thanerror_report()across the whole lifecycle of return path thread, keeping the error in an Error*.
Quoted 7 lines
- Use
migrate_set_error()to apply that captured error to the global migration object when error occured in this thread.- With above, no need to have
mark_source_rp_bad(), remove it, alongside with rp_state.error itself.Signed-off-by Peter Xu <[email protected]>
PATCH 4/7 migration: Deliver return path file error to migrate state too
In reply to Peter XuWe've already did this for most of the return path thread errors, but not yet for the IO errors happened on the return path qemufile. Do that too.
Remember to reset "err" always, because the ownership is not us anymore, otherwise we're prone to use-after-free later after recovered.
--- migration/migration.c | 7 +++++++ 1 file changed, 7 insertions(+)
migration/migration.c +7 -0
@@ -1992,6 +1992,13 @@ out: res =qemu_file_get_error (rp);if (res) { +/* We have forwarded any error in "err" already, reuse "error" */ +assert (err ==NULL ); +/* Try to deliver this file error to migration state */ +qemu_file_get_error_obj (rp, &err); +migrate_set_error (ms, err); + err =NULL ; +if (res &&migration_in_postcopy ()) {/* * Maybe there is something we can do: it looks like a
Signature · 1 line
2.41.0
Re: [PATCH 4/7] migration: Deliver return path file error to migrate state too
Quoted 7 lines · Peter Xu
We've already did this for most of the return path thread errors, but not yet for the IO errors happened on the return path qemufile. Do that too.
Remember to reset "err" always, because the ownership is not us anymore, otherwise we're prone to use-after-free later after recovered.
Signed-off-by Peter Xu <[email protected]>
PATCH 5/7 migration: Display error in query-migrate irrelevant of status
In reply to Peter XuDisplay it as long as being set, irrelevant of FAILED status. E.g., it may also be applicable to PAUSED stage of postcopy, to provide hint on what has gone wrong.
The error_mutex seems to be overlooked when referencing the error, add it to be very safe.
--- migration/migration.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-)
migration/migration.c +5 -3
@@ -1033,9 +1033,6 @@ static void fill_source_migration_info(MigrationInfo *info)break ;case MIGRATION_STATUS_FAILED : info->has_status = true; -if (s->error) { - info->error_desc =g_strdup (error_get_pretty (s->error)); - }break ;case MIGRATION_STATUS_CANCELLED : info->has_status = true;
@@ -1045,6 +1042,11 @@ static void fill_source_migration_info(MigrationInfo *info)break ; } info->status = state; + +QEMU_LOCK_GUARD (&s->error_mutex); +if (s->error) { + info->error_desc =g_strdup (error_get_pretty (s->error)); + } }static void fill_destination_migration_info (MigrationInfo *info)
Signature · 1 line
2.41.0
Re: [PATCH 5/7] migration: Display error in query-migrate irrelevant of status
Quoted 3 lines · Peter Xu
Display it as long as being set, irrelevant of FAILED status. E.g., it may also be applicable to PAUSED stage of postcopy, to provide hint on what has gone wrong.
This might have made the documentation slightly inaccurate:
# @error-desc: the human readable error description string, when
# @status is 'failed'. Clients should not attempt to parse the
# error strings. (Since 2.7)
But it's not wrong, so:
Re: [PATCH 5/7] migration: Display error in query-migrate irrelevant of status
Quoted 7 lines · Fabiano Rosas
Quoted 3 lines · Peter Xu
Display it as long as being set, irrelevant of FAILED status. E.g., it may also be applicable to PAUSED stage of postcopy, to provide hint on what has gone wrong.
This might have made the documentation slightly inaccurate:
Hmm yes, maybe I should touch that up so as to include "postcopy-paused", or just remove the statement that it must be in a "failed" stage.
Quoted 8 lines
# @error-desc: the human readable error description string, when
# @status is 'failed'. Clients should not attempt to parse the
# error strings. (Since 2.7)But it's not wrong, so:
Reviewed-by Fabiano Rosas <[email protected]>
Thanks for taking a look.
Signature · 1 line
Peter Xu
PATCH 6/7 qemufile: Always return a verbose error
In reply to Peter XuThere're a lot of cases where we only have an errno set in last_error but without a detailed error description. When this happens, try to generate an error contains the errno as a descriptive error.
This will be helpful in cases where one relies on the Error*. E.g., migration state only caches Error* in MigrationState.error. With this, we'll display correct error messages in e.g. query-migrate when the error was only set by qemu_file_set_error().
--- migration/qemu-file.c | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-)
migration/qemu-file.c +12 -3
@@ -156,15 +156,24 @@ void qemu_file_set_hooks(QEMUFile *f, const QEMUFileHooks *hooks)* * Return negative error value if there has been an error on previous * operations, return 0 if no error happened. -* Optional, it returns Error* in errp, but it may be NULL even if return value -* is not 0. * +* If errp is specified, a verbose error message will be copied over. */ int qemu_file_get_error_obj (QEMUFile *f,Error **errp) { +if (!f->last_error) { +return 0 ; + } + +/* There is an error */ if (errp) { - *errp = f->last_error_obj ?error_copy (f->last_error_obj) :NULL ; +if (f->last_error_obj) { + *errp =error_copy (f->last_error_obj); + }else { +error_setg_errno (errp, -f->last_error,"Channel error" ); + } } +return f->last_error; }
Signature · 1 line
2.41.0
PATCH 7/7 migration: Provide explicit error message for file shutdowns
In reply to Peter XuProvide an explicit reason for qemu_file_shutdown()s, which can be displayed in query-migrate when used.
This will make e.g. migrate-pause to display explicit error descriptions, from:
"error-desc": "Channel error: Input/output error"
To:
"error-desc": "Channel is explicitly shutdown by the user"
in query-migrate.
--- migration/qemu-file.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-)
migration/qemu-file.c +4 -1
@@ -87,7 +87,10 @@ int qemu_file_shutdown(QEMUFile *f)* --> guest crash! */ if (!f->last_error) { -qemu_file_set_error (f, -EIO ); +Error *err =NULL ; + +error_setg (&err,"Channel is explicitly shutdown by the user" ); +qemu_file_set_error_obj (f, -EIO , err); } if (!qio_channel_has_feature(f->ioc,
Signature · 1 line
2.41.0