Skip to contentSkip to conversation

migration: Better error handling in return path thread

PATCH Superseded Peter Xu 2 participants 12 messages Jun 28, 2023 – Jun 29, 2023

v1 of 2 Superseded Latest is v2 →

lore
Share & tools
lore

Peter Xu

author
#0 lore raw

cover migration: Better error handling in return path thread

This is a small series that reworks error handling of postcopy return path threads.

We used to contain a bunch of error_report(), converting them into error_setg() properly and deliver any of those errors to migration generic error reports (via migrate_set_error()). Then these errors can also be observed in query-migrate after postcopy is paused.

Dropped the return-path specific error reporting: mark_source_rp_bad(), because it's a duplication if we can always use migrate_set_error().

Please have a look, thanks.

Peter Xu (7):
migration: Let migrate_set_error() take ownership
migration: Introduce migrate_has_error()
migration: Refactor error handling in source return path
migration: Deliver return path file error to migrate state too
migration: Display error in query-migrate irrelevant of status
qemufile: Always return a verbose error
migration: Provide explicit error message for file shutdowns

 migration/migration.h    |   8 +-
 migration/ram.h          |   5 +-
 migration/channel.c      |   1 -
 migration/migration.c    | 168 +++++++++++++++++++++++----------------
 migration/multifd.c      |  10 +--
 migration/postcopy-ram.c |   1 -
 migration/qemu-file.c    |  20 ++++-
 migration/ram.c          |  42 +++++-----
 migration/trace-events   |   2 +-
 9 files changed, 147 insertions(+), 110 deletions(-)
Signature · 1 line
2.41.0
author
#1 lore raw

PATCH 1/7 migration: Let migrate_set_error() take ownership

migrate_set_error() used one error_copy() so it always copy an error. However that's not the major use case - the major use case is one would like to pass the error to migrate_set_error() without further touching the error.

It can be proved if we see most of the callers are freeing the error explicitly right afterwards. There're a few outliers (only if when the caller) where we can use error_copy() explicitly there.

---
 migration/migration.h    |  6 +++---
 migration/channel.c      |  1 -
 migration/migration.c    | 20 ++++++++++++++------
 migration/multifd.c      | 10 ++++------
 migration/postcopy-ram.c |  1 -
 migration/ram.c          |  1 -
 6 files changed, 21 insertions(+), 18 deletions(-)
migration/migration.h +3 -3
index 721b1c9473..32f87e3834 100644
@@ -468,8 +468,8 @@ bool  migration_has_all_channels(void);
 
 uint64_t migrate_max_downtime(void);
 
-void migrate_set_error(MigrationState *s, const Error *error);
-void migrate_fd_error(MigrationState *s, const Error *error);
+void migrate_set_error(MigrationState *s, Error *error);
+void migrate_fd_error(MigrationState *s, Error *error);
 
 void migrate_fd_connect(MigrationState *s, Error *error_in);
 
@@ -513,7 +513,7 @@ int foreach_not_ignored_block(RAMBlockIterFunc func, void *opaque);
 void migration_make_urgent_request(void);
 void migration_consume_urgent_request(void);
 bool migration_rate_limit(void);
-void migration_cancel(const Error *error);
+void migration_cancel(Error *error);
 
 void populate_vfio_info(MigrationInfo *info);
 void postcopy_temp_page_reset(PostcopyTmpPage *tmp_page);
migration/channel.c +0 -1
index ca3319a309..48b3f6abd6 100644
@@ -90,7 +90,6 @@ void migration_channel_connect(MigrationState *s,
         }
     }
     migrate_fd_connect(s, error);
-    error_free(error);
 }
 
 
migration/migration.c +14 -6
index 203d40d4c9..13dccc4c12 100644
@@ -160,7 +160,7 @@ void migration_object_init(void)
     dirty_bitmap_mig_init();
 }
 
-void migration_cancel(const Error *error)
+void migration_cancel(Error *error)
 {
     if (error) {
         migrate_set_error(current_migration, error);
@@ -1197,11 +1197,20 @@ static void migrate_fd_cleanup_bh(void *opaque)
     object_unref(OBJECT(s));
 }
 
-void migrate_set_error(MigrationState *s, const Error *error)
+/*
+ * Set error for current migration state.  The `error' ownership will be
+ * moved from the caller to MigrationState, so the caller doesn't need to
+ * free the error.
+ *
+ * If the caller still needs to reference the `error' passed in, one should
+ * use error_copy() explicitly.
+ */
+void migrate_set_error(MigrationState *s, Error *error)
 {
     QEMU_LOCK_GUARD(&s->error_mutex);
     if (!s->error) {
-        s->error = error_copy(error);
+        /* Record the first error triggered */
+        s->error = error;
     }
 }
 
@@ -1214,7 +1223,7 @@ static void migrate_error_free(MigrationState *s)
     }
 }
 
-void migrate_fd_error(MigrationState *s, const Error *error)
+void migrate_fd_error(MigrationState *s, Error *error)
 {
     trace_migrate_fd_error(error_get_pretty(error));
     assert(s->to_dst_file == NULL);
@@ -1678,7 +1687,7 @@ void qmp_migrate(const char *uri, bool has_blk, bool blk,
         if (!(has_resume && resume)) {
             yank_unregister_instance(MIGRATION_YANK_INSTANCE);
         }
-        migrate_fd_error(s, local_err);
+        migrate_fd_error(s, error_copy(local_err));
         error_propagate(errp, local_err);
         return;
     }
@@ -2595,7 +2604,6 @@ static MigThrError migration_detect_error(MigrationState *s)
 
     if (local_error) {
         migrate_set_error(s, local_error);
-        error_free(local_error);
     }
 
     if (state == MIGRATION_STATUS_POSTCOPY_ACTIVE && ret) {
migration/multifd.c +4 -6
index 3387d8277f..62bc2dbf49 100644
@@ -551,7 +551,6 @@ void multifd_save_cleanup(void)
         multifd_send_state->ops->send_cleanup(p, &local_err);
         if (local_err) {
             migrate_set_error(migrate_get_current(), local_err);
-            error_free(local_err);
         }
     }
     qemu_sem_destroy(&multifd_send_state->channels_ready);
@@ -750,7 +749,6 @@ out:
     if (local_err) {
         trace_multifd_send_error(p->id);
         multifd_send_terminate_threads(local_err);
-        error_free(local_err);
     }
 
     /*
@@ -883,7 +881,6 @@ static void multifd_new_send_channel_cleanup(MultiFDSendParams *p,
       */
      p->quit = true;
      object_unref(OBJECT(ioc));
-     error_free(err);
 }
 
 static void multifd_new_send_channel_async(QIOTask *task, gpointer opaque)
@@ -1148,7 +1145,6 @@ static void *multifd_recv_thread(void *opaque)
 
     if (local_err) {
         multifd_recv_terminate_threads(local_err);
-        error_free(local_err);
     }
     qemu_mutex_lock(&p->mutex);
     p->running = false;
@@ -1240,7 +1236,8 @@ void multifd_recv_new_channel(QIOChannel *ioc, Error **errp)
 
     id = multifd_recv_initial_packet(ioc, &local_err);
     if (id < 0) {
-        multifd_recv_terminate_threads(local_err);
+        /* Copy local error because we'll also return it to caller */
+        multifd_recv_terminate_threads(error_copy(local_err));
         error_propagate_prepend(errp, local_err,
                                 "failed to receive packet"
                                 " via multifd channel %d: ",
@@ -1253,7 +1250,8 @@ void multifd_recv_new_channel(QIOChannel *ioc, Error **errp)
     if (p->c != NULL) {
         error_setg(&local_err, "multifd: received id '%d' already setup'",
                    id);
-        multifd_recv_terminate_threads(local_err);
+        /* Copy local error because we'll also return it to caller */
+        multifd_recv_terminate_threads(error_copy(local_err));
         error_propagate(errp, local_err);
         return;
     }
migration/postcopy-ram.c +0 -1
index 5615ec29eb..6f6fb52bf1 100644
@@ -1594,7 +1594,6 @@ postcopy_preempt_send_channel_done(MigrationState *s,
 {
     if (local_err) {
         migrate_set_error(s, local_err);
-        error_free(local_err);
     } else {
         migration_ioc_register_yank(ioc);
         s->postcopy_qemufile_src = qemu_file_new_output(ioc);
migration/ram.c +0 -1
index 5283a75f02..ba4890563d 100644
@@ -4267,7 +4267,6 @@ static void ram_mig_ram_block_resized(RAMBlockNotifier *n, void *host,
          */
         error_setg(&err, "RAM block '%s' resized during precopy.", rb->idstr);
         migration_cancel(err);
-        error_free(err);
     }
 
     switch (ps) {
Signature · 1 line
2.41.0
author
#2 lore raw

PATCH 2/7 migration: Introduce migrate_has_error()

In reply to Peter Xu

Introduce a helper to detect whether MigrationState.error is set for whatever reason. It is intended to not taking the error_mutex here because neither do we reference the pointer, nor do we modify the pointer. State why it's safe to do so.

This is preparation work for any thread (e.g. source return path thread) to setup errors in an unified way to MigrationState, rather than relying on its own way to set errors (mark_source_rp_bad()).

---
 migration/migration.h |  1 +
 migration/migration.c | 15 +++++++++++++++
 2 files changed, 16 insertions(+)
migration/migration.h +1 -0
index 32f87e3834..e48317ca97 100644
@@ -469,6 +469,7 @@ bool  migration_has_all_channels(void);
 uint64_t migrate_max_downtime(void);
 
 void migrate_set_error(MigrationState *s, Error *error);
+bool migrate_has_error(MigrationState *s);
 void migrate_fd_error(MigrationState *s, Error *error);
 
 void migrate_fd_connect(MigrationState *s, Error *error_in);
migration/migration.c +15 -0
index 13dccc4c12..39674a3981 100644
@@ -1214,6 +1214,21 @@ void migrate_set_error(MigrationState *s, Error *error)
     }
 }
 
+/*
+ * Whether the migration state has error set?
+ *
+ * Note this function explicitly didn't use error_mutex, because it only
+ * reads the error pointer for a boolean status.
+ *
+ * As long as the Error* is set, it shouldn't be freed before migration
+ * cleanup, so any thread can use this helper to safely detect whether
+ * there's anything wrong happened already.
+ */
+bool migrate_has_error(MigrationState *s)
+{
+    return qatomic_read(&s->error);
+}
+
 static void migrate_error_free(MigrationState *s)
 {
     QEMU_LOCK_GUARD(&s->error_mutex);
Signature · 1 line
2.41.0
author
#3 lore raw

PATCH 3/7 migration: Refactor error handling in source return path

In reply to Peter Xu

rp_state.error was a boolean used to show error happened in return path thread. That's not only duplicating error reporting (migrate_set_error), but also not good enough in that we only do error_report() and set it to true, we never can keep a history of the exact error and show it in query-migrate.

To make this better, a few things done:

- Use error_setg() rather than error_report() across the whole lifecycle of return path thread, keeping the error in an Error*.

- Use migrate_set_error() to apply that captured error to the global migration object when error occured in this thread.

- With above, no need to have mark_source_rp_bad(), remove it, alongside with rp_state.error itself.

---
 migration/migration.h  |   1 -
 migration/ram.h        |   5 +-
 migration/migration.c  | 118 ++++++++++++++++++++---------------------
 migration/ram.c        |  41 +++++++-------
 migration/trace-events |   2 +-
 5 files changed, 82 insertions(+), 85 deletions(-)
migration/migration.h +0 -1
index e48317ca97..d9c4b753b5 100644
@@ -290,7 +290,6 @@ struct MigrationState {
         /* Protected by qemu_file_lock */
         QEMUFile     *from_dst_file;
         QemuThread    rp_thread;
-        bool          error;
         /*
          * We can also check non-zero of rp_thread, but there's no "official"
          * way to do this, so this bool makes it slightly more elegant.
migration/ram.h +3 -2
index ea1f3c25b5..02a4af2db6 100644
@@ -52,7 +52,8 @@ uint64_t ram_bytes_total(void);
 void mig_throttle_counter_reset(void);
 
 uint64_t ram_pagesize_summary(void);
-int ram_save_queue_pages(const char *rbname, ram_addr_t start, ram_addr_t len);
+int ram_save_queue_pages(const char *rbname, ram_addr_t start, ram_addr_t len,
+                         Error **errp);
 void ram_postcopy_migrated_memory_release(MigrationState *ms);
 /* For outgoing discard bitmap */
 void ram_postcopy_send_discard_bitmap(MigrationState *ms);
@@ -72,7 +73,7 @@ void ramblock_recv_bitmap_set(RAMBlock *rb, void *host_addr);
 void ramblock_recv_bitmap_set_range(RAMBlock *rb, void *host_addr, size_t nr);
 int64_t ramblock_recv_bitmap_send(QEMUFile *file,
                                   const char *block_name);
-int ram_dirty_bitmap_reload(MigrationState *s, RAMBlock *rb);
+int ram_dirty_bitmap_reload(MigrationState *s, RAMBlock *rb, Error **errp);
 bool ramblock_page_is_discarded(RAMBlock *rb, ram_addr_t start);
 void postcopy_preempt_shutdown_file(MigrationState *s);
 void *postcopy_preempt_thread(void *opaque);
migration/migration.c +57 -61

275 lines not on this page · raw message

migration/ram.c +21 -20

100 lines not on this page · raw message

migration/trace-events +1 -1
index cdaef7a1ea..98aa46c780 100644
@@ -161,7 +161,7 @@ migration_completion_postcopy_end_after_complete(void) ""
 migration_rate_limit_pre(int ms) "%d ms"
 migration_rate_limit_post(int urgent) "urgent: %d"
 migration_return_path_end_before(void) ""
-migration_return_path_end_after(int rp_error) "%d"
+migration_return_path_end_after(void) ""
 migration_thread_after_loop(void) ""
 migration_thread_file_err(void) ""
 migration_thread_setup_complete(void) ""
Signature · 1 line
2.41.0
Reviewed-by on v1 3/7
#4 lore raw

Re: [PATCH 3/7] migration: Refactor error handling in source return path

Quoted 11 lines · Peter Xu

rp_state.error was a boolean used to show error happened in return path thread. That's not only duplicating error reporting (migrate_set_error), but also not good enough in that we only do error_report() and set it to true, we never can keep a history of the exact error and show it in query-migrate.

To make this better, a few things done:

- Use error_setg() rather than error_report() across the whole lifecycle of return path thread, keeping the error in an Error*.

Quoted 7 lines

- Use migrate_set_error() to apply that captured error to the global migration object when error occured in this thread.

- With above, no need to have mark_source_rp_bad(), remove it, alongside with rp_state.error itself.

author
#5 lore raw

PATCH 4/7 migration: Deliver return path file error to migrate state too

In reply to Peter Xu

We've already did this for most of the return path thread errors, but not yet for the IO errors happened on the return path qemufile. Do that too.

Remember to reset "err" always, because the ownership is not us anymore, otherwise we're prone to use-after-free later after recovered.

---
 migration/migration.c | 7 +++++++
 1 file changed, 7 insertions(+)
migration/migration.c +7 -0
index f8c41c4d98..234dd3601d 100644
@@ -1992,6 +1992,13 @@ out:
 
     res = qemu_file_get_error(rp);
     if (res) {
+        /* We have forwarded any error in "err" already, reuse "error" */
+        assert(err == NULL);
+        /* Try to deliver this file error to migration state */
+        qemu_file_get_error_obj(rp, &err);
+        migrate_set_error(ms, err);
+        err = NULL;
+
         if (res && migration_in_postcopy()) {
             /*
              * Maybe there is something we can do: it looks like a
Signature · 1 line
2.41.0
Reviewed-by on v1 4/7
#6 lore raw

Re: [PATCH 4/7] migration: Deliver return path file error to migrate state too

Quoted 7 lines · Peter Xu

We've already did this for most of the return path thread errors, but not yet for the IO errors happened on the return path qemufile. Do that too.

Remember to reset "err" always, because the ownership is not us anymore, otherwise we're prone to use-after-free later after recovered.

author
#7 lore raw

PATCH 5/7 migration: Display error in query-migrate irrelevant of status

In reply to Peter Xu

Display it as long as being set, irrelevant of FAILED status. E.g., it may also be applicable to PAUSED stage of postcopy, to provide hint on what has gone wrong.

The error_mutex seems to be overlooked when referencing the error, add it to be very safe.

---
 migration/migration.c | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)
migration/migration.c +5 -3
index 234dd3601d..7455353918 100644
@@ -1033,9 +1033,6 @@ static void fill_source_migration_info(MigrationInfo *info)
         break;
     case MIGRATION_STATUS_FAILED:
         info->has_status = true;
-        if (s->error) {
-            info->error_desc = g_strdup(error_get_pretty(s->error));
-        }
         break;
     case MIGRATION_STATUS_CANCELLED:
         info->has_status = true;
@@ -1045,6 +1042,11 @@ static void fill_source_migration_info(MigrationInfo *info)
         break;
     }
     info->status = state;
+
+    QEMU_LOCK_GUARD(&s->error_mutex);
+    if (s->error) {
+        info->error_desc = g_strdup(error_get_pretty(s->error));
+    }
 }
 
 static void fill_destination_migration_info(MigrationInfo *info)
Signature · 1 line
2.41.0
Reviewed-by on v1 5/7
#8 lore raw

Re: [PATCH 5/7] migration: Display error in query-migrate irrelevant of status

Quoted 3 lines · Peter Xu

Display it as long as being set, irrelevant of FAILED status. E.g., it may also be applicable to PAUSED stage of postcopy, to provide hint on what has gone wrong.

This might have made the documentation slightly inaccurate:

# @error-desc: the human readable error description string, when
# @status is 'failed'. Clients should not attempt to parse the
# error strings. (Since 2.7)

But it's not wrong, so:

author
#9 lore raw

Re: [PATCH 5/7] migration: Display error in query-migrate irrelevant of status

Quoted 7 lines · Fabiano Rosas
Quoted 3 lines · Peter Xu

Display it as long as being set, irrelevant of FAILED status. E.g., it may also be applicable to PAUSED stage of postcopy, to provide hint on what has gone wrong.

This might have made the documentation slightly inaccurate:

Hmm yes, maybe I should touch that up so as to include "postcopy-paused", or just remove the statement that it must be in a "failed" stage.

Quoted 8 lines

# @error-desc: the human readable error description string, when
# @status is 'failed'. Clients should not attempt to parse the
# error strings. (Since 2.7)

But it's not wrong, so:

Thanks for taking a look.

Signature · 1 line
Peter Xu
author
#10 lore raw

PATCH 6/7 qemufile: Always return a verbose error

In reply to Peter Xu

There're a lot of cases where we only have an errno set in last_error but without a detailed error description. When this happens, try to generate an error contains the errno as a descriptive error.

This will be helpful in cases where one relies on the Error*. E.g., migration state only caches Error* in MigrationState.error. With this, we'll display correct error messages in e.g. query-migrate when the error was only set by qemu_file_set_error().

---
 migration/qemu-file.c | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)
migration/qemu-file.c +12 -3
index acc282654a..419b4092e7 100644
@@ -156,15 +156,24 @@ void qemu_file_set_hooks(QEMUFile *f, const QEMUFileHooks *hooks)
  *
  * Return negative error value if there has been an error on previous
  * operations, return 0 if no error happened.
- * Optional, it returns Error* in errp, but it may be NULL even if return value
- * is not 0.
  *
+ * If errp is specified, a verbose error message will be copied over.
  */
 int qemu_file_get_error_obj(QEMUFile *f, Error **errp)
 {
+    if (!f->last_error) {
+        return 0;
+    }
+
+    /* There is an error */
     if (errp) {
-        *errp = f->last_error_obj ? error_copy(f->last_error_obj) : NULL;
+        if (f->last_error_obj) {
+            *errp = error_copy(f->last_error_obj);
+        } else {
+            error_setg_errno(errp, -f->last_error, "Channel error");
+        }
     }
+
     return f->last_error;
 }
 
Signature · 1 line
2.41.0
author
#11 lore raw

PATCH 7/7 migration: Provide explicit error message for file shutdowns

In reply to Peter Xu

Provide an explicit reason for qemu_file_shutdown()s, which can be displayed in query-migrate when used.

This will make e.g. migrate-pause to display explicit error descriptions, from:

"error-desc": "Channel error: Input/output error"

To:

"error-desc": "Channel is explicitly shutdown by the user"

in query-migrate.

---
 migration/qemu-file.c | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)
migration/qemu-file.c +4 -1
index 419b4092e7..ff605027de 100644
@@ -87,7 +87,10 @@ int qemu_file_shutdown(QEMUFile *f)
      *      --> guest crash!
      */
     if (!f->last_error) {
-        qemu_file_set_error(f, -EIO);
+        Error *err = NULL;
+
+        error_setg(&err, "Channel is explicitly shutdown by the user");
+        qemu_file_set_error_obj(f, -EIO, err);
     }
 
     if (!qio_channel_has_feature(f->ioc,
Signature · 1 line
2.41.0