Skip to contentSkip to conversation

tcg/plugins: implement a qemu_plugin_user_exit helper

RFC PATCH In review Alex Bennée 3 participants 6 messages Jul 19, 2021 – Jul 20, 2021

lore
Share & tools
lore
#0 lore raw

RFC PATCH 1/1 tcg/plugins: implement a qemu_plugin_user_exit helper

In user-mode emulation there is a small race between preexit_cleanup and exit_group() which means we may end up calling instrumented instructions before the kernel reaps child threads. To solve this we implement a new helper which ensures the callbacks are flushed along with any translations before we let the host do it's a thing.

While we are at it make the documentation of qemu_plugin_register_atexit_cb clearer as to what the user can expect.

---
 include/qemu/plugin.h      | 12 ++++++++++++
 include/qemu/qemu-plugin.h | 13 +++++++++++++
 bsd-user/syscall.c         |  6 +++---
 linux-user/exit.c          |  2 +-
 plugins/core.c             | 33 +++++++++++++++++++++++++++++++++
 5 files changed, 62 insertions(+), 4 deletions(-)
include/qemu/plugin.h +12 -0
index 0fefbc6084..9a8438f683 100644
@@ -190,6 +190,16 @@ void qemu_plugin_add_dyn_cb_arr(GArray *arr);
 
 void qemu_plugin_disable_mem_helpers(CPUState *cpu);
 
+/**
+ * qemu_plugin_user_exit(): clean-up callbacks before calling exit callbacks
+ *
+ * This is a user-mode only helper that ensure we have fully cleared
+ * callbacks from all threads before calling the exit callbacks. This
+ * is so the plugins themselves don't have to jump through hoops to
+ * guard against race conditions.
+ */
+void qemu_plugin_user_exit(void);
+
 #else /* !CONFIG_PLUGIN */
 
 static inline void qemu_plugin_add_opts(void)
@@ -250,6 +260,8 @@ void qemu_plugin_add_dyn_cb_arr(GArray *arr)
 static inline void qemu_plugin_disable_mem_helpers(CPUState *cpu)
 { }
 
+static inline void qemu_plugin_user_exit(void)
+{ }
 #endif /* !CONFIG_PLUGIN */
 
 #endif /* QEMU_PLUGIN_H */
include/qemu/qemu-plugin.h +13 -0
index dc3496f36c..e6e815abc5 100644
@@ -549,6 +549,19 @@ void qemu_plugin_vcpu_for_each(qemu_plugin_id_t id,
 void qemu_plugin_register_flush_cb(qemu_plugin_id_t id,
                                    qemu_plugin_simple_cb_t cb);
 
+/**
+ * qemu_plugin_register_atexit_cb() - register exit callback
+ * @id: plugin ID
+ * @cb: callback
+ * @userdata: user data for callback
+ *
+ * The @cb function is called once execution has finished. Plugins
+ * should be able to free all their resources at this point much like
+ * after a reset/uninstall callback is called.
+ *
+ * In user-mode it is possible a few un-instrumented instructions from
+ * child threads may run before the host kernel reaps the threads.
+ */
 void qemu_plugin_register_atexit_cb(qemu_plugin_id_t id,
                                     qemu_plugin_udata_cb_t cb, void *userdata);
 
bsd-user/syscall.c +3 -3
index 7d986e9700..3f44311396 100644
@@ -335,7 +335,7 @@ abi_long do_freebsd_syscall(void *cpu_env, int num, abi_long arg1,
         _mcleanup();
 #endif
         gdb_exit(arg1);
-        qemu_plugin_atexit_cb();
+        qemu_plugin_user_exit();
         /* XXX: should free thread stack and CPU env */
         _exit(arg1);
         ret = 0; /* avoid warning */
@@ -437,7 +437,7 @@ abi_long do_netbsd_syscall(void *cpu_env, int num, abi_long arg1,
         _mcleanup();
 #endif
         gdb_exit(arg1);
-        qemu_plugin_atexit_cb();
+        qemu_plugin_user_exit();
         /* XXX: should free thread stack and CPU env */
         _exit(arg1);
         ret = 0; /* avoid warning */
@@ -516,7 +516,7 @@ abi_long do_openbsd_syscall(void *cpu_env, int num, abi_long arg1,
         _mcleanup();
 #endif
         gdb_exit(arg1);
-        qemu_plugin_atexit_cb();
+        qemu_plugin_user_exit();
         /* XXX: should free thread stack and CPU env */
         _exit(arg1);
         ret = 0; /* avoid warning */
linux-user/exit.c +1 -1
index 70b344048c..527e29cbc1 100644
@@ -35,5 +35,5 @@ void preexit_cleanup(CPUArchState *env, int code)
         __gcov_dump();
 #endif
         gdb_exit(code);
-        qemu_plugin_atexit_cb();
+        qemu_plugin_user_exit();
 }
plugins/core.c +33 -0
index e1bcdb570d..c573b81a96 100644
@@ -487,6 +487,39 @@ void qemu_plugin_register_atexit_cb(qemu_plugin_id_t id,
     plugin_register_cb_udata(id, QEMU_PLUGIN_EV_ATEXIT, cb, udata);
 }
 
+/*
+ * Handle exit from linux-user. Unlike the normal atexit() mechanism
+ * we need to handle the clean-up manually as it's possible threads
+ * are still running. We need to remove all callbacks from code
+ * generation, flush the current translations and then we can safely
+ * trigger the exit callbacks.
+ */
+
+void qemu_plugin_user_exit(void)
+{
+    enum qemu_plugin_event ev;
+
+    QEMU_LOCK_GUARD(&plugin.lock);
+
+    start_exclusive();
+
+    /* un-register all callbacks except the final AT_EXIT one */
+    for (ev = 0; ev < QEMU_PLUGIN_EV_MAX; ev++) {
+        if (ev != QEMU_PLUGIN_EV_ATEXIT) {
+            struct qemu_plugin_ctx *ctx;
+            QTAILQ_FOREACH(ctx, &plugin.ctxs, entry) {
+                plugin_unregister_cb__locked(ctx, ev);
+            }
+        }
+    }
+
+    tb_flush(current_cpu);
+    end_exclusive();
+
+    /* now it's safe to handle the exit case */
+    qemu_plugin_atexit_cb();
+}
+
 /*
  * Call this function after longjmp'ing to the main loop. It's possible that the
  * last instruction of a TB might have used helpers, and therefore the
Signature · 1 line
2.32.0.264.g75ae10bc75
#1 lore raw
Quoted 9 lines · Alex Bennée

In user-mode emulation there is a small race between preexit_cleanup and exit_group() which means we may end up calling instrumented instructions before the kernel reaps child threads. To solve this we implement a new helper which ensures the callbacks are flushed along with any translations before we let the host do it's a thing.

While we are at it make the documentation of qemu_plugin_register_atexit_cb clearer as to what the user can expect.

<snip>

Quoted 18 lines
plugins/core.c +17 -0
+/*
+ * Handle exit from linux-user. Unlike the normal atexit() mechanism
+ * we need to handle the clean-up manually as it's possible threads
+ * are still running. We need to remove all callbacks from code
+ * generation, flush the current translations and then we can safely
+ * trigger the exit callbacks.
+ */
+
+void qemu_plugin_user_exit(void)
+{
+    enum qemu_plugin_event ev;
+
+    QEMU_LOCK_GUARD(&plugin.lock);
+
+    start_exclusive();
+
+    /* un-register all callbacks except the final AT_EXIT one */

plugins/core.c void qemu_plugin_register_atexit_cb(qemu_plugin_id_t id, · line 516

Quoted 10 lines
plugins/core.c +10 -0
+    for (ev = 0; ev < QEMU_PLUGIN_EV_MAX; ev++) {
+        if (ev != QEMU_PLUGIN_EV_ATEXIT) {
+            struct qemu_plugin_ctx *ctx;
+            QTAILQ_FOREACH(ctx, &plugin.ctxs, entry) {
+                plugin_unregister_cb__locked(ctx, ev);
+            }
+        }
+    }
+
+    tb_flush(current_cpu);

We also need to disable memory helpers during the exclusive period as that is another route into a callback:

--8<---------------cut here---------------start------------->8---
modified plugins/core.c
@@ -498,6 +499,7 @@ void qemu_plugin_register_atexit_cb(qemu_plugin_id_t id,
void qemu_plugin_user_exit(void)

 {
     enum qemu_plugin_event ev;

+ CPUState *cpu;

     QEMU_LOCK_GUARD(&plugin.lock);

@@ -514,6 +516,11 @@ void qemu_plugin_user_exit(void)

     }
 
     tb_flush(current_cpu);

+
+ CPU_FOREACH(cpu) {
+ qemu_plugin_disable_mem_helpers(cpu);
+ }
+

     end_exclusive();
 
     /* now it's safe to handle the exit case */

--8<---------------cut here---------------end--------------->8---

Quoted 9 lines
plugins/core.c +6 -0
+    end_exclusive();
+
+    /* now it's safe to handle the exit case */
+    qemu_plugin_atexit_cb();
+}
+
 /*
  * Call this function after longjmp'ing to the main loop. It's possible that the
  * last instruction of a TB might have used helpers, and therefore the
Signature · 1 line
Alex Bennée
#2 lore raw
Quoted 59 lines · Alex Bennée
Quoted 9 lines · Alex Bennée

In user-mode emulation there is a small race between preexit_cleanup and exit_group() which means we may end up calling instrumented instructions before the kernel reaps child threads. To solve this we implement a new helper which ensures the callbacks are flushed along with any translations before we let the host do it's a thing.

While we are at it make the documentation of qemu_plugin_register_atexit_cb clearer as to what the user can expect.

<snip>

Quoted 18 lines
plugins/core.c +17 -0
+/*
+ * Handle exit from linux-user. Unlike the normal atexit() mechanism
+ * we need to handle the clean-up manually as it's possible threads
+ * are still running. We need to remove all callbacks from code
+ * generation, flush the current translations and then we can safely
+ * trigger the exit callbacks.
+ */
+
+void qemu_plugin_user_exit(void)
+{
+    enum qemu_plugin_event ev;
+
+    QEMU_LOCK_GUARD(&plugin.lock);
+
+    start_exclusive();
+
+    /* un-register all callbacks except the final AT_EXIT one */

plugins/core.c void qemu_plugin_register_atexit_cb(qemu_plugin_id_t id, · line 516

Quoted 10 lines
plugins/core.c +10 -0
+    for (ev = 0; ev < QEMU_PLUGIN_EV_MAX; ev++) {
+        if (ev != QEMU_PLUGIN_EV_ATEXIT) {
+            struct qemu_plugin_ctx *ctx;
+            QTAILQ_FOREACH(ctx, &plugin.ctxs, entry) {
+                plugin_unregister_cb__locked(ctx, ev);
+            }
+        }
+    }
+
+    tb_flush(current_cpu);

We also need to disable memory helpers during the exclusive period as that is another route into a callback:
--8<---------------cut here---------------start------------->8---
modified plugins/core.c

(quoted diff) +1 -0
@@ -498,6 +499,7 @@ void qemu_plugin_register_atexit_cb(qemu_plugin_id_t
 id,
 void qemu_plugin_user_exit(void)
 {
     enum qemu_plugin_event ev;
+    CPUState *cpu;
 
     QEMU_LOCK_GUARD(&plugin.lock);
@@ -514,6 +516,11 @@ void qemu_plugin_user_exit(void)
     }
 

Review void qemu_plugin_user_exit(void)

Quoted 10 lines
(quoted diff) +5 -1
     tb_flush(current_cpu);
+
+    CPU_FOREACH(cpu) {
+        qemu_plugin_disable_mem_helpers(cpu);
+    }
+
     end_exclusive();
 
     /* now it's safe to handle the exit case */
--8<---------------cut here---------------end--------------->8---

I think both of these are find from a bsd-user point of view.

Warner

Quoted 15 lines

plugins/core.c void qemu_plugin_register_atexit_cb(qemu_plugin_id_t id,

Quoted 8 lines
plugins/core.c +6 -0
+    end_exclusive();
+
+    /* now it's safe to handle the exit case */
+    qemu_plugin_atexit_cb();
+}
+
 /*
  * Call this function after longjmp'ing to the main loop. It's possible

that the

Quoted 1 line

* last instruction of a TB might have used helpers, and therefore the

--
Alex Bennée

author Acked-by or Reviewed-by:?
#3 lore raw
Quoted 67 lines · Warner Losh

On Mon, Jul 19, 2021, 7:57 AM Alex Bennée <[email protected]> wrote:

Quoted 9 lines · Alex Bennée

In user-mode emulation there is a small race between preexit_cleanup and exit_group() which means we may end up calling instrumented instructions before the kernel reaps child threads. To solve this we implement a new helper which ensures the callbacks are flushed along with any translations before we let the host do it's a thing.

While we are at it make the documentation of qemu_plugin_register_atexit_cb clearer as to what the user can expect.

<snip>

Quoted 18 lines
plugins/core.c +17 -0
+/*
+ * Handle exit from linux-user. Unlike the normal atexit() mechanism
+ * we need to handle the clean-up manually as it's possible threads
+ * are still running. We need to remove all callbacks from code
+ * generation, flush the current translations and then we can safely
+ * trigger the exit callbacks.
+ */
+
+void qemu_plugin_user_exit(void)
+{
+    enum qemu_plugin_event ev;
+
+    QEMU_LOCK_GUARD(&plugin.lock);
+
+    start_exclusive();
+
+    /* un-register all callbacks except the final AT_EXIT one */

plugins/core.c void qemu_plugin_register_atexit_cb(qemu_plugin_id_t id, · line 516

Quoted 10 lines
plugins/core.c +10 -0
+    for (ev = 0; ev < QEMU_PLUGIN_EV_MAX; ev++) {
+        if (ev != QEMU_PLUGIN_EV_ATEXIT) {
+            struct qemu_plugin_ctx *ctx;
+            QTAILQ_FOREACH(ctx, &plugin.ctxs, entry) {
+                plugin_unregister_cb__locked(ctx, ev);
+            }
+        }
+    }
+
+    tb_flush(current_cpu);

We also need to disable memory helpers during the exclusive period as that is another route into a callback:
--8<---------------cut here---------------start------------->8---
modified plugins/core.c
@@ -498,6 +499,7 @@ void qemu_plugin_register_atexit_cb(qemu_plugin_id_t id,
void qemu_plugin_user_exit(void)

  {
      enum qemu_plugin_event ev;

+ CPUState *cpu;

      QEMU_LOCK_GUARD(&plugin.lock);

@@ -514,6 +516,11 @@ void qemu_plugin_user_exit(void)

      }

      tb_flush(current_cpu);
 +
 +    CPU_FOREACH(cpu) {
 +        qemu_plugin_disable_mem_helpers(cpu);
 +    }
 +
      end_exclusive();

      /* now it's safe to handle the exit case */
Quoted 3 lines

--8<---------------cut here---------------end--------------->8---

I think both of these are find from a bsd-user point of view.

Acked-by or Reviewed-by:?
Signature · 1 line
Alex Bennée
Acked-by
#4 lore raw
Quoted 71 lines · Alex Bennée
Quoted 1 line · Warner Losh

On Mon, Jul 19, 2021, 7:57 AM Alex Bennée <[email protected]>

Quoted 39 lines · wrote:
Quoted 9 lines · Alex Bennée

In user-mode emulation there is a small race between preexit_cleanup and exit_group() which means we may end up calling instrumented instructions before the kernel reaps child threads. To solve this we implement a new helper which ensures the callbacks are flushed along with any translations before we let the host do it's a thing.

While we are at it make the documentation of qemu_plugin_register_atexit_cb clearer as to what the user can expect.

<snip>

Quoted 26 lines
plugins/core.c +25 -0
+/*
+ * Handle exit from linux-user. Unlike the normal atexit() mechanism
+ * we need to handle the clean-up manually as it's possible threads
+ * are still running. We need to remove all callbacks from code
+ * generation, flush the current translations and then we can safely
+ * trigger the exit callbacks.
+ */
+
+void qemu_plugin_user_exit(void)
+{
+    enum qemu_plugin_event ev;
+
+    QEMU_LOCK_GUARD(&plugin.lock);
+
+    start_exclusive();
+
+    /* un-register all callbacks except the final AT_EXIT one */
+    for (ev = 0; ev < QEMU_PLUGIN_EV_MAX; ev++) {
+        if (ev != QEMU_PLUGIN_EV_ATEXIT) {
+            struct qemu_plugin_ctx *ctx;
+            QTAILQ_FOREACH(ctx, &plugin.ctxs, entry) {
+                plugin_unregister_cb__locked(ctx, ev);
+            }
+        }
+    }
Quoted 8 lines

Review

Quoted 2 lines
(quoted diff) +2 -0
+
+    tb_flush(current_cpu);

We also need to disable memory helpers during the exclusive period as that is another route into a callback:
--8<---------------cut here---------------start------------->8---
modified plugins/core.c
@@ -498,6 +499,7 @@ void

qemu_plugin_register_atexit_cb(qemu_plugin_id_t id,

Quoted 18 lines

void qemu_plugin_user_exit(void)

  {
      enum qemu_plugin_event ev;

+ CPUState *cpu;

      QEMU_LOCK_GUARD(&plugin.lock);

@@ -514,6 +516,11 @@ void qemu_plugin_user_exit(void)

      }

      tb_flush(current_cpu);
 +
 +    CPU_FOREACH(cpu) {
 +        qemu_plugin_disable_mem_helpers(cpu);
 +    }
 +
      end_exclusive();
Quoted 7 lines
Quoted 4 lines
      /* now it's safe to handle the exit case */

--8<---------------cut here---------------end--------------->8---

I think both of these are find from a bsd-user point of view.

Acked-by or Reviewed-by:?

Sorry I wasn't clear.

Quoted 3 lines

--
Alex Bennée

Reviewed-by on v1 1/1
#5 lore raw
In reply to Alex Bennée
Quoted 11 lines · On Mon, Jul 19, 2021, 14:37 Alex Bennée <[email protected]> wrote:

In user-mode emulation there is a small race between preexit_cleanup and exit_group() which means we may end up calling instrumented instructions before the kernel reaps child threads. To solve this we implement a new helper which ensures the callbacks are flushed along with any translations before we let the host do it's a thing.

While we are at it make the documentation of qemu_plugin_register_atexit_cb clearer as to what the user can expect.

Looks great to me and I cannot reproduce the race with it installed in.

With calling qemu_plugin_disable_mem_helpers:

---

Quoted 158 lines
 include/qemu/plugin.h      | 12 ++++++++++++
 include/qemu/qemu-plugin.h | 13 +++++++++++++
 bsd-user/syscall.c         |  6 +++---
 linux-user/exit.c          |  2 +-
 plugins/core.c             | 33 +++++++++++++++++++++++++++++++++
 5 files changed, 62 insertions(+), 4 deletions(-)
include/qemu/plugin.h +2 -0
index 0fefbc6084..9a8438f683 100644
@@ -190,6 +190,16 @@ void qemu_plugin_add_dyn_cb_arr(GArray *arr);
 
 void qemu_plugin_disable_mem_helpers(CPUState *cpu);
 
+/**
+ * qemu_plugin_user_exit(): clean-up callbacks before calling exit

callbacks
+ *
+ * This is a user-mode only helper that ensure we have fully cleared
+ * callbacks from all threads before calling the exit callbacks. This
+ * is so the plugins themselves don't have to jump through hoops to
+ * guard against race conditions.
+ */

include/qemu/plugin.h +4 -0
+void qemu_plugin_user_exit(void);
+
 #else /* !CONFIG_PLUGIN */
 
 static inline void qemu_plugin_add_opts(void)
@@ -250,6 +260,8 @@ void qemu_plugin_add_dyn_cb_arr(GArray *arr)
 static inline void qemu_plugin_disable_mem_helpers(CPUState *cpu)
 { }
 
+static inline void qemu_plugin_user_exit(void)
+{ }
 #endif /* !CONFIG_PLUGIN */
 
 #endif /* QEMU_PLUGIN_H */
include/qemu/qemu-plugin.h +13 -0
index dc3496f36c..e6e815abc5 100644
@@ -549,6 +549,19 @@ void qemu_plugin_vcpu_for_each(qemu_plugin_id_t id,
 void qemu_plugin_register_flush_cb(qemu_plugin_id_t id,
                                    qemu_plugin_simple_cb_t cb);
 
+/**
+ * qemu_plugin_register_atexit_cb() - register exit callback
+ * @id: plugin ID
+ * @cb: callback
+ * @userdata: user data for callback
+ *
+ * The @cb function is called once execution has finished. Plugins
+ * should be able to free all their resources at this point much like
+ * after a reset/uninstall callback is called.
+ *
+ * In user-mode it is possible a few un-instrumented instructions from
+ * child threads may run before the host kernel reaps the threads.
+ */
 void qemu_plugin_register_atexit_cb(qemu_plugin_id_t id,
                                     qemu_plugin_udata_cb_t cb, void
 *userdata);
bsd-user/syscall.c +1 -1
index 7d986e9700..3f44311396 100644
@@ -335,7 +335,7 @@ abi_long do_freebsd_syscall(void *cpu_env, int num,
 abi_long arg1,
         _mcleanup();
 #endif
         gdb_exit(arg1);
-        qemu_plugin_atexit_cb();
+        qemu_plugin_user_exit();
         /* XXX: should free thread stack and CPU env */
         _exit(arg1);

ret = 0; /* avoid warning */

bsd-user/syscall.c +1 -1
@@ -437,7 +437,7 @@ abi_long do_netbsd_syscall(void *cpu_env, int num,
 abi_long arg1,
         _mcleanup();
 #endif
         gdb_exit(arg1);
-        qemu_plugin_atexit_cb();
+        qemu_plugin_user_exit();
         /* XXX: should free thread stack and CPU env */
         _exit(arg1);

ret = 0; /* avoid warning */

bsd-user/syscall.c +1 -1
@@ -516,7 +516,7 @@ abi_long do_openbsd_syscall(void *cpu_env, int num,
 abi_long arg1,
         _mcleanup();
 #endif
         gdb_exit(arg1);
-        qemu_plugin_atexit_cb();
+        qemu_plugin_user_exit();
         /* XXX: should free thread stack and CPU env */
         _exit(arg1);

ret = 0; /* avoid warning */

linux-user/exit.c +1 -1
index 70b344048c..527e29cbc1 100644
@@ -35,5 +35,5 @@ void preexit_cleanup(CPUArchState *env, int code)
         __gcov_dump();
 #endif
         gdb_exit(code);
-        qemu_plugin_atexit_cb();
+        qemu_plugin_user_exit();
 }
plugins/core.c +33 -0
index e1bcdb570d..c573b81a96 100644
@@ -487,6 +487,39 @@ void qemu_plugin_register_atexit_cb(qemu_plugin_id_t
 id,
     plugin_register_cb_udata(id, QEMU_PLUGIN_EV_ATEXIT, cb, udata);
 }
 
+/*
+ * Handle exit from linux-user. Unlike the normal atexit() mechanism
+ * we need to handle the clean-up manually as it's possible threads
+ * are still running. We need to remove all callbacks from code
+ * generation, flush the current translations and then we can safely
+ * trigger the exit callbacks.
+ */
+
+void qemu_plugin_user_exit(void)
+{
+    enum qemu_plugin_event ev;
+
+    QEMU_LOCK_GUARD(&plugin.lock);
+
+    start_exclusive();
+
+    /* un-register all callbacks except the final AT_EXIT one */
+    for (ev = 0; ev < QEMU_PLUGIN_EV_MAX; ev++) {
+        if (ev != QEMU_PLUGIN_EV_ATEXIT) {
+            struct qemu_plugin_ctx *ctx;
+            QTAILQ_FOREACH(ctx, &plugin.ctxs, entry) {
+                plugin_unregister_cb__locked(ctx, ev);
+            }
+        }
+    }
+
+    tb_flush(current_cpu);
+    end_exclusive();
+
+    /* now it's safe to handle the exit case */
+    qemu_plugin_atexit_cb();
+}
+
 /*
  * Call this function after longjmp'ing to the main loop. It's possible

that the
* last instruction of a TB might have used helpers, and therefore the
--
2.32.0.264.g75ae10bc75