tcg/plugins: implement a qemu_plugin_user_exit helper
RFC PATCH 1/1 tcg/plugins: implement a qemu_plugin_user_exit helper
In user-mode emulation there is a small race between preexit_cleanup and exit_group() which means we may end up calling instrumented instructions before the kernel reaps child threads. To solve this we implement a new helper which ensures the callbacks are flushed along with any translations before we let the host do it's a thing.
While we are at it make the documentation of qemu_plugin_register_atexit_cb clearer as to what the user can expect.
--- include/qemu/plugin.h | 12 ++++++++++++ include/qemu/qemu-plugin.h | 13 +++++++++++++ bsd-user/syscall.c | 6 +++--- linux-user/exit.c | 2 +- plugins/core.c | 33 +++++++++++++++++++++++++++++++++ 5 files changed, 62 insertions(+), 4 deletions(-)
include/qemu/plugin.h +12 -0
@@ -190,6 +190,16 @@ void qemu_plugin_add_dyn_cb_arr(GArray *arr);void qemu_plugin_disable_mem_helpers (CPUState *cpu); +/** +* qemu_plugin_user_exit(): clean-up callbacks before calling exit callbacks +* +* This is a user-mode only helper that ensure we have fully cleared +* callbacks from all threads before calling the exit callbacks. This +* is so the plugins themselves don't have to jump through hoops to +* guard against race conditions. +*/ +void qemu_plugin_user_exit (void ); +#else /* !CONFIG_PLUGIN */ static inline void qemu_plugin_add_opts (void )
@@ -250,6 +260,8 @@ void qemu_plugin_add_dyn_cb_arr(GArray *arr)static inline void qemu_plugin_disable_mem_helpers (CPUState *cpu) { } +static inline void qemu_plugin_user_exit (void ) +{ }#endif /* !CONFIG_PLUGIN */ #endif /* QEMU_PLUGIN_H */
include/qemu/qemu-plugin.h +13 -0
@@ -549,6 +549,19 @@ void qemu_plugin_vcpu_for_each(qemu_plugin_id_t id,void qemu_plugin_register_flush_cb (qemu_plugin_id_t id,qemu_plugin_simple_cb_t cb); +/** +* qemu_plugin_register_atexit_cb() - register exit callback +* @id: plugin ID +* @cb: callback +* @userdata: user data for callback +* +* The @cb function is called once execution has finished. Plugins +* should be able to free all their resources at this point much like +* after a reset/uninstall callback is called. +* +* In user-mode it is possible a few un-instrumented instructions from +* child threads may run before the host kernel reaps the threads. +*/ void qemu_plugin_register_atexit_cb (qemu_plugin_id_t id,qemu_plugin_udata_cb_t cb,void *userdata);
bsd-user/syscall.c +3 -3
@@ -335,7 +335,7 @@ abi_long do_freebsd_syscall(void *cpu_env, int num, abi_long arg1,_mcleanup ();#endif gdb_exit (arg1); -qemu_plugin_atexit_cb (); +qemu_plugin_user_exit ();/* XXX: should free thread stack and CPU env */ _exit (arg1); ret =0 ;/* avoid warning */
@@ -437,7 +437,7 @@ abi_long do_netbsd_syscall(void *cpu_env, int num, abi_long arg1,_mcleanup ();#endif gdb_exit (arg1); -qemu_plugin_atexit_cb (); +qemu_plugin_user_exit ();/* XXX: should free thread stack and CPU env */ _exit (arg1); ret =0 ;/* avoid warning */
@@ -516,7 +516,7 @@ abi_long do_openbsd_syscall(void *cpu_env, int num, abi_long arg1,_mcleanup ();#endif gdb_exit (arg1); -qemu_plugin_atexit_cb (); +qemu_plugin_user_exit ();/* XXX: should free thread stack and CPU env */ _exit (arg1); ret =0 ;/* avoid warning */
linux-user/exit.c +1 -1
@@ -35,5 +35,5 @@ void preexit_cleanup(CPUArchState *env, int code)__gcov_dump ();#endif gdb_exit (code); -qemu_plugin_atexit_cb (); +qemu_plugin_user_exit (); }
plugins/core.c +33 -0
@@ -487,6 +487,39 @@ void qemu_plugin_register_atexit_cb(qemu_plugin_id_t id,plugin_register_cb_udata (id,QEMU_PLUGIN_EV_ATEXIT , cb, udata); } +/* +* Handle exit from linux-user. Unlike the normal atexit() mechanism +* we need to handle the clean-up manually as it's possible threads +* are still running. We need to remove all callbacks from code +* generation, flush the current translations and then we can safely +* trigger the exit callbacks. +*/ + +void qemu_plugin_user_exit (void ) +{ +enum qemu_plugin_event ev; + +QEMU_LOCK_GUARD (&plugin.lock); + +start_exclusive (); + +/* un-register all callbacks except the final AT_EXIT one */ +for (ev =0 ; ev <QEMU_PLUGIN_EV_MAX ; ev++) { +if (ev !=QEMU_PLUGIN_EV_ATEXIT ) { +struct qemu_plugin_ctx *ctx; + QTAILQ_FOREACH(ctx, &plugin.ctxs, entry) { +plugin_unregister_cb__locked (ctx, ev); + } + } + } + +tb_flush (current_cpu); +end_exclusive (); + +/* now it's safe to handle the exit case */ +qemu_plugin_atexit_cb (); +} +/* * Call this function after longjmp'ing to the main loop. It's possible that the * last instruction of a TB might have used helpers, and therefore the
Signature · 1 line
2.32.0.264.g75ae10bc75
Quoted 9 lines · Alex Bennée
In user-mode emulation there is a small race between
preexit_cleanupandexit_group()which means we may end up calling instrumented instructions before the kernel reaps child threads. To solve this we implement a new helper which ensures the callbacks are flushed along with any translations before we let the host do it's a thing.While we are at it make the documentation of
qemu_plugin_register_atexit_cbclearer as to what the user can expect.
<snip>
Quoted 18 lines
plugins/core.c +17
-0+/* +* Handle exit from linux-user. Unlike the normal atexit() mechanism +* we need to handle the clean-up manually as it's possible threads +* are still running. We need to remove all callbacks from code +* generation, flush the current translations and then we can safely +* trigger the exit callbacks. +*/ + +void qemu_plugin_user_exit (void ) +{ +enum qemu_plugin_event ev; + +QEMU_LOCK_GUARD (&plugin.lock); + +start_exclusive (); + +/* un-register all callbacks except the final AT_EXIT one */
plugins/core.c void qemu_plugin_register_atexit_cb(qemu_plugin_id_t id, · line 516
Quoted 10 lines
plugins/core.c +10
-0+for (ev =0 ; ev <QEMU_PLUGIN_EV_MAX ; ev++) { +if (ev !=QEMU_PLUGIN_EV_ATEXIT ) { +struct qemu_plugin_ctx *ctx; + QTAILQ_FOREACH(ctx, &plugin.ctxs, entry) { +plugin_unregister_cb__locked (ctx, ev); + } + } + } + +tb_flush (current_cpu);
We also need to disable memory helpers during the exclusive period as that is another route into a callback:
--8<---------------cut here---------------start------------->8---
modified plugins/core.c
@@ -498,6 +499,7 @@ void qemu_plugin_register_atexit_cb(qemu_plugin_id_t id,
void qemu_plugin_user_exit(void)
{
enum qemu_plugin_event ev;+ CPUState *cpu;
QEMU_LOCK_GUARD(&plugin.lock);
@@ -514,6 +516,11 @@ void qemu_plugin_user_exit(void)
}
tb_flush(current_cpu);+
+ CPU_FOREACH(cpu) {
+ qemu_plugin_disable_mem_helpers(cpu);
+ }
+
end_exclusive();
/* now it's safe to handle the exit case */--8<---------------cut here---------------end--------------->8---
Quoted 9 lines
plugins/core.c +6
-0+end_exclusive (); + +/* now it's safe to handle the exit case */ +qemu_plugin_atexit_cb (); +} +/* * Call this function after longjmp'ing to the main loop. It's possible that the * last instruction of a TB might have used helpers, and therefore the
Signature · 1 line
Alex Bennée
Quoted 59 lines · Alex Bennée
Quoted 9 lines · Alex Bennée
In user-mode emulation there is a small race between
preexit_cleanupandexit_group()which means we may end up calling instrumented instructions before the kernel reaps child threads. To solve this we implement a new helper which ensures the callbacks are flushed along with any translations before we let the host do it's a thing.While we are at it make the documentation of
qemu_plugin_register_atexit_cbclearer as to what the user can expect.<snip>
Quoted 18 lines
plugins/core.c +17
-0+/* +* Handle exit from linux-user. Unlike the normal atexit() mechanism +* we need to handle the clean-up manually as it's possible threads +* are still running. We need to remove all callbacks from code +* generation, flush the current translations and then we can safely +* trigger the exit callbacks. +*/ + +void qemu_plugin_user_exit (void ) +{ +enum qemu_plugin_event ev; + +QEMU_LOCK_GUARD (&plugin.lock); + +start_exclusive (); + +/* un-register all callbacks except the final AT_EXIT one */ plugins/core.c void qemu_plugin_register_atexit_cb(qemu_plugin_id_t id, · line 516
Quoted 10 lines
plugins/core.c +10
-0+for (ev =0 ; ev <QEMU_PLUGIN_EV_MAX ; ev++) { +if (ev !=QEMU_PLUGIN_EV_ATEXIT ) { +struct qemu_plugin_ctx *ctx; + QTAILQ_FOREACH(ctx, &plugin.ctxs, entry) { +plugin_unregister_cb__locked (ctx, ev); + } + } + } + +tb_flush (current_cpu);We also need to disable memory helpers during the exclusive period as that is another route into a callback:
--8<---------------cut here---------------start------------->8---
modifiedplugins/core.c(quoted diff) +1
-0@@ -498,6 +499,7 @@ void qemu_plugin_register_atexit_cb(qemu_plugin_id_t id, void qemu_plugin_user_exit(void) { enum qemu_plugin_event ev; + CPUState *cpu; QEMU_LOCK_GUARD(&plugin.lock);@@ -514,6 +516,11 @@ void qemu_plugin_user_exit(void) }
Review void qemu_plugin_user_exit(void)
Quoted 10 lines
(quoted diff) +5
-1tb_flush(current_cpu); + + CPU_FOREACH(cpu) { + qemu_plugin_disable_mem_helpers(cpu); + } + end_exclusive(); /* now it's safe to handle the exit case */ --8<---------------cut here---------------end--------------->8---
I think both of these are find from a bsd-user point of view.
Warner
Quoted 15 lines
plugins/core.c void qemu_plugin_register_atexit_cb(qemu_plugin_id_t id,
Quoted 8 lines
plugins/core.c +6
-0+end_exclusive (); + +/* now it's safe to handle the exit case */ +qemu_plugin_atexit_cb (); +} +/* * Call this function after longjmp'ing to the main loop. It's possible that the
Quoted 1 line
* last instruction of a TB might have used helpers, and therefore the
--
Alex Bennée
Alex Bennée
Acked-by or Reviewed-by:?Quoted 67 lines · Warner Losh
On Mon, Jul 19, 2021, 7:57 AM Alex Bennée <[email protected]> wrote:
Quoted 9 lines · Alex Bennée
In user-mode emulation there is a small race between
preexit_cleanupandexit_group()which means we may end up calling instrumented instructions before the kernel reaps child threads. To solve this we implement a new helper which ensures the callbacks are flushed along with any translations before we let the host do it's a thing.While we are at it make the documentation of
qemu_plugin_register_atexit_cbclearer as to what the user can expect.<snip>
Quoted 18 lines
plugins/core.c +17
-0+/* +* Handle exit from linux-user. Unlike the normal atexit() mechanism +* we need to handle the clean-up manually as it's possible threads +* are still running. We need to remove all callbacks from code +* generation, flush the current translations and then we can safely +* trigger the exit callbacks. +*/ + +void qemu_plugin_user_exit (void ) +{ +enum qemu_plugin_event ev; + +QEMU_LOCK_GUARD (&plugin.lock); + +start_exclusive (); + +/* un-register all callbacks except the final AT_EXIT one */ plugins/core.c void qemu_plugin_register_atexit_cb(qemu_plugin_id_t id, · line 516
Quoted 10 lines
plugins/core.c +10
-0+for (ev =0 ; ev <QEMU_PLUGIN_EV_MAX ; ev++) { +if (ev !=QEMU_PLUGIN_EV_ATEXIT ) { +struct qemu_plugin_ctx *ctx; + QTAILQ_FOREACH(ctx, &plugin.ctxs, entry) { +plugin_unregister_cb__locked (ctx, ev); + } + } + } + +tb_flush (current_cpu);We also need to disable memory helpers during the exclusive period as that is another route into a callback:
--8<---------------cut here---------------start------------->8---
modifiedplugins/core.c
@@ -498,6 +499,7 @@ void qemu_plugin_register_atexit_cb(qemu_plugin_id_t id,
voidqemu_plugin_user_exit(void){ enum qemu_plugin_event ev;+ CPUState *cpu;
QEMU_LOCK_GUARD(&plugin.lock);@@ -514,6 +516,11 @@ void
qemu_plugin_user_exit(void)} tb_flush(current_cpu); + + CPU_FOREACH(cpu) { + qemu_plugin_disable_mem_helpers(cpu); + } + end_exclusive(); /* now it's safe to handle the exit case */
Quoted 3 lines
--8<---------------cut here---------------end--------------->8---
I think both of these are find from a bsd-user point of view.
Signature · 1 line
Alex Bennée
Warner Losh
Acked-byQuoted 71 lines · Alex Bennée
Quoted 1 line · Warner Losh
On Mon, Jul 19, 2021, 7:57 AM Alex Bennée <[email protected]>
Quoted 39 lines · wrote:
Quoted 9 lines · Alex Bennée
In user-mode emulation there is a small race between
preexit_cleanupandexit_group()which means we may end up calling instrumented instructions before the kernel reaps child threads. To solve this we implement a new helper which ensures the callbacks are flushed along with any translations before we let the host do it's a thing.While we are at it make the documentation of
qemu_plugin_register_atexit_cbclearer as to what the user can expect.<snip>
Quoted 26 lines
plugins/core.c +25
-0+/* +* Handle exit from linux-user. Unlike the normal atexit() mechanism +* we need to handle the clean-up manually as it's possible threads +* are still running. We need to remove all callbacks from code +* generation, flush the current translations and then we can safely +* trigger the exit callbacks. +*/ + +void qemu_plugin_user_exit (void ) +{ +enum qemu_plugin_event ev; + +QEMU_LOCK_GUARD (&plugin.lock); + +start_exclusive (); + +/* un-register all callbacks except the final AT_EXIT one */ +for (ev =0 ; ev <QEMU_PLUGIN_EV_MAX ; ev++) { +if (ev !=QEMU_PLUGIN_EV_ATEXIT ) { +struct qemu_plugin_ctx *ctx; + QTAILQ_FOREACH(ctx, &plugin.ctxs, entry) { +plugin_unregister_cb__locked (ctx, ev); + } + } + }Quoted 8 lines
Review
Quoted 2 lines
(quoted diff) +2
-0+ + tb_flush(current_cpu);We also need to disable memory helpers during the exclusive period as that is another route into a callback:
--8<---------------cut here---------------start------------->8---
modifiedplugins/core.c
@@ -498,6 +499,7 @@ voidqemu_plugin_register_atexit_cb(qemu_plugin_id_t id,
Quoted 18 lines
void
qemu_plugin_user_exit(void){ enum qemu_plugin_event ev;+ CPUState *cpu;
QEMU_LOCK_GUARD(&plugin.lock);@@ -514,6 +516,11 @@ void
qemu_plugin_user_exit(void)} tb_flush(current_cpu); + + CPU_FOREACH(cpu) { + qemu_plugin_disable_mem_helpers(cpu); + } + end_exclusive();
Quoted 7 lines
Quoted 4 lines
/* now it's safe to handle the exit case */--8<---------------cut here---------------end--------------->8---
I think both of these are find from a bsd-user point of view.
Acked-by or Reviewed-by:?
Sorry I wasn't clear.
Quoted 3 lines
--
Alex Bennée
Quoted 11 lines · On Mon, Jul 19, 2021, 14:37 Alex Bennée <[email protected]> wrote:
In user-mode emulation there is a small race between
preexit_cleanupandexit_group()which means we may end up calling instrumented instructions before the kernel reaps child threads. To solve this we implement a new helper which ensures the callbacks are flushed along with any translations before we let the host do it's a thing.While we are at it make the documentation of
qemu_plugin_register_atexit_cbclearer as to what the user can expect.Signed-off-by Alex Bennée <[email protected]>
Looks great to me and I cannot reproduce the race with it installed in.
With calling qemu_plugin_disable_mem_helpers:
---
Quoted 158 lines
include/qemu/plugin.h | 12 ++++++++++++ include/qemu/qemu-plugin.h | 13 +++++++++++++ bsd-user/syscall.c | 6 +++--- linux-user/exit.c | 2 +- plugins/core.c | 33 +++++++++++++++++++++++++++++++++ 5 files changed, 62 insertions(+), 4 deletions(-)include/qemu/plugin.h +2
-0@@ -190,6 +190,16 @@ void qemu_plugin_add_dyn_cb_arr(GArray *arr);void qemu_plugin_disable_mem_helpers (CPUState *cpu); +/** +* qemu_plugin_user_exit(): clean-up callbacks before calling exit callbacks
+ *
+ * This is a user-mode only helper that ensure we have fully cleared
+ * callbacks from all threads before calling the exit callbacks. This
+ * is so the plugins themselves don't have to jump through hoops to
+ * guard against race conditions.
+ */include/qemu/plugin.h +4
-0+void qemu_plugin_user_exit (void ); +#else /* !CONFIG_PLUGIN */ static inline void qemu_plugin_add_opts (void )@@ -250,6 +260,8 @@ void qemu_plugin_add_dyn_cb_arr(GArray *arr)static inline void qemu_plugin_disable_mem_helpers (CPUState *cpu) { } +static inline void qemu_plugin_user_exit (void ) +{ }#endif /* !CONFIG_PLUGIN */ #endif /* QEMU_PLUGIN_H */ include/qemu/qemu-plugin.h +13
-0@@ -549,6 +549,19 @@ void qemu_plugin_vcpu_for_each(qemu_plugin_id_t id,void qemu_plugin_register_flush_cb (qemu_plugin_id_t id,qemu_plugin_simple_cb_t cb); +/** +* qemu_plugin_register_atexit_cb() - register exit callback +* @id: plugin ID +* @cb: callback +* @userdata: user data for callback +* +* The @cb function is called once execution has finished. Plugins +* should be able to free all their resources at this point much like +* after a reset/uninstall callback is called. +* +* In user-mode it is possible a few un-instrumented instructions from +* child threads may run before the host kernel reaps the threads. +*/ void qemu_plugin_register_atexit_cb (qemu_plugin_id_t id,qemu_plugin_udata_cb_t cb,void *userdata);bsd-user/syscall.c +1
-1@@ -335,7 +335,7 @@ abi_long do_freebsd_syscall(void *cpu_env, int num,abi_long arg1,_mcleanup ();#endif gdb_exit (arg1); -qemu_plugin_atexit_cb (); +qemu_plugin_user_exit ();/* XXX: should free thread stack and CPU env */ _exit (arg1);ret = 0; /* avoid warning */
bsd-user/syscall.c +1
-1@@ -437,7 +437,7 @@ abi_long do_netbsd_syscall(void *cpu_env, int num,abi_long arg1,_mcleanup ();#endif gdb_exit (arg1); -qemu_plugin_atexit_cb (); +qemu_plugin_user_exit ();/* XXX: should free thread stack and CPU env */ _exit (arg1);ret = 0; /* avoid warning */
bsd-user/syscall.c +1
-1@@ -516,7 +516,7 @@ abi_long do_openbsd_syscall(void *cpu_env, int num,abi_long arg1,_mcleanup ();#endif gdb_exit (arg1); -qemu_plugin_atexit_cb (); +qemu_plugin_user_exit ();/* XXX: should free thread stack and CPU env */ _exit (arg1);ret = 0; /* avoid warning */
linux-user/exit.c +1
-1@@ -35,5 +35,5 @@ void preexit_cleanup(CPUArchState *env, int code)__gcov_dump ();#endif gdb_exit (code); -qemu_plugin_atexit_cb (); +qemu_plugin_user_exit (); }plugins/core.c +33
-0@@ -487,6 +487,39 @@ void qemu_plugin_register_atexit_cb(qemu_plugin_id_t id,plugin_register_cb_udata (id,QEMU_PLUGIN_EV_ATEXIT , cb, udata); } +/* +* Handle exit from linux-user. Unlike the normal atexit() mechanism +* we need to handle the clean-up manually as it's possible threads +* are still running. We need to remove all callbacks from code +* generation, flush the current translations and then we can safely +* trigger the exit callbacks. +*/ + +void qemu_plugin_user_exit (void ) +{ +enum qemu_plugin_event ev; + +QEMU_LOCK_GUARD (&plugin.lock); + +start_exclusive (); + +/* un-register all callbacks except the final AT_EXIT one */ +for (ev =0 ; ev <QEMU_PLUGIN_EV_MAX ; ev++) { +if (ev !=QEMU_PLUGIN_EV_ATEXIT ) { +struct qemu_plugin_ctx *ctx; + QTAILQ_FOREACH(ctx, &plugin.ctxs, entry) { +plugin_unregister_cb__locked (ctx, ev); + } + } + } + +tb_flush (current_cpu); +end_exclusive (); + +/* now it's safe to handle the exit case */ +qemu_plugin_atexit_cb (); +} +/* * Call this function after longjmp'ing to the main loop. It's possible that the
* last instruction of a TB might have used helpers, and therefore the
--
2.32.0.264.g75ae10bc75