nbd/server: do not poll within a coroutine context
PATCH v4 1/1 nbd/server: do not poll within a coroutine context
From: Zhu Yangyang <[email protected]>
Coroutines are not supposed to block. Instead, they should yield.
The client performs TLS upgrade outside of an AIOContext, during synchronous handshake; this still requires g_main_loop. But the server responds to TLS upgrade inside a coroutine, so a nested g_main_loop is wrong. Since the two callbacks no longer share more than the setting of data.complete and data.error, it's just as easy to use static helpers instead of trying to share a common code path.
---
v3: https://lists.gnu.org/archive/html/qemu-devel/2024-04/msg00375.html
in v4, factor even the struct to the .c files, avoiding a union [Vladimir]
nbd/nbd-internal.h | 10 ---------- nbd/client.c | 27 +++++++++++++++++++++++---- nbd/common.c | 11 ----------- nbd/server.c | 29 +++++++++++++++++++++++------ 4 files changed, 46 insertions(+), 31 deletions(-)
nbd/nbd-internal.h +0 -10
@@ -72,16 +72,6 @@ static inline int nbd_write(QIOChannel *ioc, const void *buffer, size_t size, return qio_channel_write_all(ioc, buffer, size, errp) < 0 ? -EIO : 0; } -struct NBDTLSHandshakeData { -GMainLoop *loop; -bool complete; -Error *error; -}; - - -void nbd_tls_handshake (QIOTask *task, -void *opaque); -int nbd_drop (QIOChannel *ioc,size_t size,Error **errp);#endif
nbd/client.c +23 -4
@@ -596,13 +596,31 @@ static int nbd_request_simple_option(QIOChannel *ioc, int opt, bool strict,return 1 ; } +/* Callback to learn when QIO TLS upgrade is complete */ +struct NBDTLSClientHandshakeData { +bool complete; +Error *error; +GMainLoop *loop; +}; + +static void nbd_client_tls_handshake (QIOTask *task,void *opaque) +{ +struct NBDTLSClientHandshakeData *data = opaque; + +qio_task_propagate_error (task, &data->error); + data->complete = true; +if (data->loop) { +g_main_loop_quit (data->loop); + } +} +static QIOChannel *nbd_receive_starttls (QIOChannel *ioc,QCryptoTLSCreds *tlscreds,const char *hostname,Error **errp) {int ret;QIOChannelTLS *tioc; -struct NBDTLSHandshakeData data = {0 }; +struct NBDTLSClientHandshakeData data = {0 }; ret =nbd_request_simple_option (ioc,NBD_OPT_STARTTLS , true, errp);if (ret <=0 ) {
@@ -619,18 +637,19 @@ static QIOChannel *nbd_receive_starttls(QIOChannel *ioc,return NULL ; }qio_channel_set_name (QIO_CHANNEL (tioc),"nbd-client-tls" ); - data.loop =g_main_loop_new (g_main_context_default (), FALSE);trace_nbd_receive_starttls_tls_handshake ();qio_channel_tls_handshake (tioc, - nbd_tls_handshake, + nbd_client_tls_handshake, &data,NULL ,NULL );if (!data.complete) { + data.loop =g_main_loop_new (g_main_context_default (), FALSE);g_main_loop_run (data.loop); +g_main_loop_unref (data.loop); } -g_main_loop_unref (data.loop); +if (data.error) {error_propagate (errp, data.error);object_unref (OBJECT (tioc));
nbd/common.c +0 -11
@@ -47,17 +47,6 @@ int nbd_drop(QIOChannel *ioc, size_t size, Error **errp) } -void nbd_tls_handshake (QIOTask *task, -void *opaque) -{ -struct NBDTLSHandshakeData *data = opaque; - -qio_task_propagate_error (task, &data->error); - data->complete = true; -g_main_loop_quit (data->loop); -} - -const char *nbd_opt_lookup (uint32_t opt) {switch (opt) {
nbd/server.c +23 -6
@@ -748,6 +748,23 @@ static int nbd_negotiate_handle_info(NBDClient *client, Error **errp)return rc; } +/* Callback to learn when QIO TLS upgrade is complete */ +struct NBDTLSServerHandshakeData { +bool complete; +Error *error; +Coroutine *co; +}; + +static void nbd_server_tls_handshake (QIOTask *task,void *opaque) +{ +struct NBDTLSServerHandshakeData *data = opaque; + +qio_task_propagate_error (task, &data->error); + data->complete = true; +if (!qemu_coroutine_entered (data->co)) { +aio_co_wake (data->co); + } +}/* Handle NBD_OPT_STARTTLS. Return NULL to drop connection, or else the * new channel for all further (now-encrypted) communication. */
@@ -756,7 +773,7 @@ static QIOChannel *nbd_negotiate_handle_starttls(NBDClient *client, {QIOChannel *ioc;QIOChannelTLS *tioc; -struct NBDTLSHandshakeData data = {0 }; +struct NBDTLSServerHandshakeData data = {0 };assert (client->opt ==NBD_OPT_STARTTLS );
@@ -777,17 +794,17 @@ static QIOChannel *nbd_negotiate_handle_starttls(NBDClient *client,qio_channel_set_name (QIO_CHANNEL (tioc),"nbd-server-tls" );trace_nbd_negotiate_handle_starttls_handshake (); - data.loop =g_main_loop_new (g_main_context_default (), FALSE); + data.co =qemu_coroutine_self ();qio_channel_tls_handshake (tioc, - nbd_tls_handshake, + nbd_server_tls_handshake, &data,NULL ,NULL ); -if (!data.complete) { -g_main_loop_run (data.loop); +while (!data.complete) { +qemu_coroutine_yield (); } -g_main_loop_unref (data.loop); +if (data.error) {object_unref (OBJECT (tioc));error_propagate (errp, data.error);
Signature · 1 line
2.44.0
Quoted 11 lines · Eric Blake
From: Zhu Yangyang <[email protected]>
Coroutines are not supposed to block. Instead, they should yield.
The client performs TLS upgrade outside of an AIOContext, during synchronous handshake; this still requires
g_main_loop. But the server responds to TLS upgrade inside a coroutine, so a nestedg_main_loopis wrong. Since the two callbacks no longer share more than the setting of data.complete and data.error, it's just as easy to use static helpers instead of trying to share a common code path.
Quoted 4 lines
Fixes f95910f ("nbd: implement TLS support in the protocol negotiation")Signed-off-by Zhu Yangyang <[email protected]>[eblake: move callbacks to their use point]Signed-off-by Eric Blake <[email protected]>
still, some notes below
Quoted 77 lines
---
v3: https://lists.gnu.org/archive/html/qemu-devel/2024-04/msg00375.html
in v4, factor even the struct to the .c files, avoiding a union [Vladimir]
nbd/nbd-internal.h | 10 ---------- nbd/client.c | 27 +++++++++++++++++++++++---- nbd/common.c | 11 ----------- nbd/server.c | 29 +++++++++++++++++++++++------ 4 files changed, 46 insertions(+), 31 deletions(-)nbd/nbd-internal.h +0
-10@@ -72,16 +72,6 @@ static inline int nbd_write(QIOChannel *ioc, const void *buffer, size_t size, return qio_channel_write_all(ioc, buffer, size, errp) < 0 ? -EIO : 0; } -struct NBDTLSHandshakeData { -GMainLoop *loop; -bool complete; -Error *error; -}; - - -void nbd_tls_handshake (QIOTask *task, -void *opaque); -int nbd_drop (QIOChannel *ioc,size_t size,Error **errp);#endif nbd/client.c +19
-2@@ -596,13 +596,31 @@ static int nbd_request_simple_option(QIOChannel *ioc, int opt, bool strict,return 1 ; } +/* Callback to learn when QIO TLS upgrade is complete */ +struct NBDTLSClientHandshakeData { +bool complete; +Error *error; +GMainLoop *loop; +}; + +static void nbd_client_tls_handshake (QIOTask *task,void *opaque) +{ +struct NBDTLSClientHandshakeData *data = opaque; + +qio_task_propagate_error (task, &data->error); + data->complete = true; +if (data->loop) { +g_main_loop_quit (data->loop); + } +} +static QIOChannel *nbd_receive_starttls (QIOChannel *ioc,QCryptoTLSCreds *tlscreds,const char *hostname,Error **errp) {int ret;QIOChannelTLS *tioc; -struct NBDTLSHandshakeData data = {0 }; +struct NBDTLSClientHandshakeData data = {0 }; ret =nbd_request_simple_option (ioc,NBD_OPT_STARTTLS , true, errp);if (ret <=0 ) {@@ -619,18 +637,19 @@ static QIOChannel *nbd_receive_starttls(QIOChannel *ioc,return NULL ; }qio_channel_set_name (QIO_CHANNEL (tioc),"nbd-client-tls" ); - data.loop =g_main_loop_new (g_main_context_default (), FALSE);trace_nbd_receive_starttls_tls_handshake (); qio_channel_tls_handshake(tioc,
nbd/client.c static QIOChannel *nbd_receive_starttls(QIOChannel *ioc, · line 650
Quoted 10 lines
nbd/client.c +3
-1- nbd_tls_handshake, + nbd_client_tls_handshake, &data,NULL , NULL);if (!data.complete) { + data.loop =g_main_loop_new (g_main_context_default (), FALSE);g_main_loop_run (data.loop); +g_main_loop_unref (data.loop);
probably good to assert(data.complete);
nbd/server.c static QIOChannel *nbd_negotiate_handle_starttls(NBDClient *client,
Quoted 66 lines
nbd/client.c +1
-1} -g_main_loop_unref (data.loop); +if (data.error) {error_propagate (errp, data.error);object_unref (OBJECT (tioc));nbd/common.c +0
-11@@ -47,17 +47,6 @@ int nbd_drop(QIOChannel *ioc, size_t size, Error **errp) } -void nbd_tls_handshake (QIOTask *task, -void *opaque) -{ -struct NBDTLSHandshakeData *data = opaque; - -qio_task_propagate_error (task, &data->error); - data->complete = true; -g_main_loop_quit (data->loop); -} - -const char *nbd_opt_lookup (uint32_t opt) {switch (opt) {nbd/server.c +18
-1@@ -748,6 +748,23 @@ static int nbd_negotiate_handle_info(NBDClient *client, Error **errp)return rc; } +/* Callback to learn when QIO TLS upgrade is complete */ +struct NBDTLSServerHandshakeData { +bool complete; +Error *error; +Coroutine *co; +}; + +static void nbd_server_tls_handshake (QIOTask *task,void *opaque) +{ +struct NBDTLSServerHandshakeData *data = opaque; + +qio_task_propagate_error (task, &data->error); + data->complete = true; +if (!qemu_coroutine_entered (data->co)) { +aio_co_wake (data->co); + } +}/* Handle NBD_OPT_STARTTLS. Return NULL to drop connection, or else the * new channel for all further (now-encrypted) communication. */ @@ -756,7 +773,7 @@ static QIOChannel *nbd_negotiate_handle_starttls(NBDClient *client, {QIOChannel *ioc;QIOChannelTLS *tioc; -struct NBDTLSHandshakeData data = {0 }; +struct NBDTLSServerHandshakeData data = {0 };assert (client->opt ==NBD_OPT_STARTTLS );@@ -777,17 +794,17 @@ static QIOChannel *nbd_negotiate_handle_starttls(NBDClient *client,
preexisting: lack coroutine_fn, as well as caller nbd_negotiate_options()
Quoted 5 lines
nbd/server.c +1
-1qio_channel_set_name (QIO_CHANNEL (tioc),"nbd-server-tls" );trace_nbd_negotiate_handle_starttls_handshake (); - data.loop =g_main_loop_new (g_main_context_default (), FALSE); + data.co =qemu_coroutine_self ();
nbd/server.c static QIOChannel *nbd_negotiate_handle_starttls(NBDClient *client, · line 804
Quoted 10 lines
nbd/server.c +2
-3qio_channel_tls_handshake (tioc, - nbd_tls_handshake, + nbd_server_tls_handshake, &data,NULL ,NULL ); -if (!data.complete) { -g_main_loop_run (data.loop); +while (!data.complete) {
not "if", but "while".. Do we expect entering from somewhere except nbd_server_tls_handshake?
if not, probably safer construction would be
if (!data.complete) {
qemu_coroutine_yield();
assert(data.complete);
}
- to avoid hiding unexpected coroutine switching bugs
Quoted 7 lines
nbd/server.c +2
-1+qemu_coroutine_yield (); } -g_main_loop_unref (data.loop); +if (data.error) {object_unref (OBJECT (tioc));error_propagate (errp, data.error);
Signature · 2 lines
Best regards, Vladimir
Quoted 12 lines · Vladimir Sementsov-Ogievskiy
Quoted 11 lines · Eric Blake
From: Zhu Yangyang <[email protected]>
Coroutines are not supposed to block. Instead, they should yield.
The client performs TLS upgrade outside of an AIOContext, during synchronous handshake; this still requires
g_main_loop. But the server responds to TLS upgrade inside a coroutine, so a nestedg_main_loopis wrong. Since the two callbacks no longer share more than the setting of data.complete and data.error, it's just as easy to use static helpers instead of trying to share a common code path.
Quoted 6 lines
Quoted 4 lines
Fixes f95910f ("nbd: implement TLS support in the protocol negotiation")Signed-off-by Zhu Yangyang <[email protected]>[eblake: move callbacks to their use point]Signed-off-by Eric Blake <[email protected]>Reviewed-by Vladimir Sementsov-Ogievskiy <[email protected]>
I'm debating whether it is worth trying to shove this into 9.0; -rc3 is very late, and the problem is pre-existing, so I'm leaning towards no. At which point, it's better to get this right.
Quoted 63 lines
still, some notes below
Quoted 60 lines
---
v3: https://lists.gnu.org/archive/html/qemu-devel/2024-04/msg00375.html
in v4, factor even the struct to the .c files, avoiding a union [Vladimir]
nbd/nbd-internal.h | 10 ---------- nbd/client.c | 27 +++++++++++++++++++++++---- nbd/common.c | 11 ----------- nbd/server.c | 29 +++++++++++++++++++++++------ 4 files changed, 46 insertions(+), 31 deletions(-)+++
b/nbd/client.cnbd/client.c +20
-3@@ -596,13 +596,31 @@ static int nbd_request_simple_option(QIOChannel *ioc, int opt, bool strict,return 1 ; } +/* Callback to learn when QIO TLS upgrade is complete */ +struct NBDTLSClientHandshakeData { +bool complete; +Error *error; +GMainLoop *loop; +}; + +static void nbd_client_tls_handshake (QIOTask *task,void *opaque) +{ +struct NBDTLSClientHandshakeData *data = opaque; + +qio_task_propagate_error (task, &data->error); + data->complete = true; +if (data->loop) { +g_main_loop_quit (data->loop); + } +} +static QIOChannel *nbd_receive_starttls (QIOChannel *ioc,QCryptoTLSCreds *tlscreds,const char *hostname,Error **errp) {int ret;QIOChannelTLS *tioc; -struct NBDTLSHandshakeData data = {0 }; +struct NBDTLSClientHandshakeData data = {0 }; ret =nbd_request_simple_option (ioc,NBD_OPT_STARTTLS , true, errp);if (ret <=0 ) {@@ -619,18 +637,19 @@ static QIOChannel *nbd_receive_starttls(QIOChannel *ioc,return NULL ; }qio_channel_set_name (QIO_CHANNEL (tioc),"nbd-client-tls" ); - data.loop =g_main_loop_new (g_main_context_default (), FALSE);trace_nbd_receive_starttls_tls_handshake ();qio_channel_tls_handshake (tioc, - nbd_tls_handshake, + nbd_client_tls_handshake, &data,NULL ,NULL );
Quoted 6 lines
nbd/client.c static QIOChannel *nbd_receive_starttls(QIOChannel *ioc, · line 650
Quoted 4 lines
nbd/client.c +2
-0if (!data.complete) { + data.loop =g_main_loop_new (g_main_context_default (), FALSE);g_main_loop_run (data.loop); +g_main_loop_unref (data.loop);probably good to assert(data.complete);
Seems reasonable.
Quoted 32 lines
Quoted 32 lines
nbd/server.c +19
-1+++ b/nbd/server.c@@ -748,6 +748,23 @@ static int nbd_negotiate_handle_info(NBDClient *client, Error **errp)return rc; } +/* Callback to learn when QIO TLS upgrade is complete */ +struct NBDTLSServerHandshakeData { +bool complete; +Error *error; +Coroutine *co; +}; + +static void nbd_server_tls_handshake (QIOTask *task,void *opaque) +{ +struct NBDTLSServerHandshakeData *data = opaque; + +qio_task_propagate_error (task, &data->error); + data->complete = true; +if (!qemu_coroutine_entered (data->co)) { +aio_co_wake (data->co); + } +}/* Handle NBD_OPT_STARTTLS. Return NULL to drop connection, or else the * new channel for all further (now-encrypted) communication. */ @@ -756,7 +773,7 @@ static QIOChannel *nbd_negotiate_handle_starttls(NBDClient *client, {QIOChannel *ioc;QIOChannelTLS *tioc; -struct NBDTLSHandshakeData data = {0 }; +struct NBDTLSServerHandshakeData data = {0 };
Quoted 5 lines
nbd/server.c static QIOChannel *nbd_negotiate_handle_starttls(NBDClient *client,
Quoted 3 lines
nbd/server.c +0
-0assert (client->opt ==NBD_OPT_STARTTLS );@@ -777,17 +794,17 @@ static QIOChannel *nbd_negotiate_handle_starttls(NBDClient *client,preexisting: lack
coroutine_fn, as well as callernbd_negotiate_options()
Indeed, so now would not hurt to add them now that a callback is no longer shared between callers in different context. But probably as a separate patch.
Quoted 19 lines
Quoted 6 lines
nbd/server.c +1
-1qio_channel_set_name (QIO_CHANNEL (tioc),"nbd-server-tls" );trace_nbd_negotiate_handle_starttls_handshake (); - data.loop =g_main_loop_new (g_main_context_default (), FALSE); + data.co =qemu_coroutine_self ();nbd/server.c static QIOChannel *nbd_negotiate_handle_starttls(NBDClient *client, · line 804
Quoted 10 lines
nbd/server.c +2
-3qio_channel_tls_handshake (tioc, - nbd_tls_handshake, + nbd_server_tls_handshake, &data,NULL ,NULL ); -if (!data.complete) { -g_main_loop_run (data.loop); +while (!data.complete) {not "if", but "while".. Do we expect entering from somewhere except
nbd_server_tls_handshake?
Quoted 8 lines
if not, probably safer construction would be
if (!data.complete) { qemu_coroutine_yield(); assert(data.complete); }- to avoid hiding unexpected coroutine switching bugs
TLS handshake is early enough in the NBD connection that there should not be any parallel I/O yet (we can't switch to parallel outstanding transactions until after successful NBD_OPT_GO), so I like your idea of asserting that the coroutine is entered at most once.
v5 coming up
Signature · 3 lines
Eric Blake, Principal Software Engineer Red Hat, Inc. Virtualization: qemu.org | libguestfs.org