io: follow coroutine AioContext in qio_channel_yield()
v3 cover io: follow coroutine AioContext in qio_channel_yield()
v3:
- Fix wrong copy-pasted function name in doc comment [Eric]
- Fix "match" -> "matches" in comment [Eric]
v2:
- Add Patch 1 & 2 to remove unused NBD aio_context arguments and dead code [Fabiano]
- Remove attach/detach prototypes from "io/channel.h" [Daniel]
- Add utility function to set fd handlers [Daniel]
The ongoing QEMU multi-queue block layer effort makes it possible for multiple threads to process I/O in parallel. The nbd block driver is not compatible with the multi-queue block layer yet because QIOChannel cannot be used easily from coroutines running in multiple threads. This series changes the QIOChannel API to make that possible.
Stefan Hajnoczi (4):
nbd: drop unused nbd_receive_negotiate() aio_context argument
nbd: drop unused nbd_start_negotiate() aio_context argument
io: check there are no qio_channel_yield() coroutines during
->finalize()
io: follow coroutine AioContext in qio_channel_yield()
include/block/nbd.h | 3 +- include/io/channel-util.h | 23 ++++++ include/io/channel.h | 69 ++++++++--------- include/qemu/vhost-user-server.h | 1 + block/nbd.c | 11 +-- io/channel-command.c | 10 ++- io/channel-file.c | 9 ++- io/channel-null.c | 3 +- io/channel-socket.c | 9 ++- io/channel-tls.c | 6 +- io/channel-util.c | 24 ++++++ io/channel.c | 124 ++++++++++++++++++++++--------- migration/channel-block.c | 3 +- nbd/client-connection.c | 3 +- nbd/client.c | 14 +--- nbd/server.c | 14 +--- qemu-nbd.c | 4 +- scsi/qemu-pr-helper.c | 4 +- util/vhost-user-server.c | 27 +++++-- 19 files changed, 228 insertions(+), 133 deletions(-)
Signature · 1 line
2.41.0
PATCH v3 1/4 nbd: drop unused nbd_receive_negotiate() aio_context argument
aio_context is always NULL, so drop it.
--- include/block/nbd.h | 3 +-- nbd/client-connection.c | 3 +-- nbd/client.c | 5 ++--- qemu-nbd.c | 4 ++-- 4 files changed, 6 insertions(+), 9 deletions(-)
include/block/nbd.h +1 -2
@@ -324,8 +324,7 @@ typedef struct NBDExportInfo {char **contexts; } NBDExportInfo; -int nbd_receive_negotiate (AioContext *aio_context,QIOChannel *ioc, -QCryptoTLSCreds *tlscreds, +int nbd_receive_negotiate (QIOChannel *ioc,QCryptoTLSCreds *tlscreds,const char *hostname,QIOChannel **outioc,NBDExportInfo *info,Error **errp);void nbd_free_export_list (NBDExportInfo *info,int count);
nbd/client-connection.c +1 -2
@@ -146,8 +146,7 @@ static int nbd_connect(QIOChannelSocket *sioc, SocketAddress *addr,return 0 ; } - ret =nbd_receive_negotiate (NULL ,QIO_CHANNEL (sioc), tlscreds, - tlshostname, + ret =nbd_receive_negotiate (QIO_CHANNEL (sioc), tlscreds, tlshostname, outioc, info, errp);if (ret <0 ) {/*
nbd/client.c +2 -3
@@ -1014,8 +1014,7 @@ static int nbd_negotiate_finish_oldstyle(QIOChannel *ioc, NBDExportInfo *info,* Returns: negative errno: failure talking to server * 0: server is connected */ -int nbd_receive_negotiate (AioContext *aio_context,QIOChannel *ioc, -QCryptoTLSCreds *tlscreds, +int nbd_receive_negotiate (QIOChannel *ioc,QCryptoTLSCreds *tlscreds,const char *hostname,QIOChannel **outioc,NBDExportInfo *info,Error **errp) {
@@ -1027,7 +1026,7 @@ int nbd_receive_negotiate(AioContext *aio_context, QIOChannel *ioc,assert (info->name &&strlen (info->name) <=NBD_MAX_STRING_SIZE );trace_nbd_receive_negotiate_name (info->name); - result =nbd_start_negotiate (aio_context, ioc, tlscreds, hostname, outioc, + result =nbd_start_negotiate (NULL , ioc, tlscreds, hostname, outioc, info->structured_reply, &zeroes, errp);if (result <0 ) {return result;
qemu-nbd.c +2 -2
@@ -295,8 +295,8 @@ static void *nbd_client_thread(void *arg) goto out; } -if (nbd_receive_negotiate (NULL ,QIO_CHANNEL (sioc), -NULL ,NULL ,NULL , &info, &local_error) <0 ) { +if (nbd_receive_negotiate (QIO_CHANNEL (sioc),NULL ,NULL ,NULL , + &info, &local_error) <0 ) {if (local_error) {error_report_err (local_error); }
Signature · 1 line
2.41.0
PATCH v3 2/4 nbd: drop unused nbd_start_negotiate() aio_context argument
In reply to Stefan Hajnocziaio_context is always NULL, so drop it.
--- nbd/client.c | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-)
nbd/client.c +3 -8
@@ -877,8 +877,7 @@ static int nbd_list_meta_contexts(QIOChannel *ioc,* Returns: negative errno: failure talking to server * non-negative: enum NBDMode describing server abilities */ -static int nbd_start_negotiate (AioContext *aio_context,QIOChannel *ioc, -QCryptoTLSCreds *tlscreds, +static int nbd_start_negotiate (QIOChannel *ioc,QCryptoTLSCreds *tlscreds,const char *hostname,QIOChannel **outioc,bool structured_reply,bool *zeroes,Error **errp)
@@ -946,10 +945,6 @@ static int nbd_start_negotiate(AioContext *aio_context, QIOChannel *ioc,return -EINVAL ; } ioc = *outioc; -if (aio_context) { -qio_channel_set_blocking (ioc, false,NULL ); -qio_channel_attach_aio_context (ioc, aio_context); - } }else {error_setg (errp,"Server does not support STARTTLS" );return -EINVAL ;
@@ -1026,7 +1021,7 @@ int nbd_receive_negotiate(QIOChannel *ioc, QCryptoTLSCreds *tlscreds,assert (info->name &&strlen (info->name) <=NBD_MAX_STRING_SIZE );trace_nbd_receive_negotiate_name (info->name); - result =nbd_start_negotiate (NULL , ioc, tlscreds, hostname, outioc, + result =nbd_start_negotiate (ioc, tlscreds, hostname, outioc, info->structured_reply, &zeroes, errp);if (result <0 ) {return result;
@@ -1149,7 +1144,7 @@ int nbd_receive_export_list(QIOChannel *ioc, QCryptoTLSCreds *tlscreds,QIOChannel *sioc =NULL ; *info =NULL ; - result =nbd_start_negotiate (NULL , ioc, tlscreds, hostname, &sioc, true, + result =nbd_start_negotiate (ioc, tlscreds, hostname, &sioc, true,NULL , errp);if (tlscreds && sioc) { ioc = sioc;
Signature · 1 line
2.41.0
PATCH v3 3/4 io: check there are no qio_channel_yield() coroutines during ->finalize()
In reply to Stefan HajnocziCallers must clean up their coroutines before calling object_unref(OBJECT(ioc)) to prevent an fd handler leak. Add an assertion to check this.
This patch is preparation for the fd handler changes that follow.
--- io/channel.c | 4 ++++ 1 file changed, 4 insertions(+)
io/channel.c +4 -0
@@ -653,6 +653,10 @@ static void qio_channel_finalize(Object *obj) {QIOChannel *ioc =QIO_CHANNEL (obj); +/* Must not have coroutines in qio_channel_yield() */ +assert (!ioc->read_coroutine); +assert (!ioc->write_coroutine); +g_free (ioc->name);#ifdef _WIN32
Signature · 1 line
2.41.0
PATCH v3 4/4 io: follow coroutine AioContext in qio_channel_yield()
In reply to Stefan HajnocziThe ongoing QEMU multi-queue block layer effort makes it possible for multiple threads to process I/O in parallel. The nbd block driver is not compatible with the multi-queue block layer yet because QIOChannel cannot be used easily from coroutines running in multiple threads. This series changes the QIOChannel API to make that possible.
In the current API, calling qio_channel_attach_aio_context() sets the AioContext where qio_channel_yield() installs an fd handler prior to yielding:
qio_channel_attach_aio_context(ioc, my_ctx); ... qio_channel_yield(ioc); // my_ctx is used here ... qio_channel_detach_aio_context(ioc);
This API design has limitations: reading and writing must be done in the same AioContext and moving between AioContexts involves a cumbersome sequence of API calls that is not suitable for doing on a per-request basis.
There is no fundamental reason why a QIOChannel needs to run within the same AioContext every time qio_channel_yield() is called. QIOChannel only uses the AioContext while inside qio_channel_yield(). The rest of the time, QIOChannel is independent of any AioContext.
In the new API, qio_channel_yield() queries the AioContext from the current coroutine using qemu_coroutine_get_aio_context(). There is no need to explicitly attach/detach AioContexts anymore and qio_channel_attach_aio_context() and qio_channel_detach_aio_context() are gone. One coroutine can read from the QIOChannel while another coroutine writes from a different AioContext.
This API change allows the nbd block driver to use QIOChannel from any thread. It's important to keep in mind that the block driver already synchronizes QIOChannel access and ensures that two coroutines never read simultaneously or write simultaneously.
This patch updates all users of qio_channel_attach_aio_context() to the new API. Most conversions are simple, but vhost-user-server requires a new qemu_coroutine_yield() call to quiesce the vu_client_trip() coroutine when not attached to any AioContext.
While the API is has become simpler, there is one wart: QIOChannel has a special case for the iohandler AioContext (used for handlers that must not run in nested event loops). I didn't find an elegant way preserve that behavior, so I added a new API called qio_channel_set_follow_coroutine_ctx(ioc, true|false) for opting in to the new AioContext model. By default QIOChannel uses the iohandler AioHandler. Code that formerly called qio_channel_attach_aio_context() now calls qio_channel_set_follow_coroutine_ctx(ioc, true) once after the QIOChannel is created.
--- include/io/channel-util.h | 23 ++++++ include/io/channel.h | 69 ++++++++---------- include/qemu/vhost-user-server.h | 1 + block/nbd.c | 11 +-- io/channel-command.c | 10 ++- io/channel-file.c | 9 ++- io/channel-null.c | 3 +- io/channel-socket.c | 9 ++- io/channel-tls.c | 6 +- io/channel-util.c | 24 +++++++ io/channel.c | 120 ++++++++++++++++++++++--------- migration/channel-block.c | 3 +- nbd/server.c | 14 +--- scsi/qemu-pr-helper.c | 4 +- util/vhost-user-server.c | 27 +++++-- 15 files changed, 216 insertions(+), 117 deletions(-)
include/io/channel-util.h +23 -0
@@ -49,4 +49,27 @@QIOChannel *qio_channel_new_fd (int fd,Error **errp); +/** +* qio_channel_util_set_aio_fd_handler: +* @read_fd: the file descriptor for the read handler +* @read_ctx: the AioContext for the read handler +* @io_read: the read handler +* @write_fd: the file descriptor for the write handler +* @write_ctx: the AioContext for the write handler +* @io_write: the write handler +* @opaque: the opaque argument to the read and write handler +* +* Set the read and write handlers when @read_ctx and @write_ctx are non-NULL, +* respectively. To leave a handler in its current state, pass a NULL +* AioContext. To clear a handler, pass a non-NULL AioContext and a NULL +* handler. +*/ +void qio_channel_util_set_aio_fd_handler (int read_fd, +AioContext *read_ctx, +IOHandler *io_read, +int write_fd, +AioContext *write_ctx, +IOHandler *io_write, +void *opaque); +#endif /* QIO_CHANNEL_UTIL_H */
include/io/channel.h +30 -39
112 lines not on this page · raw message
include/qemu/vhost-user-server.h +1 -0
@@ -43,6 +43,7 @@ typedef struct {unsigned int in_flight;/* atomic */ /* Protected by ctx lock */ +bool in_qio_channel_yield;bool wait_idle;VuDev vu_dev;QIOChannel *ioc;/* The I/O channel with the client */
block/nbd.c +1 -10
@@ -352,7 +352,7 @@ int coroutine_fn nbd_co_do_establish_connection(BlockDriverState *bs, }qio_channel_set_blocking (s->ioc, false,NULL ); -qio_channel_attach_aio_context (s->ioc,bdrv_get_aio_context (bs)); +qio_channel_set_follow_coroutine_ctx (s->ioc, true);/* successfully connected */ WITH_QEMU_LOCK_GUARD(&s->requests_lock) {
@@ -397,7 +397,6 @@ static void coroutine_fn GRAPH_RDLOCK nbd_reconnect_attempt(BDRVNBDState *s)/* Finalize previous connection if any */ if (s->ioc) { -qio_channel_detach_aio_context (s->ioc);yank_unregister_function (BLOCKDEV_YANK_INSTANCE (s->bs->node_name), nbd_yank, s->bs);object_unref (OBJECT (s->ioc));
@@ -2089,10 +2088,6 @@ static void nbd_attach_aio_context(BlockDriverState *bs,* the reconnect_delay_timer cannot be active here. */ assert (!s->reconnect_delay_timer); - -if (s->ioc) { -qio_channel_attach_aio_context (s->ioc, new_context); - } }static void nbd_detach_aio_context (BlockDriverState *bs)
@@ -2101,10 +2096,6 @@ static void nbd_detach_aio_context(BlockDriverState *bs)assert (!s->open_timer);assert (!s->reconnect_delay_timer); - -if (s->ioc) { -qio_channel_detach_aio_context (s->ioc); - } }static BlockDriver bdrv_nbd = {
io/channel-command.c +7 -3
@@ -20,6 +20,7 @@#include "qemu/osdep.h" #include "io/channel-command.h" +#include "io/channel-util.h" #include "io/channel-watch.h" #include "qapi/error.h" #include "qemu/module.h"
@@ -331,14 +332,17 @@ static int qio_channel_command_close(QIOChannel *ioc,static void qio_channel_command_set_aio_fd_handler (QIOChannel *ioc, -AioContext *ctx, +AioContext *read_ctx,IOHandler *io_read, +AioContext *write_ctx,IOHandler *io_write,void *opaque) {QIOChannelCommand *cioc =QIO_CHANNEL_COMMAND (ioc); -aio_set_fd_handler (ctx, cioc->readfd, io_read,NULL ,NULL ,NULL , opaque); -aio_set_fd_handler (ctx, cioc->writefd,NULL , io_write,NULL ,NULL , opaque); + +qio_channel_util_set_aio_fd_handler (cioc->readfd, read_ctx, io_read, + cioc->writefd, write_ctx, io_write, + opaque); }
io/channel-file.c +7 -2
@@ -20,6 +20,7 @@#include "qemu/osdep.h" #include "io/channel-file.h" +#include "io/channel-util.h" #include "io/channel-watch.h" #include "qapi/error.h" #include "qemu/module.h"
@@ -192,13 +193,17 @@ static int qio_channel_file_close(QIOChannel *ioc,static void qio_channel_file_set_aio_fd_handler (QIOChannel *ioc, -AioContext *ctx, +AioContext *read_ctx,IOHandler *io_read, +AioContext *write_ctx,IOHandler *io_write,void *opaque) {QIOChannelFile *fioc =QIO_CHANNEL_FILE (ioc); -aio_set_fd_handler (ctx, fioc->fd, io_read, io_write,NULL ,NULL , opaque); + +qio_channel_util_set_aio_fd_handler (fioc->fd, read_ctx, io_read, + fioc->fd, write_ctx, io_write, + opaque); } static GSource *qio_channel_file_create_watch(QIOChannel *ioc,
io/channel-null.c +2 -1
@@ -128,8 +128,9 @@ qio_channel_null_close(QIOChannel *ioc,static void qio_channel_null_set_aio_fd_handler(QIOChannel *ioc G_GNUC_UNUSED, - AioContext *ctx G_GNUC_UNUSED, + AioContext *read_ctx G_GNUC_UNUSED, IOHandler *io_read G_GNUC_UNUSED, + AioContext *write_ctx G_GNUC_UNUSED, IOHandler *io_write G_GNUC_UNUSED, void *opaque G_GNUC_UNUSED) {
io/channel-socket.c +7 -2
@@ -22,6 +22,7 @@#include "qapi/qapi-visit-sockets.h" #include "qemu/module.h" #include "io/channel-socket.h" +#include "io/channel-util.h" #include "io/channel-watch.h" #include "trace.h" #include "qapi/clone-visitor.h"
@@ -893,13 +894,17 @@ qio_channel_socket_shutdown(QIOChannel *ioc, }static void qio_channel_socket_set_aio_fd_handler (QIOChannel *ioc, -AioContext *ctx, +AioContext *read_ctx,IOHandler *io_read, +AioContext *write_ctx,IOHandler *io_write,void *opaque) {QIOChannelSocket *sioc =QIO_CHANNEL_SOCKET (ioc); -aio_set_fd_handler (ctx, sioc->fd, io_read, io_write,NULL ,NULL , opaque); + +qio_channel_util_set_aio_fd_handler (sioc->fd, read_ctx, io_read, + sioc->fd, write_ctx, io_write, + opaque); } static GSource *qio_channel_socket_create_watch(QIOChannel *ioc,
io/channel-tls.c +4 -2
@@ -388,14 +388,16 @@ static int qio_channel_tls_close(QIOChannel *ioc, }static void qio_channel_tls_set_aio_fd_handler (QIOChannel *ioc, -AioContext *ctx, +AioContext *read_ctx,IOHandler *io_read, +AioContext *write_ctx,IOHandler *io_write,void *opaque) {QIOChannelTLS *tioc =QIO_CHANNEL_TLS (ioc); -qio_channel_set_aio_fd_handler (tioc->master, ctx, io_read, io_write, opaque); +qio_channel_set_aio_fd_handler (tioc->master, read_ctx, io_read, + write_ctx, io_write, opaque); }typedef struct QIOChannelTLSSource QIOChannelTLSSource ;
io/channel-util.c +24 -0
@@ -36,3 +36,27 @@ QIOChannel *qio_channel_new_fd(int fd, }return ioc; } + + +void qio_channel_util_set_aio_fd_handler (int read_fd, +AioContext *read_ctx, +IOHandler *io_read, +int write_fd, +AioContext *write_ctx, +IOHandler *io_write, +void *opaque) +{ +if (read_fd == write_fd && read_ctx == write_ctx) { +aio_set_fd_handler (read_ctx, read_fd, io_read, io_write, +NULL ,NULL , opaque); + }else { +if (read_ctx) { +aio_set_fd_handler (read_ctx, read_fd, io_read,NULL , +NULL ,NULL , opaque); + } +if (write_ctx) { +aio_set_fd_handler (write_ctx, write_fd,NULL , io_write, +NULL ,NULL , opaque); + } + } +}
io/channel.c +86 -34
173 lines not on this page · raw message
migration/channel-block.c +2 -1
@@ -158,8 +158,9 @@ qio_channel_block_close(QIOChannel *ioc,static void qio_channel_block_set_aio_fd_handler (QIOChannel *ioc, -AioContext *ctx, +AioContext *read_ctx,IOHandler *io_read, +AioContext *write_ctx,IOHandler *io_write,void *opaque) {
nbd/server.c +1 -13
@@ -1333,6 +1333,7 @@ static coroutine_fn int nbd_negotiate(NBDClient *client, Error **errp)*/ qio_channel_set_blocking (client->ioc, false,NULL ); +qio_channel_set_follow_coroutine_ctx (client->ioc, true);trace_nbd_negotiate_begin ();memcpy (buf,"NBDMAGIC" ,8 );
@@ -1352,11 +1353,6 @@ static coroutine_fn int nbd_negotiate(NBDClient *client, Error **errp)return ret; } -/* Attach the channel to the same AioContext as the export */ -if (client->exp && client->exp->common.ctx) { -qio_channel_attach_aio_context (client->ioc, client->exp->common.ctx); - } -assert (!client->optlen);trace_nbd_negotiate_success ();
@@ -1465,7 +1461,6 @@ void nbd_client_put(NBDClient *client)*/ assert (client->closing); -qio_channel_detach_aio_context (client->ioc);object_unref (OBJECT (client->sioc));object_unref (OBJECT (client->ioc));if (client->tlscreds) {
@@ -1544,8 +1539,6 @@ static void blk_aio_attached(AioContext *ctx, void *opaque) exp->common.ctx = ctx; QTAILQ_FOREACH(client, &exp->clients, next) { -qio_channel_attach_aio_context (client->ioc, ctx); -assert (client->nb_requests ==0 );assert (client->recv_coroutine ==NULL );assert (client->send_coroutine ==NULL );
@@ -1555,14 +1548,9 @@ static void blk_aio_attached(AioContext *ctx, void *opaque)static void blk_aio_detach (void *opaque) {NBDExport *exp = opaque; -NBDClient *client;trace_nbd_blk_aio_detach (exp->name, exp->common.ctx); - QTAILQ_FOREACH(client, &exp->clients, next) { -qio_channel_detach_aio_context (client->ioc); - } - exp->common.ctx =NULL ; }
scsi/qemu-pr-helper.c +1 -3
@@ -735,8 +735,7 @@ static void coroutine_fn prh_co_entry(void *opaque)qio_channel_set_blocking (QIO_CHANNEL (client->ioc), false,NULL ); -qio_channel_attach_aio_context (QIO_CHANNEL (client->ioc), -qemu_get_aio_context ()); +qio_channel_set_follow_coroutine_ctx (QIO_CHANNEL (client->ioc), true);/* A very simple negotiation for future extensibility. No features * are defined so write 0.
@@ -796,7 +795,6 @@ static void coroutine_fn prh_co_entry(void *opaque) } out: -qio_channel_detach_aio_context (QIO_CHANNEL (client->ioc));object_unref (OBJECT (client->ioc));g_free (client); }
util/vhost-user-server.c +20 -7
@@ -127,7 +127,14 @@ vu_message_read(VuDev *vu_dev, int conn_fd, VhostUserMsg *vmsg)if (rc <0 ) {if (rc ==QIO_CHANNEL_ERR_BLOCK ) {assert (local_err ==NULL ); -qio_channel_yield (ioc,G_IO_IN ); +if (server->ctx) { + server->in_qio_channel_yield = true; +qio_channel_yield (ioc,G_IO_IN ); + server->in_qio_channel_yield = false; + }else { +/* Wait until attached to an AioContext again */ +qemu_coroutine_yield (); + }continue ; }else {error_report_err (local_err);
@@ -278,7 +285,7 @@ set_watch(VuDev *vu_dev, int fd, int vu_evt, vu_fd_watch->fd = fd; vu_fd_watch->cb = cb;qemu_socket_set_nonblock (fd); -aio_set_fd_handler (server->ioc->ctx, fd, kick_handler, +aio_set_fd_handler (server->ctx, fd, kick_handler,NULL ,NULL ,NULL , vu_fd_watch); vu_fd_watch->vu_dev = vu_dev; vu_fd_watch->pvt = pvt;
@@ -299,7 +306,7 @@ static void remove_watch(VuDev *vu_dev, int fd)if (!vu_fd_watch) {return ; } -aio_set_fd_handler (server->ioc->ctx, fd,NULL ,NULL ,NULL ,NULL ,NULL ); +aio_set_fd_handler (server->ctx, fd,NULL ,NULL ,NULL ,NULL ,NULL );QTAILQ_REMOVE (&server->vu_fd_watches, vu_fd_watch, next);g_free (vu_fd_watch);
@@ -344,6 +351,8 @@ static void vu_accept(QIONetListener *listener, QIOChannelSocket *sioc,/* TODO vu_message_write() spins if non-blocking! */ qio_channel_set_blocking (server->ioc, false,NULL ); +qio_channel_set_follow_coroutine_ctx (server->ioc, true); + server->co_trip =qemu_coroutine_create (vu_client_trip, server);aio_context_acquire (server->ctx);
@@ -399,13 +408,12 @@ void vhost_user_server_attach_aio_context(VuServer *server, AioContext *ctx)return ; } -qio_channel_attach_aio_context (server->ioc, ctx); - QTAILQ_FOREACH(vu_fd_watch, &server->vu_fd_watches, next) {aio_set_fd_handler (ctx, vu_fd_watch->fd, kick_handler,NULL ,NULL ,NULL , vu_fd_watch); } +assert (!server->in_qio_channel_yield);aio_co_schedule (ctx, server->co_trip); }
@@ -419,11 +427,16 @@ void vhost_user_server_detach_aio_context(VuServer *server)aio_set_fd_handler (server->ctx, vu_fd_watch->fd,NULL ,NULL ,NULL ,NULL , vu_fd_watch); } - -qio_channel_detach_aio_context (server->ioc); } server->ctx =NULL ; + +if (server->ioc) { +if (server->in_qio_channel_yield) { +/* Stop receiving the next vhost-user message */ +qio_channel_wake_read (server->ioc); + } + } } bool vhost_user_server_start(VuServer *server,
Signature · 1 line
2.41.0
Quoted 6 lines · Stefan Hajnoczi
The ongoing QEMU multi-queue block layer effort makes it possible for multiple threads to process I/O in parallel. The nbd block driver is not compatible with the multi-queue block layer yet because QIOChannel cannot be used easily from coroutines running in multiple threads. This series changes the QIOChannel API to make that possible.
...
Quoted 33 lines
This API change allows the nbd block driver to use QIOChannel from any thread. It's important to keep in mind that the block driver already synchronizes QIOChannel access and ensures that two coroutines never read simultaneously or write simultaneously.
This patch updates all users of
qio_channel_attach_aio_context()to the new API. Most conversions are simple, but vhost-user-server requires a newqemu_coroutine_yield()call to quiesce thevu_client_trip()coroutine when not attached to any AioContext.While the API is has become simpler, there is one wart: QIOChannel has a special case for the iohandler AioContext (used for handlers that must not run in nested event loops). I didn't find an elegant way preserve that behavior, so I added a new API called
qio_channel_set_follow_coroutine_ctx(ioc, true|false)for opting in to the new AioContext model. By default QIOChannel uses the iohandler AioHandler. Code that formerly calledqio_channel_attach_aio_context()now callsqio_channel_set_follow_coroutine_ctx(ioc, true)once after the QIOChannel is created.Signed-off-by Stefan Hajnoczi <[email protected]>Reviewed-by Eric Blake <[email protected]>--- include/io/channel-util.h | 23 ++++++ include/io/channel.h | 69 ++++++++---------- include/qemu/vhost-user-server.h | 1 + block/nbd.c | 11 +-- io/channel-command.c | 10 ++- io/channel-file.c | 9 ++- io/channel-null.c | 3 +- io/channel-socket.c | 9 ++-
Quoted 8 lines
io/channel-tls.c | 6 +- io/channel-util.c | 24 +++++++ io/channel.c | 120 ++++++++++++++++++++++--------- migration/channel-block.c | 3 +- nbd/server.c | 14 +--- scsi/qemu-pr-helper.c | 4 +- util/vhost-user-server.c | 27 +++++-- 15 files changed, 216 insertions(+), 117 deletions(-)
Looks like migration/rdma.c is also impacted:
../migration/rdma.c: In function ‘qio_channel_rdma_class_init’:../migration/rdma.c:4037:38: error: assignment to ‘void (*)(QIOChannel *, AioContext *, void (*)(void *), AioContext *, void (*)(void *), void *)’ from incompatible pointer type ‘void (*)(QIOChannel *, AioContext *, void (*)(void *), void (*)(void *), void *)’ [-Werror=incompatible-pointer-types]
4037 | ioc_klass->io_set_aio_fd_handler = qio_channel_rdma_set_aio_fd_handler;
| ^
I'm squashing this in:
migration/rdma.c +13 -12
@@ -3103,22 +3103,23 @@ static GSource *qio_channel_rdma_create_watch(QIOChannel *ioc, }static void qio_channel_rdma_set_aio_fd_handler (QIOChannel *ioc, -AioContext *ctx, -IOHandler *io_read, -IOHandler *io_write, -void *opaque) +AioContext *read_ctx, +IOHandler *io_read, +AioContext *write_ctx, +IOHandler *io_write, +void *opaque) {QIOChannelRDMA *rioc =QIO_CHANNEL_RDMA (ioc);if (io_read) { -aio_set_fd_handler (ctx, rioc->rdmain->recv_comp_channel->fd, io_read, - io_write,NULL ,NULL , opaque); -aio_set_fd_handler (ctx, rioc->rdmain->send_comp_channel->fd, io_read, - io_write,NULL ,NULL , opaque); +aio_set_fd_handler (read_ctx, rioc->rdmain->recv_comp_channel->fd, + io_read, io_write,NULL ,NULL , opaque); +aio_set_fd_handler (read_ctx, rioc->rdmain->send_comp_channel->fd, + io_read, io_write,NULL ,NULL , opaque); }else { -aio_set_fd_handler (ctx, rioc->rdmaout->recv_comp_channel->fd, io_read, - io_write,NULL ,NULL , opaque); -aio_set_fd_handler (ctx, rioc->rdmaout->send_comp_channel->fd, io_read, - io_write,NULL ,NULL , opaque); +aio_set_fd_handler (write_ctx, rioc->rdmaout->recv_comp_channel->fd, + io_read, io_write,NULL ,NULL , opaque); +aio_set_fd_handler (write_ctx, rioc->rdmaout->send_comp_channel->fd, + io_read, io_write,NULL ,NULL , opaque); } }
Signature · 3 lines
Eric Blake, Principal Software Engineer Red Hat, Inc. Virtualization: qemu.org | libguestfs.org
Quoted 88 lines · Eric Blake
Quoted 6 lines · Stefan Hajnoczi
The ongoing QEMU multi-queue block layer effort makes it possible for multiple threads to process I/O in parallel. The nbd block driver is not compatible with the multi-queue block layer yet because QIOChannel cannot be used easily from coroutines running in multiple threads. This series changes the QIOChannel API to make that possible.
...
Quoted 33 lines
This API change allows the nbd block driver to use QIOChannel from any thread. It's important to keep in mind that the block driver already synchronizes QIOChannel access and ensures that two coroutines never read simultaneously or write simultaneously.
This patch updates all users of
qio_channel_attach_aio_context()to the new API. Most conversions are simple, but vhost-user-server requires a newqemu_coroutine_yield()call to quiesce thevu_client_trip()coroutine when not attached to any AioContext.While the API is has become simpler, there is one wart: QIOChannel has a special case for the iohandler AioContext (used for handlers that must not run in nested event loops). I didn't find an elegant way preserve that behavior, so I added a new API called
qio_channel_set_follow_coroutine_ctx(ioc, true|false)for opting in to the new AioContext model. By default QIOChannel uses the iohandler AioHandler. Code that formerly calledqio_channel_attach_aio_context()now callsqio_channel_set_follow_coroutine_ctx(ioc, true)once after the QIOChannel is created.Signed-off-by Stefan Hajnoczi <[email protected]>Reviewed-by Eric Blake <[email protected]>--- include/io/channel-util.h | 23 ++++++ include/io/channel.h | 69 ++++++++---------- include/qemu/vhost-user-server.h | 1 + block/nbd.c | 11 +-- io/channel-command.c | 10 ++- io/channel-file.c | 9 ++- io/channel-null.c | 3 +- io/channel-socket.c | 9 ++-Quoted 8 lines
io/channel-tls.c | 6 +- io/channel-util.c | 24 +++++++ io/channel.c | 120 ++++++++++++++++++++++--------- migration/channel-block.c | 3 +- nbd/server.c | 14 +--- scsi/qemu-pr-helper.c | 4 +- util/vhost-user-server.c | 27 +++++-- 15 files changed, 216 insertions(+), 117 deletions(-)Looks like
migration/rdma.cis also impacted:
../migration/rdma.c: In function ‘qio_channel_rdma_class_init’:../migration/rdma.c:4037:38: error: assignment to ‘void (*)(QIOChannel *, AioContext *, void (*)(void *), AioContext *, void (*)(void *), void *)’ from incompatible pointer type ‘void (*)(QIOChannel *, AioContext *, void (*)(void *), void (*)(void *), void *)’ [-Werror=incompatible-pointer-types]
4037 |ioc_klass->io_set_aio_fd_handler=qio_channel_rdma_set_aio_fd_handler;
| ^I'm squashing this in:
migration/rdma.c +9
-8@@ -3103,22 +3103,23 @@ static GSource *qio_channel_rdma_create_watch(QIOChannel *ioc, }static void qio_channel_rdma_set_aio_fd_handler (QIOChannel *ioc, -AioContext *ctx, -IOHandler *io_read, -IOHandler *io_write, -void *opaque) +AioContext *read_ctx, +IOHandler *io_read, +AioContext *write_ctx, +IOHandler *io_write, +void *opaque) {QIOChannelRDMA *rioc =QIO_CHANNEL_RDMA (ioc);if (io_read) { -aio_set_fd_handler (ctx, rioc->rdmain->recv_comp_channel->fd, io_read, - io_write,NULL ,NULL , opaque); -aio_set_fd_handler (ctx, rioc->rdmain->send_comp_channel->fd, io_read, - io_write,NULL ,NULL , opaque); +aio_set_fd_handler (read_ctx, rioc->rdmain->recv_comp_channel->fd, + io_read, io_write,NULL ,NULL , opaque); +aio_set_fd_handler (read_ctx, rioc->rdmain->send_comp_channel->fd, + io_read, io_write,NULL ,NULL , opaque); }else {
migration/rdma.c static GSource *qio_channel_rdma_create_watch(QIOChannel *ioc, · line 3124
Quoted 10 lines
migration/rdma.c +4
-4-aio_set_fd_handler (ctx, rioc->rdmaout->recv_comp_channel->fd, io_read, - io_write,NULL ,NULL , opaque); -aio_set_fd_handler (ctx, rioc->rdmaout->send_comp_channel->fd, io_read, - io_write,NULL ,NULL , opaque); +aio_set_fd_handler (write_ctx, rioc->rdmaout->recv_comp_channel->fd, + io_read, io_write,NULL ,NULL , opaque); +aio_set_fd_handler (write_ctx, rioc->rdmaout->send_comp_channel->fd, + io_read, io_write,NULL ,NULL , opaque); } }
Thank you!
Stefan
Quoted 7 lines · Stefan Hajnoczi
v3:
- Fix wrong copy-pasted function name in doc comment [Eric]
- Fix "match" -> "matches" in comment [Eric]
v2:
- Add Patch 1 & 2 to remove unused NBDaio_contextarguments and dead code [Fabiano]
- Remove attach/detach prototypes from "io/channel.h" [Daniel]
- Add utility function to set fd handlers [Daniel]
I'm queuing this version (and not v2) through my NBD tree.
Quoted 13 lines
The ongoing QEMU multi-queue block layer effort makes it possible for multiple threads to process I/O in parallel. The nbd block driver is not compatible with the multi-queue block layer yet because QIOChannel cannot be used easily from coroutines running in multiple threads. This series changes the QIOChannel API to make that possible.
Stefan Hajnoczi (4):
nbd: drop unusednbd_receive_negotiate()aio_contextargument
nbd: drop unusednbd_start_negotiate()aio_contextargument
io: check there are noqio_channel_yield()coroutines during->finalize()io: follow coroutine AioContext in
qio_channel_yield()
Signature · 3 lines
Eric Blake, Principal Software Engineer Red Hat, Inc. Virtualization: qemu.org | libguestfs.org