make netlayer re-entrant
v2 cover make netlayer re-entrant
From: Liu Ping Fan <[email protected]>
changes v1->v2
Because the NetQueue is exposed not just from qemu_send_queue_xx(), so separating locks -- use NetQueue(NetQueue->lock) to protect the queue, and use nc->send_lock for the access of nc->peer so we can protect the sender from the remover.
Question:
Use AioContexts for hub or the raw GSource
Liu Ping Fan (5):
net: spread hub on AioContexts
net: hub use lock to protect ports list
net: introduce lock to protect NetQueue
net: introduce lock to protect NetClientState's peer's access
net: make netclient re-entrant with refcnt
hw/qdev-properties-system.c | 15 +++++ include/block/aio.h | 1 + include/net/net.h | 12 ++++ include/net/queue.h | 14 +++++ main-loop.c | 5 ++ net/hub.c | 81 +++++++++++++++++++++++++++-- net/net.c | 122 ++++++++++++++++++++++++++++++++++++++++-- net/queue.c | 19 +++++-- net/slirp.c | 3 +- 9 files changed, 257 insertions(+), 15 deletions(-)
Signature · 1 line
1.7.4.4
PATCH v2 1/5 net: spread hub on AioContexts
From: Liu Ping Fan <[email protected]>
Forward packet to other hub ports by their AioContext.
--- hw/qdev-properties-system.c | 1 + include/block/aio.h | 1 + include/net/net.h | 5 +++++ include/net/queue.h | 14 ++++++++++++++ main-loop.c | 5 +++++ net/hub.c | 33 ++++++++++++++++++++++++++++++--- net/net.c | 1 + net/queue.c | 4 ++-- 8 files changed, 59 insertions(+), 5 deletions(-)
hw/qdev-properties-system.c +1 -0
@@ -307,6 +307,7 @@ static void set_vlan(Object *obj, Visitor *v, void *opaque, name, prop->info->name);return ; } + hubport->info->bind_ctx (hubport,qemu_get_aio_context ()); *ptr = hubport; }
include/block/aio.h +1 -0
@@ -229,6 +229,7 @@ bool qemu_aio_wait(void);void qemu_aio_set_event_notifier (EventNotifier *notifier,EventNotifierHandler *io_read,AioFlushEventNotifierHandler *io_flush); +AioContext *qemu_get_aio_context (void );#ifdef CONFIG_POSIX void qemu_aio_set_fd_handler(int fd,
include/net/net.h +5 -0
@@ -8,6 +8,9 @@#include "net/queue.h" #include "migration/vmstate.h" #include "qapi-types.h" +#include "qemu/thread.h" +#include "block/aio.h" +#define MAX_QUEUE_NUM 1024
@@ -44,6 +47,7 @@ typedef ssize_t (NetReceiveIOV)(NetClientState *, const struct iovec *, int);typedef void (NetCleanup ) (NetClientState *);typedef void (LinkStatusChanged )(NetClientState *);typedef void (NetClientDestructor )(NetClientState *); +typedef void (NetClientBindCtx )(NetClientState *,AioContext *); typedef struct NetClientInfo { NetClientOptionsKind type;
@@ -55,6 +59,7 @@ typedef struct NetClientInfo {NetCleanup *cleanup;LinkStatusChanged *link_status_changed;NetPoll *poll; +NetClientBindCtx *bind_ctx; } NetClientInfo;struct NetClientState {
include/net/queue.h +14 -0
@@ -38,6 +38,20 @@ NetQueue *qemu_new_net_queue(void *opaque);void qemu_del_net_queue (NetQueue *queue); +void qemu_net_queue_append (NetQueue *queue, +NetClientState *sender, +unsigned flags, +const uint8_t *buf, +size_t size, +NetPacketSent *sent_cb); + +void qemu_net_queue_append_iov (NetQueue *queue, +NetClientState *sender, +unsigned flags, +const struct iovec *iov, +int iovcnt, +NetPacketSent *sent_cb); + ssize_t qemu_net_queue_send(NetQueue *queue, NetClientState *sender, unsigned flags,
main-loop.c +5 -0
@@ -109,6 +109,11 @@ static int qemu_signal_init(void)static AioContext *qemu_aio_context; +AioContext *qemu_get_aio_context (void ) +{ +return qemu_aio_context; +} +void qemu_notify_event (void ) {if (!qemu_aio_context) {
net/hub.c +30 -3
@@ -31,6 +31,8 @@ typedef struct NetHubPort {QLIST_ENTRY (NetHubPort) next;NetHub *hub;int id; +EventNotifier e; +AioContext *ctx; } NetHubPort;struct NetHub {
@@ -52,11 +54,20 @@ static ssize_t net_hub_receive(NetHub *hub, NetHubPort *source_port,continue ; } -qemu_send_packet (&port->nc, buf, len); +qemu_net_queue_append (port->nc.peer->send_queue, &port->nc, +QEMU_NET_PACKET_FLAG_NONE , buf, len,NULL ); +event_notifier_set (&port->e); }return len; } +static void hub_port_deliver_packet (void *opaque) +{ +NetHubPort *port = (NetHubPort *)opaque; + +qemu_net_queue_flush (port->nc.peer->send_queue); +} +static ssize_t net_hub_receive_iov (NetHub *hub,NetHubPort *source_port,const struct iovec *iov,int iovcnt) {
@@ -68,7 +79,9 @@ static ssize_t net_hub_receive_iov(NetHub *hub, NetHubPort *source_port,continue ; } -qemu_sendv_packet (&port->nc, iov, iovcnt); +qemu_net_queue_append_iov (port->nc.peer->send_queue, &port->nc, +QEMU_NET_PACKET_FLAG_NONE , iov, iovcnt,NULL ); +event_notifier_set (&port->e); }return len; }
@@ -126,9 +139,22 @@ static void net_hub_port_cleanup(NetClientState *nc) {NetHubPort *port =DO_UPCAST (NetHubPort, nc, nc); +if (port->ctx) { +aio_set_fd_handler (port->ctx,event_notifier_get_fd (&port->e), +NULL ,NULL ,NULL ,NULL ); + }QLIST_REMOVE (port, next); } +static void net_hub_port_bind_ctx (NetClientState *nc,AioContext *ctx) +{ +NetHubPort *port =DO_UPCAST (NetHubPort, nc, nc); + + port->ctx = ctx; +aio_set_fd_handler (ctx,event_notifier_get_fd (&port->e), + hub_port_deliver_packet,NULL ,NULL , port); +} +static NetClientInfo net_hub_port_info = { .type =NET_CLIENT_OPTIONS_KIND_HUBPORT , .size =sizeof (NetHubPort),
@@ -136,6 +162,7 @@ static NetClientInfo net_hub_port_info = { .receive = net_hub_port_receive, .receive_iov = net_hub_port_receive_iov, .cleanup = net_hub_port_cleanup, + .bind_ctx = net_hub_port_bind_ctx, };static NetHubPort *net_hub_port_new (NetHub *hub,const char *name)
@@ -155,7 +182,7 @@ static NetHubPort *net_hub_port_new(NetHub *hub, const char *name) port =DO_UPCAST (NetHubPort, nc, nc); port->id = id; port->hub = hub; - +event_notifier_init (&port->e,0 );QLIST_INSERT_HEAD (&hub->ports, port, next);return port;
net/net.c +1 -0
@@ -781,6 +781,7 @@ static int net_client_init1(const void *object, int is_netdev, Error **errp) (opts->kind != NET_CLIENT_OPTIONS_KIND_NIC || !opts->nic->has_netdev)) { peer =net_hub_add_port (u.net->has_vlan ? u.net->vlan :0 ,NULL ); + peer->info->bind_ctx (peer,qemu_get_aio_context ()); }if (net_client_init_fun[opts->kind](opts, name, peer) <0 ) {
net/queue.c +2 -2
@@ -87,7 +87,7 @@ void qemu_del_net_queue(NetQueue *queue)g_free (queue); } -static void qemu_net_queue_append(NetQueue *queue, +void qemu_net_queue_append(NetQueue *queue, NetClientState *sender, unsigned flags, const uint8_t *buf,
@@ -110,7 +110,7 @@ static void qemu_net_queue_append(NetQueue *queue,QTAILQ_INSERT_TAIL (&queue->packets, packet, entry); } -static void qemu_net_queue_append_iov(NetQueue *queue, +void qemu_net_queue_append_iov(NetQueue *queue, NetClientState *sender, unsigned flags, const struct iovec *iov,
Signature · 1 line
1.7.4.4
Re: [Qemu-devel] [PATCH v2 1/5] net: spread hub on AioContexts
Il 07/03/2013 03:53, Liu Ping Fan ha scritto:
Quoted 125 lines
From: Liu Ping Fan <[email protected]>
Forward packet to other hub ports by their AioContext.
Signed-off-by Liu Ping Fan <[email protected]>--- hw/qdev-properties-system.c | 1 + include/block/aio.h | 1 + include/net/net.h | 5 +++++ include/net/queue.h | 14 ++++++++++++++ main-loop.c | 5 +++++ net/hub.c | 33 ++++++++++++++++++++++++++++++--- net/net.c | 1 + net/queue.c | 4 ++-- 8 files changed, 59 insertions(+), 5 deletions(-)hw/qdev-properties-system.c +1
-0@@ -307,6 +307,7 @@ static void set_vlan(Object *obj, Visitor *v, void *opaque, name, prop->info->name);return ; } + hubport->info->bind_ctx (hubport,qemu_get_aio_context ()); *ptr = hubport; }include/block/aio.h +1
-0@@ -229,6 +229,7 @@ bool qemu_aio_wait(void);void qemu_aio_set_event_notifier (EventNotifier *notifier,EventNotifierHandler *io_read,AioFlushEventNotifierHandler *io_flush); +AioContext *qemu_get_aio_context (void );#ifdef CONFIG_POSIX void qemu_aio_set_fd_handler(int fd,include/net/net.h +5
-0@@ -8,6 +8,9 @@#include "net/queue.h" #include "migration/vmstate.h" #include "qapi-types.h" +#include "qemu/thread.h" +#include "block/aio.h" +#define MAX_QUEUE_NUM 1024@@ -44,6 +47,7 @@ typedef ssize_t (NetReceiveIOV)(NetClientState *, const struct iovec *, int);typedef void (NetCleanup ) (NetClientState *);typedef void (LinkStatusChanged )(NetClientState *);typedef void (NetClientDestructor )(NetClientState *); +typedef void (NetClientBindCtx )(NetClientState *,AioContext *); typedef struct NetClientInfo { NetClientOptionsKind type;@@ -55,6 +59,7 @@ typedef struct NetClientInfo {NetCleanup *cleanup;LinkStatusChanged *link_status_changed;NetPoll *poll; +NetClientBindCtx *bind_ctx; } NetClientInfo;struct NetClientState {include/net/queue.h +14
-0@@ -38,6 +38,20 @@ NetQueue *qemu_new_net_queue(void *opaque);void qemu_del_net_queue (NetQueue *queue); +void qemu_net_queue_append (NetQueue *queue, +NetClientState *sender, +unsigned flags, +const uint8_t *buf, +size_t size, +NetPacketSent *sent_cb); + +void qemu_net_queue_append_iov (NetQueue *queue, +NetClientState *sender, +unsigned flags, +const struct iovec *iov, +int iovcnt, +NetPacketSent *sent_cb); + ssize_t qemu_net_queue_send(NetQueue *queue, NetClientState *sender, unsigned flags,main-loop.c +5
-0@@ -109,6 +109,11 @@ static int qemu_signal_init(void)static AioContext *qemu_aio_context; +AioContext *qemu_get_aio_context (void ) +{ +return qemu_aio_context; +} +void qemu_notify_event (void ) {if (!qemu_aio_context) {net/hub.c +2
-0@@ -31,6 +31,8 @@ typedef struct NetHubPort {QLIST_ENTRY (NetHubPort) next;NetHub *hub;int id; +EventNotifier e; +AioContext *ctx; } NetHubPort;struct NetHub {
net/hub.c static ssize_t net_hub_receive(NetHub *hub, NetHubPort *source_port, · line 59
Quoted 7 lines
net/hub.c +3
-1@@ -52,11 +54,20 @@ static ssize_t net_hub_receive(NetHub *hub, NetHubPort *source_port,continue ; } -qemu_send_packet (&port->nc, buf, len); +qemu_net_queue_append (port->nc.peer->send_queue, &port->nc, +QEMU_NET_PACKET_FLAG_NONE , buf, len,NULL ); +event_notifier_set (&port->e);
Why are the context and the EventNotifier a property of the port, rather than applicable to the NetClientState?
Please explain the design and where you want to go. We do not have a crystal ball :) and lack of explanation can only prevent people from looking at your patches.
The absolute best would be if every tiny step along the way provides a benefit. For example, the patches that introduced AioContext enabled asynchronous I/O on Windows. Without such an incentive, it is very difficult that patches progress beyond the RFC phase.
Paolo
Quoted 100 lines
net/hub.c +25
-2}return len; } +static void hub_port_deliver_packet (void *opaque) +{ +NetHubPort *port = (NetHubPort *)opaque; + +qemu_net_queue_flush (port->nc.peer->send_queue); +} +static ssize_t net_hub_receive_iov (NetHub *hub,NetHubPort *source_port,const struct iovec *iov,int iovcnt) {@@ -68,7 +79,9 @@ static ssize_t net_hub_receive_iov(NetHub *hub, NetHubPort *source_port,continue ; } -qemu_sendv_packet (&port->nc, iov, iovcnt); +qemu_net_queue_append_iov (port->nc.peer->send_queue, &port->nc, +QEMU_NET_PACKET_FLAG_NONE , iov, iovcnt,NULL ); +event_notifier_set (&port->e); }return len; }@@ -126,9 +139,22 @@ static void net_hub_port_cleanup(NetClientState *nc) {NetHubPort *port =DO_UPCAST (NetHubPort, nc, nc); +if (port->ctx) { +aio_set_fd_handler (port->ctx,event_notifier_get_fd (&port->e), +NULL ,NULL ,NULL ,NULL ); + }QLIST_REMOVE (port, next); } +static void net_hub_port_bind_ctx (NetClientState *nc,AioContext *ctx) +{ +NetHubPort *port =DO_UPCAST (NetHubPort, nc, nc); + + port->ctx = ctx; +aio_set_fd_handler (ctx,event_notifier_get_fd (&port->e), + hub_port_deliver_packet,NULL ,NULL , port); +} +static NetClientInfo net_hub_port_info = { .type =NET_CLIENT_OPTIONS_KIND_HUBPORT , .size =sizeof (NetHubPort),@@ -136,6 +162,7 @@ static NetClientInfo net_hub_port_info = { .receive = net_hub_port_receive, .receive_iov = net_hub_port_receive_iov, .cleanup = net_hub_port_cleanup, + .bind_ctx = net_hub_port_bind_ctx, };static NetHubPort *net_hub_port_new (NetHub *hub,const char *name)@@ -155,7 +182,7 @@ static NetHubPort *net_hub_port_new(NetHub *hub, const char *name) port =DO_UPCAST (NetHubPort, nc, nc); port->id = id; port->hub = hub; - +event_notifier_init (&port->e,0 );QLIST_INSERT_HEAD (&hub->ports, port, next);return port;net/net.c +1
-0@@ -781,6 +781,7 @@ static int net_client_init1(const void *object, int is_netdev, Error **errp) (opts->kind != NET_CLIENT_OPTIONS_KIND_NIC || !opts->nic->has_netdev)) { peer =net_hub_add_port (u.net->has_vlan ? u.net->vlan :0 ,NULL ); + peer->info->bind_ctx (peer,qemu_get_aio_context ()); }if (net_client_init_fun[opts->kind](opts, name, peer) <0 ) {net/queue.c +2
-2@@ -87,7 +87,7 @@ void qemu_del_net_queue(NetQueue *queue)g_free (queue); } -static void qemu_net_queue_append(NetQueue *queue, +void qemu_net_queue_append(NetQueue *queue, NetClientState *sender, unsigned flags, const uint8_t *buf,@@ -110,7 +110,7 @@ static void qemu_net_queue_append(NetQueue *queue,QTAILQ_INSERT_TAIL (&queue->packets, packet, entry); } -static void qemu_net_queue_append_iov(NetQueue *queue, +void qemu_net_queue_append_iov(NetQueue *queue, NetClientState *sender, unsigned flags, const struct iovec *iov,
Re: [Qemu-devel] [PATCH v2 1/5] net: spread hub on AioContexts
Quoted 129 lines · Paolo Bonzini
Il 07/03/2013 03:53, Liu Ping Fan ha scritto:
Quoted 128 lines
From: Liu Ping Fan <[email protected]>
Forward packet to other hub ports by their AioContext.
Signed-off-by Liu Ping Fan <[email protected]>--- hw/qdev-properties-system.c | 1 + include/block/aio.h | 1 + include/net/net.h | 5 +++++ include/net/queue.h | 14 ++++++++++++++ main-loop.c | 5 +++++ net/hub.c | 33 ++++++++++++++++++++++++++++++--- net/net.c | 1 + net/queue.c | 4 ++-- 8 files changed, 59 insertions(+), 5 deletions(-)hw/qdev-properties-system.c +1
-0@@ -307,6 +307,7 @@ static void set_vlan(Object *obj, Visitor *v, void *opaque, name, prop->info->name);return ; } + hubport->info->bind_ctx (hubport,qemu_get_aio_context ()); *ptr = hubport; }include/block/aio.h +1
-0@@ -229,6 +229,7 @@ bool qemu_aio_wait(void);void qemu_aio_set_event_notifier (EventNotifier *notifier,EventNotifierHandler *io_read,AioFlushEventNotifierHandler *io_flush); +AioContext *qemu_get_aio_context (void );#ifdef CONFIG_POSIX void qemu_aio_set_fd_handler(int fd,include/net/net.h +5
-0@@ -8,6 +8,9 @@#include "net/queue.h" #include "migration/vmstate.h" #include "qapi-types.h" +#include "qemu/thread.h" +#include "block/aio.h" +#define MAX_QUEUE_NUM 1024@@ -44,6 +47,7 @@ typedef ssize_t (NetReceiveIOV)(NetClientState *, const struct iovec *, int);typedef void (NetCleanup ) (NetClientState *);typedef void (LinkStatusChanged )(NetClientState *);typedef void (NetClientDestructor )(NetClientState *); +typedef void (NetClientBindCtx )(NetClientState *,AioContext *); typedef struct NetClientInfo { NetClientOptionsKind type;@@ -55,6 +59,7 @@ typedef struct NetClientInfo {NetCleanup *cleanup;LinkStatusChanged *link_status_changed;NetPoll *poll; +NetClientBindCtx *bind_ctx; } NetClientInfo;struct NetClientState {include/net/queue.h +14
-0@@ -38,6 +38,20 @@ NetQueue *qemu_new_net_queue(void *opaque);void qemu_del_net_queue (NetQueue *queue); +void qemu_net_queue_append (NetQueue *queue, +NetClientState *sender, +unsigned flags, +const uint8_t *buf, +size_t size, +NetPacketSent *sent_cb); + +void qemu_net_queue_append_iov (NetQueue *queue, +NetClientState *sender, +unsigned flags, +const struct iovec *iov, +int iovcnt, +NetPacketSent *sent_cb); + ssize_t qemu_net_queue_send(NetQueue *queue, NetClientState *sender, unsigned flags,main-loop.c +5
-0@@ -109,6 +109,11 @@ static int qemu_signal_init(void)static AioContext *qemu_aio_context; +AioContext *qemu_get_aio_context (void ) +{ +return qemu_aio_context; +} +void qemu_notify_event (void ) {if (!qemu_aio_context) {net/hub.c +2
-0@@ -31,6 +31,8 @@ typedef struct NetHubPort {QLIST_ENTRY (NetHubPort) next;NetHub *hub;int id; +EventNotifier e; +AioContext *ctx; } NetHubPort;struct NetHub {@@ -52,11 +54,20 @@ static ssize_t net_hub_receive(NetHub *hub, NetHubPort *source_port,continue ; }
Quoted 8 lines
net/hub.c static ssize_t net_hub_receive(NetHub *hub, NetHubPort *source_port, · line 59
Quoted 4 lines
net/hub.c +3
-1-qemu_send_packet (&port->nc, buf, len); +qemu_net_queue_append (port->nc.peer->send_queue, &port->nc, +QEMU_NET_PACKET_FLAG_NONE , buf, len,NULL ); +event_notifier_set (&port->e);Why are the context and the EventNotifier a property of the port, rather than applicable to the NetClientState?
Yes, embed context into NetClientState is more reasonable, but as for EventNotifier, considering about if we port tap onto context, the tap do not have EventNotifier.
Quoted 4 lines
Please explain the design and where you want to go. We do not have a crystal ball :) and lack of explanation can only prevent people from looking at your patches.
Thanks for your advice. I have documented(will continue to update) the whole motivation and plan on
http://wiki.qemu.org/Features/network_reentrant#benefit_of_network_layer_re-entrant
For this patch, hub model is one of the obstacle to make network layer multi-thread.
Regards,
Pingfan
Quoted 108 lines
The absolute best would be if every tiny step along the way provides a benefit. For example, the patches that introduced AioContext enabled asynchronous I/O on Windows. Without such an incentive, it is very difficult that patches progress beyond the RFC phase.
Paolo
Quoted 100 lines
net/hub.c +25
-2}return len; } +static void hub_port_deliver_packet (void *opaque) +{ +NetHubPort *port = (NetHubPort *)opaque; + +qemu_net_queue_flush (port->nc.peer->send_queue); +} +static ssize_t net_hub_receive_iov (NetHub *hub,NetHubPort *source_port,const struct iovec *iov,int iovcnt) {@@ -68,7 +79,9 @@ static ssize_t net_hub_receive_iov(NetHub *hub, NetHubPort *source_port,continue ; } -qemu_sendv_packet (&port->nc, iov, iovcnt); +qemu_net_queue_append_iov (port->nc.peer->send_queue, &port->nc, +QEMU_NET_PACKET_FLAG_NONE , iov, iovcnt,NULL ); +event_notifier_set (&port->e); }return len; }@@ -126,9 +139,22 @@ static void net_hub_port_cleanup(NetClientState *nc) {NetHubPort *port =DO_UPCAST (NetHubPort, nc, nc); +if (port->ctx) { +aio_set_fd_handler (port->ctx,event_notifier_get_fd (&port->e), +NULL ,NULL ,NULL ,NULL ); + }QLIST_REMOVE (port, next); } +static void net_hub_port_bind_ctx (NetClientState *nc,AioContext *ctx) +{ +NetHubPort *port =DO_UPCAST (NetHubPort, nc, nc); + + port->ctx = ctx; +aio_set_fd_handler (ctx,event_notifier_get_fd (&port->e), + hub_port_deliver_packet,NULL ,NULL , port); +} +static NetClientInfo net_hub_port_info = { .type =NET_CLIENT_OPTIONS_KIND_HUBPORT , .size =sizeof (NetHubPort),@@ -136,6 +162,7 @@ static NetClientInfo net_hub_port_info = { .receive = net_hub_port_receive, .receive_iov = net_hub_port_receive_iov, .cleanup = net_hub_port_cleanup, + .bind_ctx = net_hub_port_bind_ctx, };static NetHubPort *net_hub_port_new (NetHub *hub,const char *name)@@ -155,7 +182,7 @@ static NetHubPort *net_hub_port_new(NetHub *hub, const char *name) port =DO_UPCAST (NetHubPort, nc, nc); port->id = id; port->hub = hub; - +event_notifier_init (&port->e,0 );QLIST_INSERT_HEAD (&hub->ports, port, next);return port;net/net.c +1
-0@@ -781,6 +781,7 @@ static int net_client_init1(const void *object, int is_netdev, Error **errp) (opts->kind != NET_CLIENT_OPTIONS_KIND_NIC || !opts->nic->has_netdev)) { peer =net_hub_add_port (u.net->has_vlan ? u.net->vlan :0 ,NULL ); + peer->info->bind_ctx (peer,qemu_get_aio_context ()); }if (net_client_init_fun[opts->kind](opts, name, peer) <0 ) {net/queue.c +2
-2@@ -87,7 +87,7 @@ void qemu_del_net_queue(NetQueue *queue)g_free (queue); } -static void qemu_net_queue_append(NetQueue *queue, +void qemu_net_queue_append(NetQueue *queue, NetClientState *sender, unsigned flags, const uint8_t *buf,@@ -110,7 +110,7 @@ static void qemu_net_queue_append(NetQueue *queue,QTAILQ_INSERT_TAIL (&queue->packets, packet, entry); } -static void qemu_net_queue_append_iov(NetQueue *queue, +void qemu_net_queue_append_iov(NetQueue *queue, NetClientState *sender, unsigned flags, const struct iovec *iov,
Re: [Qemu-devel] [PATCH v2 1/5] net: spread hub on AioContexts
Il 13/03/2013 03:26, liu ping fan ha scritto:
Quoted 11 lines
Quoted 7 lines
Quoted 7 lines
Quoted 3 lines
net/hub.c static ssize_t net_hub_receive(NetHub *hub, NetHubPort *source_port, · line 59
Quoted 3 lines
net/hub.c +3
-0+qemu_net_queue_append (port->nc.peer->send_queue, &port->nc, +QEMU_NET_PACKET_FLAG_NONE , buf, len,NULL ); +event_notifier_set (&port->e);Why are the context and the EventNotifier a property of the port, rather than applicable to the NetClientState?
Yes, embed context into NetClientState is more reasonable, but as for EventNotifier, considering about if we port tap onto context, the tap do not have EventNotifier.
There doesn't even need to be an EventNotifier, instead you can pass the NetClientState's AioContext to the queue and use a bottom half on the AioContext.
Furthermore, the bottom half should be completely transparent. Callers can keep using qemu_net_queue_flush, qemu_net_queue_flush schedules the bottom half (perhaps, with an optimization, it only does that if the queue is not empty), the handler actually performs the flush.
Paolo
PATCH v2 2/5 net: hub use lock to protect ports list
In reply to Liu Ping FanFrom: Liu Ping Fan <[email protected]>
--- net/hub.c | 27 ++++++++++++++++++++++++++- 1 files changed, 26 insertions(+), 1 deletions(-)
net/hub.c +26 -1
@@ -39,6 +39,8 @@ struct NetHub {int id;QLIST_ENTRY (NetHub) next;int num_ports; +/* protect ports */ +QemuMutex lock; QLIST_HEAD(, NetHubPort) ports; };
@@ -49,6 +51,7 @@ static ssize_t net_hub_receive(NetHub *hub, NetHubPort *source_port, {NetHubPort *port; +qemu_mutex_lock (&hub->lock); QLIST_FOREACH(port, &hub->ports, next) {if (port == source_port) {continue ;
@@ -58,6 +61,7 @@ static ssize_t net_hub_receive(NetHub *hub, NetHubPort *source_port, QEMU_NET_PACKET_FLAG_NONE, buf, len, NULL);event_notifier_set (&port->e); } +qemu_mutex_unlock (&hub->lock);return len; }
@@ -74,6 +78,7 @@ static ssize_t net_hub_receive_iov(NetHub *hub, NetHubPort *source_port,NetHubPort *port;ssize_t len =iov_size (iov, iovcnt); +qemu_mutex_lock (&hub->lock); QLIST_FOREACH(port, &hub->ports, next) {if (port == source_port) {continue ;
@@ -83,6 +88,7 @@ static ssize_t net_hub_receive_iov(NetHub *hub, NetHubPort *source_port, QEMU_NET_PACKET_FLAG_NONE, iov, iovcnt, NULL);event_notifier_set (&port->e); } +qemu_mutex_unlock (&hub->lock);return len; }
@@ -93,6 +99,7 @@ static NetHub *net_hub_new(int id) hub =g_malloc (sizeof (*hub)); hub->id = id; hub->num_ports =0 ; +qemu_mutex_init (&hub->lock);QLIST_INIT (&hub->ports);QLIST_INSERT_HEAD (&hubs, hub, next);
@@ -106,16 +113,19 @@ static int net_hub_port_can_receive(NetClientState *nc)NetHubPort *src_port =DO_UPCAST (NetHubPort, nc, nc);NetHub *hub = src_port->hub; +qemu_mutex_lock (&hub->lock); QLIST_FOREACH(port, &hub->ports, next) {if (port == src_port) {continue ; }if (qemu_can_send_packet (&port->nc)) { +qemu_mutex_unlock (&hub->lock);return 1 ; } } +qemu_mutex_unlock (&hub->lock);return 0 ; }
@@ -183,7 +193,9 @@ static NetHubPort *net_hub_port_new(NetHub *hub, const char *name) port->id = id; port->hub = hub;event_notifier_init (&port->e,0 ); +qemu_mutex_lock (&hub->lock);QLIST_INSERT_HEAD (&hub->ports, port, next); +qemu_mutex_unlock (&hub->lock);return port; }
@@ -224,13 +236,16 @@ NetClientState *net_hub_find_client_by_name(int hub_id, const char *name) QLIST_FOREACH(hub, &hubs, next) {if (hub->id == hub_id) { +qemu_mutex_lock (&hub->lock); QLIST_FOREACH(port, &hub->ports, next) { peer = port->nc.peer;if (peer &&strcmp (peer->name, name) ==0 ) { +qemu_mutex_unlock (&hub->lock);return peer; } } +qemu_mutex_unlock (&hub->lock); } }return NULL ;
@@ -247,12 +262,15 @@ NetClientState *net_hub_port_find(int hub_id) QLIST_FOREACH(hub, &hubs, next) {if (hub->id == hub_id) { +qemu_mutex_lock (&hub->lock); QLIST_FOREACH(port, &hub->ports, next) { nc = port->nc.peer;if (!nc) { +qemu_mutex_unlock (&hub->lock);return &(port->nc); } } +qemu_mutex_unlock (&hub->lock);break ; } }
@@ -271,12 +289,14 @@ void net_hub_info(Monitor *mon) QLIST_FOREACH(hub, &hubs, next) {monitor_printf (mon,"hub %d\n" , hub->id); +qemu_mutex_lock (&hub->lock); QLIST_FOREACH(port, &hub->ports, next) {if (port->nc.peer) {monitor_printf (mon," \\ " );print_net_client (mon, port->nc.peer); } } +qemu_mutex_unlock (&hub->lock); } }
@@ -333,6 +353,7 @@ void net_hub_check_clients(void) QLIST_FOREACH(hub, &hubs, next) {int has_nic =0 , has_host_dev =0 ; +qemu_mutex_lock (&hub->lock); QLIST_FOREACH(port, &hub->ports, next) { peer = port->nc.peer;if (!peer) {
@@ -355,6 +376,7 @@ void net_hub_check_clients(void)break ; } } +qemu_mutex_unlock (&hub->lock);if (has_host_dev && !has_nic) {fprintf (stderr,"Warning: vlan %d with no nics\n" , hub->id); }
@@ -370,12 +392,15 @@ bool net_hub_flush(NetClientState *nc) {NetHubPort *port;NetHubPort *source_port =DO_UPCAST (NetHubPort, nc, nc); +NetHub *hub = source_port->hub;int ret =0 ; - QLIST_FOREACH(port, &source_port->hub->ports, next) { +qemu_mutex_lock (&hub->lock); + QLIST_FOREACH(port, &hub->ports, next) {if (port != source_port) { ret +=qemu_net_queue_flush (port->nc.send_queue); } } +qemu_mutex_unlock (&hub->lock);return ret ? true : false; }
Signature · 1 line
1.7.4.4
Re: [Qemu-devel] [PATCH v2 2/5] net: hub use lock to protect ports list
Il 07/03/2013 03:53, Liu Ping Fan ha scritto:
Quoted 6 lines
From: Liu Ping Fan <[email protected]>
Signed-off-by Liu Ping Fan <[email protected]>--- net/hub.c | 27 ++++++++++++++++++++++++++- 1 files changed, 26 insertions(+), 1 deletions(-)
You are introducing three locks. Please document for each one how can concurrent access to the lock happen, and also the lock hierarchy.
Paolo
PATCH v2 3/5 net: introduce lock to protect NetQueue
In reply to Liu Ping FanFrom: Liu Ping Fan <[email protected]>
--- net/queue.c | 11 +++++++++++ 1 files changed, 11 insertions(+), 0 deletions(-)
net/queue.c +11 -0
@@ -53,6 +53,7 @@ struct NetQueue {uint32_t nq_maxlen;uint32_t nq_count; +QemuMutex lock; QTAILQ_HEAD(packets, NetPacket) packets; unsigned delivering : 1;
@@ -68,6 +69,7 @@ NetQueue *qemu_new_net_queue(void *opaque) queue->nq_maxlen =10000 ; queue->nq_count =0 ; +qemu_mutex_init (&queue->lock);QTAILQ_INIT (&queue->packets); queue->delivering =0 ;
@@ -107,7 +109,9 @@ void qemu_net_queue_append(NetQueue *queue,memcpy (packet->data, buf, size); queue->nq_count++; +qemu_mutex_lock (&queue->lock);QTAILQ_INSERT_TAIL (&queue->packets, packet, entry); +qemu_mutex_unlock (&queue->lock); } void qemu_net_queue_append_iov(NetQueue *queue,
@@ -142,7 +146,9 @@ void qemu_net_queue_append_iov(NetQueue *queue, } queue->nq_count++; +qemu_mutex_lock (&queue->lock);QTAILQ_INSERT_TAIL (&queue->packets, packet, entry); +qemu_mutex_unlock (&queue->lock); } static ssize_t qemu_net_queue_deliver(NetQueue *queue,
@@ -229,6 +235,7 @@ void qemu_net_queue_purge(NetQueue *queue, NetClientState *from) {NetPacket *packet, *next; +qemu_mutex_lock (&queue->lock); QTAILQ_FOREACH_SAFE(packet, &queue->packets, entry, next) {if (packet->sender == from) {QTAILQ_REMOVE (&queue->packets, packet, entry);
@@ -236,10 +243,12 @@ void qemu_net_queue_purge(NetQueue *queue, NetClientState *from)g_free (packet); } } +qemu_mutex_unlock (&queue->lock); }bool qemu_net_queue_flush (NetQueue *queue) { +qemu_mutex_lock (&queue->lock);while (!QTAILQ_EMPTY (&queue->packets)) {NetPacket *packet;int ret;
@@ -256,6 +265,7 @@ bool qemu_net_queue_flush(NetQueue *queue)if (ret ==0 ) { queue->nq_count++;QTAILQ_INSERT_HEAD (&queue->packets, packet, entry); +qemu_mutex_unlock (&queue->lock);return false; }
@@ -265,5 +275,6 @@ bool qemu_net_queue_flush(NetQueue *queue)g_free (packet); } +qemu_mutex_unlock (&queue->lock);return true; }
Signature · 1 line
1.7.4.4
PATCH v2 4/5 net: introduce lock to protect NetClientState's peer's access
In reply to Liu Ping FanFrom: Liu Ping Fan <[email protected]>
Introduce nc->send_lock, it shield off the race of nc->peer's reader and deleter. With it, after deleter finish, no new qemu_send_packet_xx() can reach ->send_queue, so no new reference(packet->sender) to nc will be appended to nc->peer->send_queue.
--- include/net/net.h | 4 +++ net/hub.c | 18 ++++++++++++ net/net.c | 77 ++++++++++++++++++++++++++++++++++++++++++++++++++--- net/queue.c | 4 +- 4 files changed, 97 insertions(+), 6 deletions(-)
include/net/net.h +4 -0
@@ -66,6 +66,8 @@ struct NetClientState {NetClientInfo *info;int link_down;QTAILQ_ENTRY (NetClientState) next; +/* protect the race access of peer between deleter and sender */ +QemuMutex send_lock;NetClientState *peer;NetQueue *send_queue;char *model;
@@ -78,6 +80,7 @@ struct NetClientState { typedef struct NICState { NetClientState *ncs; + NetClientState **pending_peer; NICConf *conf; void *opaque; bool peer_deleted;
@@ -105,6 +108,7 @@ NetClientState *qemu_find_vlan_client_by_name(Monitor *mon, int vlan_id, const char *client_str);typedef void (*qemu_nic_foreach )(NICState *nic,void *opaque);void qemu_foreach_nic (qemu_nic_foreach func,void *opaque); +int qemu_can_send_packet_nolock (NetClientState *sender);int qemu_can_send_packet (NetClientState *nc);ssize_t qemu_sendv_packet (NetClientState *nc,const struct iovec *iov,int iovcnt);
net/hub.c +18 -0
@@ -57,8 +57,14 @@ static ssize_t net_hub_receive(NetHub *hub, NetHubPort *source_port,continue ; } +qemu_mutex_lock (&port->nc.send_lock); +if (!port->nc.peer) { +qemu_mutex_unlock (&port->nc.send_lock); +continue ; + }qemu_net_queue_append (port->nc.peer->send_queue, &port->nc,QEMU_NET_PACKET_FLAG_NONE , buf, len,NULL ); +qemu_mutex_unlock (&port->nc.send_lock);event_notifier_set (&port->e); }qemu_mutex_unlock (&hub->lock);
@@ -69,7 +75,13 @@ static void hub_port_deliver_packet(void *opaque) {NetHubPort *port = (NetHubPort *)opaque; +qemu_mutex_lock (&port->nc.send_lock); +if (!port->nc.peer) { +qemu_mutex_unlock (&port->nc.send_lock); +return ; + }qemu_net_queue_flush (port->nc.peer->send_queue); +qemu_mutex_unlock (&port->nc.send_lock); } static ssize_t net_hub_receive_iov(NetHub *hub, NetHubPort *source_port,
@@ -84,8 +96,14 @@ static ssize_t net_hub_receive_iov(NetHub *hub, NetHubPort *source_port,continue ; } +qemu_mutex_lock (&port->nc.send_lock); +if (!port->nc.peer) { +qemu_mutex_unlock (&port->nc.send_lock); +continue ; + }qemu_net_queue_append_iov (port->nc.peer->send_queue, &port->nc,QEMU_NET_PACKET_FLAG_NONE , iov, iovcnt,NULL ); +qemu_mutex_unlock (&port->nc.send_lock);event_notifier_set (&port->e); }qemu_mutex_unlock (&hub->lock);
net/net.c +73 -4
169 lines not on this page · raw message
net/queue.c +2 -2
@@ -190,7 +190,7 @@ ssize_t qemu_net_queue_send(NetQueue *queue, {ssize_t ret; -if (queue->delivering || !qemu_can_send_packet (sender)) { +if (queue->delivering || !qemu_can_send_packet_nolock (sender)) {qemu_net_queue_append (queue, sender, flags, data, size, sent_cb);return 0 ; }
@@ -215,7 +215,7 @@ ssize_t qemu_net_queue_send_iov(NetQueue *queue, {ssize_t ret; -if (queue->delivering || !qemu_can_send_packet (sender)) { +if (queue->delivering || !qemu_can_send_packet_nolock (sender)) {qemu_net_queue_append_iov (queue, sender, flags, iov, iovcnt, sent_cb);return 0 ; }
Signature · 1 line
1.7.4.4
Re: [Qemu-devel] [PATCH v2 4/5] net: introduce lock to protect NetClientState's peer's access
Il 07/03/2013 03:53, Liu Ping Fan ha scritto:
Quoted 50 lines
From: Liu Ping Fan <[email protected]>
Introduce
nc->send_lock, it shield off the race of nc->peer's reader and deleter. With it, after deleter finish, no newqemu_send_packet_xx()can reach ->send_queue, so no newreference(packet->sender)to nc will be appended tonc->peer->send_queue.Signed-off-by Liu Ping Fan <[email protected]>--- include/net/net.h | 4 +++ net/hub.c | 18 ++++++++++++ net/net.c | 77 ++++++++++++++++++++++++++++++++++++++++++++++++++--- net/queue.c | 4 +- 4 files changed, 97 insertions(+), 6 deletions(-)include/net/net.h +4
-0@@ -66,6 +66,8 @@ struct NetClientState {NetClientInfo *info;int link_down;QTAILQ_ENTRY (NetClientState) next; +/* protect the race access of peer between deleter and sender */ +QemuMutex send_lock;NetClientState *peer;NetQueue *send_queue;char *model;@@ -78,6 +80,7 @@ struct NetClientState { typedef struct NICState { NetClientState *ncs; + NetClientState **pending_peer; NICConf *conf; void *opaque; bool peer_deleted;@@ -105,6 +108,7 @@ NetClientState *qemu_find_vlan_client_by_name(Monitor *mon, int vlan_id, const char *client_str);typedef void (*qemu_nic_foreach )(NICState *nic,void *opaque);void qemu_foreach_nic (qemu_nic_foreach func,void *opaque); +int qemu_can_send_packet_nolock (NetClientState *sender);int qemu_can_send_packet (NetClientState *nc);ssize_t qemu_sendv_packet (NetClientState *nc,const struct iovec *iov,int iovcnt);net/hub.c +0
-0@@ -57,8 +57,14 @@ static ssize_t net_hub_receive(NetHub *hub, NetHubPort *source_port,continue ;
net/hub.c static ssize_t net_hub_receive(NetHub *hub, NetHubPort *source_port, · line 67
Quoted 10 lines
net/hub.c +6
-0} +qemu_mutex_lock (&port->nc.send_lock); +if (!port->nc.peer) { +qemu_mutex_unlock (&port->nc.send_lock); +continue ; + }qemu_net_queue_append (port->nc.peer->send_queue, &port->nc,QEMU_NET_PACKET_FLAG_NONE , buf, len,NULL ); +qemu_mutex_unlock (&port->nc.send_lock);
Do you really need to lock everything? Can you just wrap the peer with a ref/unref, like
NetClientState *net_client_get_peer(NetClientState *nc)
{
NetClientState *peer;
qemu_mutex_lock(&nc->send_lock);
peer = nc->peer;
if (peer) {
net_client_ref(peer);
}
qemu_mutex_unlock(&nc->send_lock);
return peer;
}and then
- qemu_net_queue_append(port->nc.peer->send_queue, &port->nc,
+ peer = net_client_get_peer(&port->nc);
+ if (!peer) {
+ continue;
+ }
+ qemu_net_queue_append(peer->send_queue, &port->nc,
QEMU_NET_PACKET_FLAG_NONE, buf, len, NULL);+ net_client_unref(peer);
Paolo
Quoted 238 lines
net/hub.c +12
-0event_notifier_set (&port->e); }qemu_mutex_unlock (&hub->lock);@@ -69,7 +75,13 @@ static void hub_port_deliver_packet(void *opaque) {NetHubPort *port = (NetHubPort *)opaque; +qemu_mutex_lock (&port->nc.send_lock); +if (!port->nc.peer) { +qemu_mutex_unlock (&port->nc.send_lock); +return ; + }qemu_net_queue_flush (port->nc.peer->send_queue); +qemu_mutex_unlock (&port->nc.send_lock); } static ssize_t net_hub_receive_iov(NetHub *hub, NetHubPort *source_port,@@ -84,8 +96,14 @@ static ssize_t net_hub_receive_iov(NetHub *hub, NetHubPort *source_port,continue ; } +qemu_mutex_lock (&port->nc.send_lock); +if (!port->nc.peer) { +qemu_mutex_unlock (&port->nc.send_lock); +continue ; + }qemu_net_queue_append_iov (port->nc.peer->send_queue, &port->nc,QEMU_NET_PACKET_FLAG_NONE , iov, iovcnt,NULL ); +qemu_mutex_unlock (&port->nc.send_lock);event_notifier_set (&port->e); }qemu_mutex_unlock (&hub->lock);net/net.c +73
-4@@ -207,6 +207,7 @@ static void qemu_net_client_setup(NetClientState *nc, nc->peer = peer; peer->peer = nc; } +qemu_mutex_init (&nc->send_lock);QTAILQ_INSERT_TAIL (&net_clients, nc, next); nc->send_queue =qemu_new_net_queue (nc);@@ -246,6 +247,7 @@ NICState *qemu_new_nic(NetClientInfo *info, nic->ncs = (void *)nic + info->size; nic->conf = conf; nic->opaque = opaque; + nic->pending_peer =g_malloc0 (sizeof (NetClientState *) * queues);for (i =0 ; i < queues; i++) { qemu_net_client_setup(&nic->ncs[i], info, peers[i], model, name,@@ -304,6 +306,36 @@ static void qemu_free_net_client(NetClientState *nc) } } +/* elimate the reference and sync with exit of rx/tx action. +* And flush out peer's queue. +*/ +static void qemu_net_client_detach_flush (NetClientState *nc) +{ +NetClientState *peer; + +/* Fixme? Assume this function the only place to detach peer from @nc? +* Then reader and deleter are sequent. So here can we save the lock? +*/ +qemu_mutex_lock (&nc->send_lock); + peer = nc->peer; +qemu_mutex_unlock (&nc->send_lock); + +if (peer) { +/* exclude the race with tx to @nc */ +qemu_mutex_lock (&peer->send_lock); + peer->peer =NULL ; +qemu_mutex_unlock (&peer->send_lock); + } + +/* exclude the race with tx from @nc */ +qemu_mutex_lock (&nc->send_lock); + nc->peer =NULL ; +if (peer) { +qemu_net_queue_purge (peer->send_queue, nc); + } +qemu_mutex_unlock (&nc->send_lock); +} +void qemu_del_net_client (NetClientState *nc) {NetClientState *ncs[MAX_QUEUE_NUM ];@@ -334,7 +366,9 @@ void qemu_del_net_client(NetClientState *nc) }for (i =0 ; i < queues; i++) { +qemu_net_client_detach_flush (ncs[i]);qemu_cleanup_net_client (ncs[i]); + nic->pending_peer[i] = ncs[i]; }return ;@@ -343,6 +377,7 @@ void qemu_del_net_client(NetClientState *nc)assert (nc->info->type !=NET_CLIENT_OPTIONS_KIND_NIC );for (i =0 ; i < queues; i++) { +qemu_net_client_detach_flush (ncs[i]);qemu_cleanup_net_client (ncs[i]);qemu_free_net_client (ncs[i]); }@@ -355,17 +390,19 @@ void qemu_del_nic(NICState *nic)/* If this is a peer NIC and peer has already been deleted, free it now. */ if (nic->peer_deleted) {for (i =0 ; i < queues; i++) { -qemu_free_net_client (qemu_get_subqueue (nic, i)->peer); +qemu_free_net_client (nic->pending_peer[i]); } }for (i = queues -1 ; i >=0 ; i--) {NetClientState *nc =qemu_get_subqueue (nic, i); +qemu_net_client_detach_flush (nc);qemu_cleanup_net_client (nc);qemu_free_net_client (nc); } +g_free (nic->pending_peer);g_free (nic); }@@ -382,7 +419,7 @@ void qemu_foreach_nic(qemu_nic_foreach func, void *opaque) } } -int qemu_can_send_packet (NetClientState *sender) +int qemu_can_send_packet_nolock (NetClientState *sender) {if (!sender->peer) {return 1 ;@@ -397,6 +434,28 @@ int qemu_can_send_packet(NetClientState *sender)return 1 ; } +int qemu_can_send_packet (NetClientState *sender) +{ +int ret =1 ; + +qemu_mutex_lock (&sender->send_lock); +if (!sender->peer) { + goto unlock; + } + +if (sender->peer->receive_disabled) { + ret =0 ; + goto unlock; + }else if (sender->peer->info->can_receive && + !sender->peer->info->can_receive (sender->peer)) { + ret =0 ; + goto unlock; + } +unlock: +qemu_mutex_unlock (&sender->send_lock); +return ret; +} + ssize_t qemu_deliver_packet(NetClientState *sender, unsigned flags, const uint8_t *data,@@ -460,19 +519,24 @@ static ssize_t qemu_send_packet_async_with_flags(NetClientState *sender, NetPacketSent *sent_cb) {NetQueue *queue; +ssize_t sz;#ifdef DEBUG_NET printf ("qemu_send_packet_async:\n" );hex_dump (stdout, buf, size);#endif +qemu_mutex_lock (&sender->send_lock);if (sender->link_down || !sender->peer) { +qemu_mutex_unlock (&sender->send_lock);return size; } queue = sender->peer->send_queue; -return qemu_net_queue_send (queue, sender, flags, buf, size, sent_cb); + sz =qemu_net_queue_send (queue, sender, flags, buf, size, sent_cb); +qemu_mutex_unlock (&sender->send_lock); +return sz; } ssize_t qemu_send_packet_async(NetClientState *sender,@@ -540,16 +604,21 @@ ssize_t qemu_sendv_packet_async(NetClientState *sender, NetPacketSent *sent_cb) {NetQueue *queue; +ssize_t sz; +qemu_mutex_lock (&sender->send_lock);if (sender->link_down || !sender->peer) { +qemu_mutex_unlock (&sender->send_lock);return iov_size (iov, iovcnt); } queue = sender->peer->send_queue; -return qemu_net_queue_send_iov (queue, sender, + sz =qemu_net_queue_send_iov (queue, sender,QEMU_NET_PACKET_FLAG_NONE , iov, iovcnt, sent_cb); +qemu_mutex_unlock (&sender->send_lock); +return sz; }ssize_t net/queue.c +2
-2@@ -190,7 +190,7 @@ ssize_t qemu_net_queue_send(NetQueue *queue, {ssize_t ret; -if (queue->delivering || !qemu_can_send_packet (sender)) { +if (queue->delivering || !qemu_can_send_packet_nolock (sender)) {qemu_net_queue_append (queue, sender, flags, data, size, sent_cb);return 0 ; }@@ -215,7 +215,7 @@ ssize_t qemu_net_queue_send_iov(NetQueue *queue, {ssize_t ret; -if (queue->delivering || !qemu_can_send_packet (sender)) { +if (queue->delivering || !qemu_can_send_packet_nolock (sender)) {qemu_net_queue_append_iov (queue, sender, flags, iov, iovcnt, sent_cb);return 0 ; }
Re: [Qemu-devel] [PATCH v2 4/5] net: introduce lock to protect NetClientState's peer's access
Quoted 80 lines · Paolo Bonzini
Il 07/03/2013 03:53, Liu Ping Fan ha scritto:
Quoted 50 lines
From: Liu Ping Fan <[email protected]>
Introduce
nc->send_lock, it shield off the race of nc->peer's reader and deleter. With it, after deleter finish, no newqemu_send_packet_xx()can reach ->send_queue, so no newreference(packet->sender)to nc will be appended tonc->peer->send_queue.Signed-off-by Liu Ping Fan <[email protected]>--- include/net/net.h | 4 +++ net/hub.c | 18 ++++++++++++ net/net.c | 77 ++++++++++++++++++++++++++++++++++++++++++++++++++--- net/queue.c | 4 +- 4 files changed, 97 insertions(+), 6 deletions(-)include/net/net.h +4
-0@@ -66,6 +66,8 @@ struct NetClientState {NetClientInfo *info;int link_down;QTAILQ_ENTRY (NetClientState) next; +/* protect the race access of peer between deleter and sender */ +QemuMutex send_lock;NetClientState *peer;NetQueue *send_queue;char *model;@@ -78,6 +80,7 @@ struct NetClientState { typedef struct NICState { NetClientState *ncs; + NetClientState **pending_peer; NICConf *conf; void *opaque; bool peer_deleted;@@ -105,6 +108,7 @@ NetClientState *qemu_find_vlan_client_by_name(Monitor *mon, int vlan_id, const char *client_str);typedef void (*qemu_nic_foreach )(NICState *nic,void *opaque);void qemu_foreach_nic (qemu_nic_foreach func,void *opaque); +int qemu_can_send_packet_nolock (NetClientState *sender);int qemu_can_send_packet (NetClientState *nc);ssize_t qemu_sendv_packet (NetClientState *nc,const struct iovec *iov,int iovcnt);net/hub.c +0
-0@@ -57,8 +57,14 @@ static ssize_t net_hub_receive(NetHub *hub, NetHubPort *source_port,continue ;net/hub.c static ssize_t net_hub_receive(NetHub *hub, NetHubPort *source_port, · line 67
Quoted 10 lines
net/hub.c +6
-0} +qemu_mutex_lock (&port->nc.send_lock); +if (!port->nc.peer) { +qemu_mutex_unlock (&port->nc.send_lock); +continue ; + }qemu_net_queue_append (port->nc.peer->send_queue, &port->nc,QEMU_NET_PACKET_FLAG_NONE , buf, len,NULL ); +qemu_mutex_unlock (&port->nc.send_lock);Do you really need to lock everything? Can you just wrap the peer with a ref/unref, like
NetClientState *net_client_get_peer(NetClientState *nc)
{ NetClientState *peer; qemu_mutex_lock(&nc->send_lock); peer = nc->peer; if (peer) { net_client_ref(peer); } qemu_mutex_unlock(&nc->send_lock); return peer; }and then
Review
Quoted 8 lines
(quoted diff) +6
-1- qemu_net_queue_append(port->nc.peer->send_queue, &port->nc, + peer = net_client_get_peer(&port->nc); + if (!peer) { + continue; + } + qemu_net_queue_append(peer->send_queue, &port->nc, QEMU_NET_PACKET_FLAG_NONE, buf, len, NULL); + net_client_unref(peer);
Oh, seems that I do not explain very clearly in the commit log. The lock does not only protect against the reclaimer( and this can be achieved by refcnt as your codes), but also sync between remover and
sender. If the NetClientState being removed, the remover will be
like:
nc->peer = NULL;
----------> Here opens the gap for in-flight sender, and
refcnt can not work
flush out reference from its peer's send_queue;Thanks and regards,
Pingfan
Quoted 241 lines
Paolo
Quoted 238 lines
net/hub.c +12
-0event_notifier_set (&port->e); }qemu_mutex_unlock (&hub->lock);@@ -69,7 +75,13 @@ static void hub_port_deliver_packet(void *opaque) {NetHubPort *port = (NetHubPort *)opaque; +qemu_mutex_lock (&port->nc.send_lock); +if (!port->nc.peer) { +qemu_mutex_unlock (&port->nc.send_lock); +return ; + }qemu_net_queue_flush (port->nc.peer->send_queue); +qemu_mutex_unlock (&port->nc.send_lock); } static ssize_t net_hub_receive_iov(NetHub *hub, NetHubPort *source_port,@@ -84,8 +96,14 @@ static ssize_t net_hub_receive_iov(NetHub *hub, NetHubPort *source_port,continue ; } +qemu_mutex_lock (&port->nc.send_lock); +if (!port->nc.peer) { +qemu_mutex_unlock (&port->nc.send_lock); +continue ; + }qemu_net_queue_append_iov (port->nc.peer->send_queue, &port->nc,QEMU_NET_PACKET_FLAG_NONE , iov, iovcnt,NULL ); +qemu_mutex_unlock (&port->nc.send_lock);event_notifier_set (&port->e); }qemu_mutex_unlock (&hub->lock);net/net.c +73
-4@@ -207,6 +207,7 @@ static void qemu_net_client_setup(NetClientState *nc, nc->peer = peer; peer->peer = nc; } +qemu_mutex_init (&nc->send_lock);QTAILQ_INSERT_TAIL (&net_clients, nc, next); nc->send_queue =qemu_new_net_queue (nc);@@ -246,6 +247,7 @@ NICState *qemu_new_nic(NetClientInfo *info, nic->ncs = (void *)nic + info->size; nic->conf = conf; nic->opaque = opaque; + nic->pending_peer =g_malloc0 (sizeof (NetClientState *) * queues);for (i =0 ; i < queues; i++) { qemu_net_client_setup(&nic->ncs[i], info, peers[i], model, name,@@ -304,6 +306,36 @@ static void qemu_free_net_client(NetClientState *nc) } } +/* elimate the reference and sync with exit of rx/tx action. +* And flush out peer's queue. +*/ +static void qemu_net_client_detach_flush (NetClientState *nc) +{ +NetClientState *peer; + +/* Fixme? Assume this function the only place to detach peer from @nc? +* Then reader and deleter are sequent. So here can we save the lock? +*/ +qemu_mutex_lock (&nc->send_lock); + peer = nc->peer; +qemu_mutex_unlock (&nc->send_lock); + +if (peer) { +/* exclude the race with tx to @nc */ +qemu_mutex_lock (&peer->send_lock); + peer->peer =NULL ; +qemu_mutex_unlock (&peer->send_lock); + } + +/* exclude the race with tx from @nc */ +qemu_mutex_lock (&nc->send_lock); + nc->peer =NULL ; +if (peer) { +qemu_net_queue_purge (peer->send_queue, nc); + } +qemu_mutex_unlock (&nc->send_lock); +} +void qemu_del_net_client (NetClientState *nc) {NetClientState *ncs[MAX_QUEUE_NUM ];@@ -334,7 +366,9 @@ void qemu_del_net_client(NetClientState *nc) }for (i =0 ; i < queues; i++) { +qemu_net_client_detach_flush (ncs[i]);qemu_cleanup_net_client (ncs[i]); + nic->pending_peer[i] = ncs[i]; }return ;@@ -343,6 +377,7 @@ void qemu_del_net_client(NetClientState *nc)assert (nc->info->type !=NET_CLIENT_OPTIONS_KIND_NIC );for (i =0 ; i < queues; i++) { +qemu_net_client_detach_flush (ncs[i]);qemu_cleanup_net_client (ncs[i]);qemu_free_net_client (ncs[i]); }@@ -355,17 +390,19 @@ void qemu_del_nic(NICState *nic)/* If this is a peer NIC and peer has already been deleted, free it now. */ if (nic->peer_deleted) {for (i =0 ; i < queues; i++) { -qemu_free_net_client (qemu_get_subqueue (nic, i)->peer); +qemu_free_net_client (nic->pending_peer[i]); } }for (i = queues -1 ; i >=0 ; i--) {NetClientState *nc =qemu_get_subqueue (nic, i); +qemu_net_client_detach_flush (nc);qemu_cleanup_net_client (nc);qemu_free_net_client (nc); } +g_free (nic->pending_peer);g_free (nic); }@@ -382,7 +419,7 @@ void qemu_foreach_nic(qemu_nic_foreach func, void *opaque) } } -int qemu_can_send_packet (NetClientState *sender) +int qemu_can_send_packet_nolock (NetClientState *sender) {if (!sender->peer) {return 1 ;@@ -397,6 +434,28 @@ int qemu_can_send_packet(NetClientState *sender)return 1 ; } +int qemu_can_send_packet (NetClientState *sender) +{ +int ret =1 ; + +qemu_mutex_lock (&sender->send_lock); +if (!sender->peer) { + goto unlock; + } + +if (sender->peer->receive_disabled) { + ret =0 ; + goto unlock; + }else if (sender->peer->info->can_receive && + !sender->peer->info->can_receive (sender->peer)) { + ret =0 ; + goto unlock; + } +unlock: +qemu_mutex_unlock (&sender->send_lock); +return ret; +} + ssize_t qemu_deliver_packet(NetClientState *sender, unsigned flags, const uint8_t *data,@@ -460,19 +519,24 @@ static ssize_t qemu_send_packet_async_with_flags(NetClientState *sender, NetPacketSent *sent_cb) {NetQueue *queue; +ssize_t sz;#ifdef DEBUG_NET printf ("qemu_send_packet_async:\n" );hex_dump (stdout, buf, size);#endif +qemu_mutex_lock (&sender->send_lock);if (sender->link_down || !sender->peer) { +qemu_mutex_unlock (&sender->send_lock);return size; } queue = sender->peer->send_queue; -return qemu_net_queue_send (queue, sender, flags, buf, size, sent_cb); + sz =qemu_net_queue_send (queue, sender, flags, buf, size, sent_cb); +qemu_mutex_unlock (&sender->send_lock); +return sz; } ssize_t qemu_send_packet_async(NetClientState *sender,@@ -540,16 +604,21 @@ ssize_t qemu_sendv_packet_async(NetClientState *sender, NetPacketSent *sent_cb) {NetQueue *queue; +ssize_t sz; +qemu_mutex_lock (&sender->send_lock);if (sender->link_down || !sender->peer) { +qemu_mutex_unlock (&sender->send_lock);return iov_size (iov, iovcnt); } queue = sender->peer->send_queue; -return qemu_net_queue_send_iov (queue, sender, + sz =qemu_net_queue_send_iov (queue, sender,QEMU_NET_PACKET_FLAG_NONE , iov, iovcnt, sent_cb); +qemu_mutex_unlock (&sender->send_lock); +return sz; }ssize_t net/queue.c +2
-2@@ -190,7 +190,7 @@ ssize_t qemu_net_queue_send(NetQueue *queue, {ssize_t ret; -if (queue->delivering || !qemu_can_send_packet (sender)) { +if (queue->delivering || !qemu_can_send_packet_nolock (sender)) {qemu_net_queue_append (queue, sender, flags, data, size, sent_cb);return 0 ; }@@ -215,7 +215,7 @@ ssize_t qemu_net_queue_send_iov(NetQueue *queue, {ssize_t ret; -if (queue->delivering || !qemu_can_send_packet (sender)) { +if (queue->delivering || !qemu_can_send_packet_nolock (sender)) {qemu_net_queue_append_iov (queue, sender, flags, iov, iovcnt, sent_cb);return 0 ; }
Re: [Qemu-devel] [PATCH v2 4/5] net: introduce lock to protect NetClientState's peer's access
Il 13/03/2013 02:26, liu ping fan ha scritto:
Quoted 94 lines · Paolo Bonzini
Quoted 80 lines
Il 07/03/2013 03:53, Liu Ping Fan ha scritto:
Quoted 50 lines
From: Liu Ping Fan <[email protected]>
Introduce
nc->send_lock, it shield off the race of nc->peer's reader and deleter. With it, after deleter finish, no newqemu_send_packet_xx()can reach ->send_queue, so no newreference(packet->sender)to nc will be appended tonc->peer->send_queue.Signed-off-by Liu Ping Fan <[email protected]>--- include/net/net.h | 4 +++ net/hub.c | 18 ++++++++++++ net/net.c | 77 ++++++++++++++++++++++++++++++++++++++++++++++++++--- net/queue.c | 4 +- 4 files changed, 97 insertions(+), 6 deletions(-)include/net/net.h +4
-0@@ -66,6 +66,8 @@ struct NetClientState {NetClientInfo *info;int link_down;QTAILQ_ENTRY (NetClientState) next; +/* protect the race access of peer between deleter and sender */ +QemuMutex send_lock;NetClientState *peer;NetQueue *send_queue;char *model;@@ -78,6 +80,7 @@ struct NetClientState { typedef struct NICState { NetClientState *ncs; + NetClientState **pending_peer; NICConf *conf; void *opaque; bool peer_deleted;@@ -105,6 +108,7 @@ NetClientState *qemu_find_vlan_client_by_name(Monitor *mon, int vlan_id, const char *client_str);typedef void (*qemu_nic_foreach )(NICState *nic,void *opaque);void qemu_foreach_nic (qemu_nic_foreach func,void *opaque); +int qemu_can_send_packet_nolock (NetClientState *sender);int qemu_can_send_packet (NetClientState *nc);ssize_t qemu_sendv_packet (NetClientState *nc,const struct iovec *iov,int iovcnt);net/hub.c +0
-0@@ -57,8 +57,14 @@ static ssize_t net_hub_receive(NetHub *hub, NetHubPort *source_port,continue ;net/hub.c static ssize_t net_hub_receive(NetHub *hub, NetHubPort *source_port, · line 67
Quoted 10 lines
net/hub.c +6
-0} +qemu_mutex_lock (&port->nc.send_lock); +if (!port->nc.peer) { +qemu_mutex_unlock (&port->nc.send_lock); +continue ; + }qemu_net_queue_append (port->nc.peer->send_queue, &port->nc,QEMU_NET_PACKET_FLAG_NONE , buf, len,NULL ); +qemu_mutex_unlock (&port->nc.send_lock);Do you really need to lock everything? Can you just wrap the peer with a ref/unref, like
NetClientState *net_client_get_peer(NetClientState *nc)
{ NetClientState *peer; qemu_mutex_lock(&nc->send_lock); peer = nc->peer; if (peer) { net_client_ref(peer); } qemu_mutex_unlock(&nc->send_lock); return peer; }and then
Review
Quoted 8 lines
(quoted diff) +6
-1- qemu_net_queue_append(port->nc.peer->send_queue, &port->nc, + peer = net_client_get_peer(&port->nc); + if (!peer) { + continue; + } + qemu_net_queue_append(peer->send_queue, &port->nc, QEMU_NET_PACKET_FLAG_NONE, buf, len, NULL); + net_client_unref(peer);Oh, seems that I do not explain very clearly in the commit log. The lock does not only protect against the reclaimer( and this can be achieved by refcnt as your codes), but also sync between remover and
sender. If the NetClientState being removed, the remover will be
like:
Quoted 4 lines
nc->peer = NULL;
----------> Here opens the gap for in-flight sender, and
refcnt can not work
flush out reference from its peer's send_queue;What's the problem if the dying peer is still used momentarily? The next unref will drop the last reference and qemu_del_net_queue will free the packet that was just appended.
Paolo
Re: [Qemu-devel] [PATCH v2 4/5] net: introduce lock to protect NetClientState's peer's access
Quoted 100 lines · Paolo Bonzini
Il 13/03/2013 02:26, liu ping fan ha scritto:
Quoted 94 lines · Paolo Bonzini
Quoted 80 lines
Il 07/03/2013 03:53, Liu Ping Fan ha scritto:
Quoted 50 lines
From: Liu Ping Fan <[email protected]>
Introduce
nc->send_lock, it shield off the race of nc->peer's reader and deleter. With it, after deleter finish, no newqemu_send_packet_xx()can reach ->send_queue, so no newreference(packet->sender)to nc will be appended tonc->peer->send_queue.Signed-off-by Liu Ping Fan <[email protected]>--- include/net/net.h | 4 +++ net/hub.c | 18 ++++++++++++ net/net.c | 77 ++++++++++++++++++++++++++++++++++++++++++++++++++--- net/queue.c | 4 +- 4 files changed, 97 insertions(+), 6 deletions(-)include/net/net.h +4
-0@@ -66,6 +66,8 @@ struct NetClientState {NetClientInfo *info;int link_down;QTAILQ_ENTRY (NetClientState) next; +/* protect the race access of peer between deleter and sender */ +QemuMutex send_lock;NetClientState *peer;NetQueue *send_queue;char *model;@@ -78,6 +80,7 @@ struct NetClientState { typedef struct NICState { NetClientState *ncs; + NetClientState **pending_peer; NICConf *conf; void *opaque; bool peer_deleted;@@ -105,6 +108,7 @@ NetClientState *qemu_find_vlan_client_by_name(Monitor *mon, int vlan_id, const char *client_str);typedef void (*qemu_nic_foreach )(NICState *nic,void *opaque);void qemu_foreach_nic (qemu_nic_foreach func,void *opaque); +int qemu_can_send_packet_nolock (NetClientState *sender);int qemu_can_send_packet (NetClientState *nc);ssize_t qemu_sendv_packet (NetClientState *nc,const struct iovec *iov,int iovcnt);net/hub.c +0
-0@@ -57,8 +57,14 @@ static ssize_t net_hub_receive(NetHub *hub, NetHubPort *source_port,continue ;net/hub.c static ssize_t net_hub_receive(NetHub *hub, NetHubPort *source_port, · line 67
Quoted 10 lines
net/hub.c +6
-0} +qemu_mutex_lock (&port->nc.send_lock); +if (!port->nc.peer) { +qemu_mutex_unlock (&port->nc.send_lock); +continue ; + }qemu_net_queue_append (port->nc.peer->send_queue, &port->nc,QEMU_NET_PACKET_FLAG_NONE , buf, len,NULL ); +qemu_mutex_unlock (&port->nc.send_lock);Do you really need to lock everything? Can you just wrap the peer with a ref/unref, like
NetClientState *net_client_get_peer(NetClientState *nc)
{ NetClientState *peer; qemu_mutex_lock(&nc->send_lock); peer = nc->peer; if (peer) { net_client_ref(peer); } qemu_mutex_unlock(&nc->send_lock); return peer; }and then
Review
Quoted 8 lines
(quoted diff) +6
-1- qemu_net_queue_append(port->nc.peer->send_queue, &port->nc, + peer = net_client_get_peer(&port->nc); + if (!peer) { + continue; + } + qemu_net_queue_append(peer->send_queue, &port->nc, QEMU_NET_PACKET_FLAG_NONE, buf, len, NULL); + net_client_unref(peer);Oh, seems that I do not explain very clearly in the commit log. The lock does not only protect against the reclaimer( and this can be achieved by refcnt as your codes), but also sync between remover and
sender. If the NetClientState being removed, the remover will be
like:Quoted 4 lines
nc->peer = NULL; ----------> Here opens the gap for in-flight sender, and refcnt can not work flush out reference from its peer's send_queue;
Quoted 4 lines
What's the problem if the dying peer is still used momentarily? The next unref will drop the last reference and
qemu_del_net_queuewill free the packet that was just appended.
The deletion of NetClientState-A does mean that its peer will be delete, and so the peer's qemu_del_net_queue. And each packet referring to A still holds a ref, and finally A will not be reclaimed.
Regards,
Pingfan
Quoted 1 line
Paolo
PATCH v2 5/5 net: make netclient re-entrant with refcnt
In reply to Liu Ping FanFrom: Liu Ping Fan <[email protected]>
With refcnt, NetClientState's caller can run agaist reclaimer.
--- hw/qdev-properties-system.c | 14 +++++++++++++ include/net/net.h | 3 ++ net/hub.c | 3 ++ net/net.c | 46 ++++++++++++++++++++++++++++++++++++++++-- net/slirp.c | 3 +- 5 files changed, 65 insertions(+), 4 deletions(-)
hw/qdev-properties-system.c +14 -0
@@ -301,6 +301,7 @@ static void set_vlan(Object *obj, Visitor *v, void *opaque,return ; } +/* inc ref, released when unset property */ hubport =net_hub_port_find (id);if (!hubport) { error_set(errp, QERR_INVALID_PARAMETER_VALUE,
@@ -311,11 +312,24 @@ static void set_vlan(Object *obj, Visitor *v, void *opaque, *ptr = hubport; } +static void release_vlan (Object *obj,const char *name,void *opaque) +{ +DeviceState *dev =DEVICE (obj); +Property *prop = opaque; +NICPeers *peers_ptr =qdev_get_prop_ptr (dev, prop); +NetClientState **ptr = &peers_ptr->ncs[0 ]; + +if (*ptr) { +netclient_unref (*ptr); + } +} +PropertyInfo qdev_prop_vlan = { .name ="vlan" , .print = print_vlan, .get = get_vlan, .set = set_vlan, + .release = release_vlan, }; int qdev_prop_set_drive(DeviceState *dev, const char *name,
include/net/net.h +3 -0
@@ -63,6 +63,7 @@ typedef struct NetClientInfo { } NetClientInfo;struct NetClientState { +int ref;NetClientInfo *info;int link_down;QTAILQ_ENTRY (NetClientState) next;
@@ -89,6 +90,8 @@ typedef struct NICState {NetClientState *qemu_find_netdev (const char *id);int qemu_find_net_clients_except (const char *id,NetClientState **ncs,NetClientOptionsKind type,int max); +void netclient_ref (NetClientState *nc); +void netclient_unref (NetClientState *nc); NetClientState *qemu_new_net_client(NetClientInfo *info, NetClientState *peer, const char *model,
net/hub.c +3 -0
@@ -259,6 +259,7 @@ NetClientState *net_hub_find_client_by_name(int hub_id, const char *name) peer = port->nc.peer;if (peer &&strcmp (peer->name, name) ==0 ) { +netclient_ref (peer);qemu_mutex_unlock (&hub->lock);return peer; }
@@ -284,6 +285,7 @@ NetClientState *net_hub_port_find(int hub_id) QLIST_FOREACH(port, &hub->ports, next) { nc = port->nc.peer;if (!nc) { +netclient_ref (&port->nc);qemu_mutex_unlock (&hub->lock);return &(port->nc); }
@@ -294,6 +296,7 @@ NetClientState *net_hub_port_find(int hub_id) } nc =net_hub_add_port (hub_id,NULL ); +netclient_ref (nc);return nc; }
net/net.c +43 -3
203 lines not on this page · raw message
net/slirp.c +2 -1
@@ -346,7 +346,7 @@ void net_slirp_hostfwd_remove(Monitor *mon, const QDict *qdict) err =slirp_remove_hostfwd (QTAILQ_FIRST (&slirp_stacks)->slirp, is_udp, host_addr, host_port); - +netclient_unref (&s->nc);monitor_printf (mon,"host forwarding rule for %s %s\n" , src_str, err ?"not found" :"removed" );return ;
@@ -437,6 +437,7 @@ void net_slirp_hostfwd_add(Monitor *mon, const QDict *qdict) }if (s) {slirp_hostfwd (s, redir_str,0 ); +netclient_unref (&s->nc); } }
Signature · 1 line
1.7.4.4
Re: [Qemu-devel] [PATCH v2 5/5] net: make netclient re-entrant with refcnt
Il 07/03/2013 03:53, Liu Ping Fan ha scritto:
Quoted 4 lines
From: Liu Ping Fan <[email protected]>
With refcnt, NetClientState's caller can run agaist reclaimer.
Quoted 8 lines
Signed-off-by Liu Ping Fan <[email protected]>--- hw/qdev-properties-system.c | 14 +++++++++++++ include/net/net.h | 3 ++ net/hub.c | 3 ++ net/net.c | 46 ++++++++++++++++++++++++++++++++++++++++-- net/slirp.c | 3 +- 5 files changed, 65 insertions(+), 4 deletions(-)
The patch makes sense, but the commit message does not yet.
Please add documentation to include/net/net.h, specifying the functions that add a reference to the returned NetClientState.
Paolo