*** make netlayer re-entrant ***
cover *** make netlayer re-entrant ***
From: Liu Ping Fan <[email protected]>
This series aim to make netlayer re-entrant, so netlayer can run out of biglock safely.
Liu Ping Fan (3):
net: spread hub on AioContexts
net: introduce lock to protect NetClientState's send_queue
net: make netclient re-entrant with refcnt
hw/qdev-properties-system.c | 15 ++++++ include/block/aio.h | 1 + include/net/net.h | 12 +++++ include/net/queue.h | 15 ++++++ main-loop.c | 5 ++ net/hub.c | 81 ++++++++++++++++++++++++++++++-- net/net.c | 109 ++++++++++++++++++++++++++++++++++++++---- net/queue.c | 19 ++++++-- net/slirp.c | 3 +- 9 files changed, 239 insertions(+), 21 deletions(-)
Signature · 1 line
1.7.4.4
PATCH 1/3 net: spread hub on AioContexts
From: Liu Ping Fan <[email protected]>
Forward packet to other hub ports by their AioContext.
--- hw/qdev-properties-system.c | 1 + include/block/aio.h | 1 + include/net/net.h | 5 +++++ include/net/queue.h | 14 ++++++++++++++ main-loop.c | 5 +++++ net/hub.c | 33 ++++++++++++++++++++++++++++++--- net/net.c | 1 + net/queue.c | 4 ++-- 8 files changed, 59 insertions(+), 5 deletions(-)
hw/qdev-properties-system.c +1 -0
@@ -307,6 +307,7 @@ static void set_vlan(Object *obj, Visitor *v, void *opaque, name, prop->info->name);return ; } + hubport->info->reside (hubport,qemu_get_aio_context ()); *ptr = hubport; }
include/block/aio.h +1 -0
@@ -229,6 +229,7 @@ bool qemu_aio_wait(void);void qemu_aio_set_event_notifier (EventNotifier *notifier,EventNotifierHandler *io_read,AioFlushEventNotifierHandler *io_flush); +AioContext *qemu_get_aio_context (void );#ifdef CONFIG_POSIX void qemu_aio_set_fd_handler(int fd,
include/net/net.h +5 -0
@@ -8,6 +8,9 @@#include "net/queue.h" #include "migration/vmstate.h" #include "qapi-types.h" +#include "qemu/thread.h" +#include "block/aio.h" +#define MAX_QUEUE_NUM 1024
@@ -44,6 +47,7 @@ typedef ssize_t (NetReceiveIOV)(NetClientState *, const struct iovec *, int);typedef void (NetCleanup ) (NetClientState *);typedef void (LinkStatusChanged )(NetClientState *);typedef void (NetClientDestructor )(NetClientState *); +typedef void (NetClientReside )(NetClientState *,AioContext *); typedef struct NetClientInfo { NetClientOptionsKind type;
@@ -55,6 +59,7 @@ typedef struct NetClientInfo {NetCleanup *cleanup;LinkStatusChanged *link_status_changed;NetPoll *poll; +NetClientReside *reside; } NetClientInfo;struct NetClientState {
include/net/queue.h +14 -0
@@ -38,6 +38,20 @@ NetQueue *qemu_new_net_queue(void *opaque);void qemu_del_net_queue (NetQueue *queue); +void qemu_net_queue_append (NetQueue *queue, +NetClientState *sender, +unsigned flags, +const uint8_t *buf, +size_t size, +NetPacketSent *sent_cb); + +void qemu_net_queue_append_iov (NetQueue *queue, +NetClientState *sender, +unsigned flags, +const struct iovec *iov, +int iovcnt, +NetPacketSent *sent_cb); + ssize_t qemu_net_queue_send(NetQueue *queue, NetClientState *sender, unsigned flags,
main-loop.c +5 -0
@@ -109,6 +109,11 @@ static int qemu_signal_init(void)static AioContext *qemu_aio_context; +AioContext *qemu_get_aio_context (void ) +{ +return qemu_aio_context; +} +void qemu_notify_event (void ) {if (!qemu_aio_context) {
net/hub.c +30 -3
@@ -31,6 +31,8 @@ typedef struct NetHubPort {QLIST_ENTRY (NetHubPort) next;NetHub *hub;int id; +EventNotifier e; +AioContext *ctx; } NetHubPort;struct NetHub {
@@ -52,11 +54,20 @@ static ssize_t net_hub_receive(NetHub *hub, NetHubPort *source_port,continue ; } -qemu_send_packet (&port->nc, buf, len); +qemu_net_queue_append (port->nc.peer->send_queue, &port->nc, +QEMU_NET_PACKET_FLAG_NONE , buf, len,NULL ); +event_notifier_set (&port->e); }return len; } +static void hub_port_deliver_packet (void *opaque) +{ +NetHubPort *port = (NetHubPort *)opaque; + +qemu_net_queue_flush (port->nc.peer->send_queue); +} +static ssize_t net_hub_receive_iov (NetHub *hub,NetHubPort *source_port,const struct iovec *iov,int iovcnt) {
@@ -68,7 +79,9 @@ static ssize_t net_hub_receive_iov(NetHub *hub, NetHubPort *source_port,continue ; } -qemu_sendv_packet (&port->nc, iov, iovcnt); +qemu_net_queue_append_iov (port->nc.peer->send_queue, &port->nc, +QEMU_NET_PACKET_FLAG_NONE , iov, iovcnt,NULL ); +event_notifier_set (&port->e); }return len; }
@@ -126,9 +139,22 @@ static void net_hub_port_cleanup(NetClientState *nc) {NetHubPort *port =DO_UPCAST (NetHubPort, nc, nc); +if (port->ctx) { +aio_set_fd_handler (port->ctx,event_notifier_get_fd (&port->e), +NULL ,NULL ,NULL ,NULL ); + }QLIST_REMOVE (port, next); } +static void net_hub_port_reside (NetClientState *nc,AioContext *ctx) +{ +NetHubPort *port =DO_UPCAST (NetHubPort, nc, nc); + + port->ctx = ctx; +aio_set_fd_handler (ctx,event_notifier_get_fd (&port->e), + hub_port_deliver_packet,NULL ,NULL , port); +} +static NetClientInfo net_hub_port_info = { .type =NET_CLIENT_OPTIONS_KIND_HUBPORT , .size =sizeof (NetHubPort),
@@ -136,6 +162,7 @@ static NetClientInfo net_hub_port_info = { .receive = net_hub_port_receive, .receive_iov = net_hub_port_receive_iov, .cleanup = net_hub_port_cleanup, + .reside = net_hub_port_reside, };static NetHubPort *net_hub_port_new (NetHub *hub,const char *name)
@@ -155,7 +182,7 @@ static NetHubPort *net_hub_port_new(NetHub *hub, const char *name) port =DO_UPCAST (NetHubPort, nc, nc); port->id = id; port->hub = hub; - +event_notifier_init (&port->e,0 );QLIST_INSERT_HEAD (&hub->ports, port, next);return port;
net/net.c +1 -0
@@ -776,6 +776,7 @@ static int net_client_init1(const void *object, int is_netdev, Error **errp) (opts->kind != NET_CLIENT_OPTIONS_KIND_NIC || !opts->nic->has_netdev)) { peer =net_hub_add_port (u.net->has_vlan ? u.net->vlan :0 ,NULL ); + peer->info->reside (peer,qemu_get_aio_context ()); }if (net_client_init_fun[opts->kind](opts, name, peer) <0 ) {
net/queue.c +2 -2
@@ -83,7 +83,7 @@ void qemu_del_net_queue(NetQueue *queue)g_free (queue); } -static void qemu_net_queue_append(NetQueue *queue, +void qemu_net_queue_append(NetQueue *queue, NetClientState *sender, unsigned flags, const uint8_t *buf,
@@ -102,7 +102,7 @@ static void qemu_net_queue_append(NetQueue *queue,QTAILQ_INSERT_TAIL (&queue->packets, packet, entry); } -static void qemu_net_queue_append_iov(NetQueue *queue, +void qemu_net_queue_append_iov(NetQueue *queue, NetClientState *sender, unsigned flags, const struct iovec *iov,
Signature · 1 line
1.7.4.4
Re: [Qemu-devel] [PATCH 1/3] net: spread hub on AioContexts
include/net/net.h typedef ssize_t (NetReceiveIOV)(NetClientState *, const struct iovec *, int); · line 50
Quoted 5 lines · Liu Ping Fan
include/net/net.h +1
-0@@ -44,6 +47,7 @@ typedef ssize_t (NetReceiveIOV)(NetClientState *, const struct iovec *, int);typedef void (NetCleanup ) (NetClientState *);typedef void (LinkStatusChanged )(NetClientState *);typedef void (NetClientDestructor )(NetClientState *); +typedef void (NetClientReside )(NetClientState *,AioContext *);
I suggest renaming to NetClientBindAioContext or NetClientBindCtx. "Reside" is passive (like to "I live in France") whereas "bind" is an action ("I am moving to France").
Re: [Qemu-devel] [PATCH 1/3] net: spread hub on AioContexts
Quoted 10 lines · Stefan Hajnoczi
include/net/net.h typedef ssize_t (NetReceiveIOV)(NetClientState *, const struct iovec *, int); · line 50
Quoted 5 lines · Liu Ping Fan
include/net/net.h +1
-0@@ -44,6 +47,7 @@ typedef ssize_t (NetReceiveIOV)(NetClientState *, const struct iovec *, int);typedef void (NetCleanup ) (NetClientState *);typedef void (LinkStatusChanged )(NetClientState *);typedef void (NetClientDestructor )(NetClientState *); +typedef void (NetClientReside )(NetClientState *,AioContext *);I suggest renaming to NetClientBindAioContext or NetClientBindCtx. "Reside" is passive (like to "I live in France") whereas "bind" is an action ("I am moving to France").
Will fix. Thanks
PATCH 2/3 net: introduce lock to protect NetClientState's send_queue
In reply to Liu Ping FanFrom: Liu Ping Fan <[email protected]>
Use nc->transfer_lock to protect the nc->peer->send_queue. All of the deleter and senders will sync on this lock, so we can also survive across unplug.
--- include/net/net.h | 4 +++ include/net/queue.h | 1 + net/hub.c | 21 +++++++++++++- net/net.c | 72 ++++++++++++++++++++++++++++++++++++++++++++++++--- net/queue.c | 15 +++++++++- 5 files changed, 105 insertions(+), 8 deletions(-)
include/net/net.h +4 -0
@@ -63,6 +63,8 @@ typedef struct NetClientInfo { } NetClientInfo;struct NetClientState { +/* protect peer's send_queue */ +QemuMutex transfer_lock;NetClientInfo *info;int link_down;QTAILQ_ENTRY (NetClientState) next;
@@ -78,6 +80,7 @@ struct NetClientState { typedef struct NICState { NetClientState ncs[MAX_QUEUE_NUM]; + NetClientState *pending_peer[MAX_QUEUE_NUM]; NICConf *conf; void *opaque; bool peer_deleted;
@@ -105,6 +108,7 @@ NetClientState *qemu_find_vlan_client_by_name(Monitor *mon, int vlan_id, const char *client_str);typedef void (*qemu_nic_foreach )(NICState *nic,void *opaque);void qemu_foreach_nic (qemu_nic_foreach func,void *opaque); +int qemu_can_send_packet_nolock (NetClientState *sender);int qemu_can_send_packet (NetClientState *nc);ssize_t qemu_sendv_packet (NetClientState *nc,const struct iovec *iov,int iovcnt);
include/net/queue.h +1 -0
@@ -67,6 +67,7 @@ ssize_t qemu_net_queue_send_iov(NetQueue *queue, NetPacketSent *sent_cb);void qemu_net_queue_purge (NetQueue *queue,NetClientState *from); +void qemu_net_queue_purge_all (NetQueue *queue);bool qemu_net_queue_flush (NetQueue *queue);#endif /* QEMU_NET_QUEUE_H */
net/hub.c +19 -2
@@ -53,9 +53,14 @@ static ssize_t net_hub_receive(NetHub *hub, NetHubPort *source_port,if (port == source_port) {continue ; } - +qemu_mutex_lock (&port->nc.transfer_lock); +if (!port->nc.peer) { +qemu_mutex_unlock (&port->nc.transfer_lock); +continue ; + }qemu_net_queue_append (port->nc.peer->send_queue, &port->nc,QEMU_NET_PACKET_FLAG_NONE , buf, len,NULL ); +qemu_mutex_unlock (&port->nc.transfer_lock);event_notifier_set (&port->e); }return len;
@@ -65,7 +70,13 @@ static void hub_port_deliver_packet(void *opaque) {NetHubPort *port = (NetHubPort *)opaque; +qemu_mutex_lock (&port->nc.transfer_lock); +if (!port->nc.peer) { +qemu_mutex_unlock (&port->nc.transfer_lock); +return ; + }qemu_net_queue_flush (port->nc.peer->send_queue); +qemu_mutex_unlock (&port->nc.transfer_lock); } static ssize_t net_hub_receive_iov(NetHub *hub, NetHubPort *source_port,
@@ -78,10 +89,16 @@ static ssize_t net_hub_receive_iov(NetHub *hub, NetHubPort *source_port,if (port == source_port) {continue ; } - +qemu_mutex_lock (&port->nc.transfer_lock); +if (!port->nc.peer) { +qemu_mutex_unlock (&port->nc.transfer_lock); +continue ; + }qemu_net_queue_append_iov (port->nc.peer->send_queue, &port->nc,QEMU_NET_PACKET_FLAG_NONE , iov, iovcnt,NULL ); +qemu_mutex_unlock (&port->nc.transfer_lock);event_notifier_set (&port->e); + }return len; }
net/net.c +68 -4
169 lines not on this page · raw message
net/queue.c +13 -2
@@ -24,6 +24,7 @@#include "net/queue.h" #include "qemu/queue.h" #include "net/net.h" +#include "qom/object.h" /* The delivery handler may only return zero if it will call * qemu_net_queue_flush() when it determines that it is once again able
@@ -172,7 +173,7 @@ ssize_t qemu_net_queue_send(NetQueue *queue, {ssize_t ret; -if (queue->delivering || !qemu_can_send_packet (sender)) { +if (queue->delivering || !qemu_can_send_packet_nolock (sender)) {qemu_net_queue_append (queue, sender, flags, data, size, sent_cb);return 0 ; }
@@ -197,7 +198,7 @@ ssize_t qemu_net_queue_send_iov(NetQueue *queue, {ssize_t ret; -if (queue->delivering || !qemu_can_send_packet (sender)) { +if (queue->delivering || !qemu_can_send_packet_nolock (sender)) {qemu_net_queue_append_iov (queue, sender, flags, iov, iovcnt, sent_cb);return 0 ; }
@@ -225,6 +226,16 @@ void qemu_net_queue_purge(NetQueue *queue, NetClientState *from) } } +void qemu_net_queue_purge_all (NetQueue *queue) +{ +NetPacket *packet, *next; + + QTAILQ_FOREACH_SAFE(packet, &queue->packets, entry, next) { +QTAILQ_REMOVE (&queue->packets, packet, entry); +g_free (packet); + } +} +bool qemu_net_queue_flush (NetQueue *queue) {while (!QTAILQ_EMPTY (&queue->packets)) {
Signature · 1 line
1.7.4.4
Re: [Qemu-devel] [PATCH 2/3] net: introduce lock to protect NetClientState's send_queue
Quoted 3 lines · Liu Ping Fan
From: Liu Ping Fan <[email protected]>
Use
nc->transfer_lockto protect thenc->peer->send_queue. All of the
Please use consistent names: the lock protects ->send_queue so it's best called send_queue_lock or send_lock.
Quoted 26 lines
deleter and senders will sync on this lock, so we can also survive across unplug.
Signed-off-by Liu Ping Fan <[email protected]>--- include/net/net.h | 4 +++ include/net/queue.h | 1 + net/hub.c | 21 +++++++++++++- net/net.c | 72 ++++++++++++++++++++++++++++++++++++++++++++++++--- net/queue.c | 15 +++++++++- 5 files changed, 105 insertions(+), 8 deletions(-)include/net/net.h +2
-0@@ -63,6 +63,8 @@ typedef struct NetClientInfo { } NetClientInfo;struct NetClientState { +/* protect peer's send_queue */ +QemuMutex transfer_lock;NetClientInfo *info;int link_down;QTAILQ_ENTRY (NetClientState) next;
include/net/net.h struct NetClientState { · line 83
Quoted 4 lines
include/net/net.h +1
-0@@ -78,6 +80,7 @@ struct NetClientState { typedef struct NICState { NetClientState ncs[MAX_QUEUE_NUM]; + NetClientState *pending_peer[MAX_QUEUE_NUM];
Please rebase onto github.com/stefanha/qemu.git net. ncs[] is no longer statically sized to MAX_QUEUE_NUM.
Quoted 28 lines
include/net/net.h +1
-0NICConf *conf;void *opaque;bool peer_deleted;@@ -105,6 +108,7 @@ NetClientState *qemu_find_vlan_client_by_name(Monitor *mon, int vlan_id, const char *client_str);typedef void (*qemu_nic_foreach )(NICState *nic,void *opaque);void qemu_foreach_nic (qemu_nic_foreach func,void *opaque); +int qemu_can_send_packet_nolock (NetClientState *sender);int qemu_can_send_packet (NetClientState *nc);ssize_t qemu_sendv_packet (NetClientState *nc,const struct iovec *iov,int iovcnt);include/net/queue.h +1
-0@@ -67,6 +67,7 @@ ssize_t qemu_net_queue_send_iov(NetQueue *queue, NetPacketSent *sent_cb);void qemu_net_queue_purge (NetQueue *queue,NetClientState *from); +void qemu_net_queue_purge_all (NetQueue *queue);bool qemu_net_queue_flush (NetQueue *queue);#endif /* QEMU_NET_QUEUE_H */ net/hub.c +0
-0
net/hub.c static ssize_t net_hub_receive(NetHub *hub, NetHubPort *source_port, · line 57
Quoted 6 lines
net/hub.c +2
-1@@ -53,9 +53,14 @@ static ssize_t net_hub_receive(NetHub *hub, NetHubPort *source_port,if (port == source_port) {continue ; } - +qemu_mutex_lock (&port->nc.transfer_lock); +if (!port->nc.peer) {
.peer is protected by transfer_lock too? This was not documented above and I think it's not necessary to protect .peer?
Quoted 54 lines
net/hub.c +17
-1+qemu_mutex_unlock (&port->nc.transfer_lock); +continue ; + }qemu_net_queue_append (port->nc.peer->send_queue, &port->nc,QEMU_NET_PACKET_FLAG_NONE , buf, len,NULL ); +qemu_mutex_unlock (&port->nc.transfer_lock);event_notifier_set (&port->e); }return len;@@ -65,7 +70,13 @@ static void hub_port_deliver_packet(void *opaque) {NetHubPort *port = (NetHubPort *)opaque; +qemu_mutex_lock (&port->nc.transfer_lock); +if (!port->nc.peer) { +qemu_mutex_unlock (&port->nc.transfer_lock); +return ; + }qemu_net_queue_flush (port->nc.peer->send_queue); +qemu_mutex_unlock (&port->nc.transfer_lock); } static ssize_t net_hub_receive_iov(NetHub *hub, NetHubPort *source_port,@@ -78,10 +89,16 @@ static ssize_t net_hub_receive_iov(NetHub *hub, NetHubPort *source_port,if (port == source_port) {continue ; } - +qemu_mutex_lock (&port->nc.transfer_lock); +if (!port->nc.peer) { +qemu_mutex_unlock (&port->nc.transfer_lock); +continue ; + }qemu_net_queue_append_iov (port->nc.peer->send_queue, &port->nc,QEMU_NET_PACKET_FLAG_NONE , iov, iovcnt,NULL ); +qemu_mutex_unlock (&port->nc.transfer_lock);event_notifier_set (&port->e); + }return len; }net/net.c +1
-0@@ -207,6 +207,7 @@ static void qemu_net_client_setup(NetClientState *nc, nc->peer = peer; peer->peer = nc; } +qemu_mutex_init (&nc->transfer_lock);QTAILQ_INSERT_TAIL (&net_clients, nc, next); nc->send_queue =qemu_new_net_queue (nc);
net/net.c void *qemu_get_nic_opaque(NetClientState *nc) · line 289
Quoted 4 lines
net/net.c +1
-0@@ -285,6 +286,7 @@ void *qemu_get_nic_opaque(NetClientState *nc)static void qemu_cleanup_net_client (NetClientState *nc) { +/* This is the place where may be out of big lock, when dev finalized */
I don't understand this comment.
net/net.c static void qemu_free_net_client(NetClientState *nc) · line 313
Quoted 9 lines
net/net.c +2
-0QTAILQ_REMOVE (&net_clients, nc, next);if (nc->info->cleanup) {@@ -307,6 +309,28 @@ static void qemu_free_net_client(NetClientState *nc) } } +/* exclude race with rx/tx path, flush out peer's queue */ +static void qemu_flushout_net_client (NetClientState *nc)
This function detaches the peer, the name should reflect that.
net/net.c static void qemu_free_net_client(NetClientState *nc) · line 323
Quoted 10 lines
net/net.c +10
-0+{ +NetClientState *peer; + +/* sync on receive path */ + peer = nc->peer; +if (peer) { +qemu_mutex_lock (&peer->transfer_lock); + peer->peer =NULL ; +qemu_mutex_unlock (&peer->transfer_lock); + }
This is weird. You don't lock to read nc->peer but you do lock to write peer->peer. If you use a lock it must be used consistently.
Re: [Qemu-devel] [PATCH 2/3] net: introduce lock to protect NetClientState's send_queue
Il 04/03/2013 15:49, Stefan Hajnoczi ha scritto:
Quoted 4 lines
Quoted 1 line
Use
nc->transfer_lockto protect thenc->peer->send_queue. All of thePlease use consistent names: the lock protects ->send_queue so it's best called
send_queue_lockorsend_lock.
In fact, it's a bit strange to use nc->something_lock to lock something in nc->peer. Please add the lock to NetQueue and include/net/queue.h.
nc->peer shouldn't need a lock. It is immutable, isn't it?
Paolo
Re: [Qemu-devel] [PATCH 2/3] net: introduce lock to protect NetClientState's send_queue
Quoted 9 lines · Paolo Bonzini
Il 04/03/2013 15:49, Stefan Hajnoczi ha scritto:
Quoted 4 lines
Quoted 1 line
Use
nc->transfer_lockto protect thenc->peer->send_queue. All of thePlease use consistent names: the lock protects ->send_queue so it's best called
send_queue_lockorsend_lock.In fact, it's a bit strange to use
nc->something_lockto lock something innc->peer. Please add the lock to NetQueue andinclude/net/queue.h.
The lock also servers as exclusion between NetClientState's sender and remover. So if we apply lock on NetQueue, it can not achieve the goal.
Quoted 2 lines
nc->peershouldn't need a lock. It is immutable, isn't it?
The sender should run against hot-uplug.
Thanks and regards,
Pingfan
Quoted 1 line
Paolo
Re: [Qemu-devel] [PATCH 2/3] net: introduce lock to protect NetClientState's send_queue
Il 05/03/2013 03:45, liu ping fan ha scritto:
Quoted 6 lines
Quoted 2 lines
In fact, it's a bit strange to use
nc->something_lockto lock something innc->peer. Please add the lock to NetQueue andinclude/net/queue.h.The lock also servers as exclusion between NetClientState's sender and remover. So if we apply lock on NetQueue, it can not achieve the goal.
Ok, but please document it.
Paolo
Re: [Qemu-devel] [PATCH 2/3] net: introduce lock to protect NetClientState's send_queue
In reply to Stefan HajnocziQuoted 8 lines · Stefan Hajnoczi
Quoted 3 lines · Liu Ping Fan
From: Liu Ping Fan <[email protected]>
Use
nc->transfer_lockto protect thenc->peer->send_queue. All of thePlease use consistent names: the lock protects ->send_queue so it's best called
send_queue_lockorsend_lock.
OK.
Quoted 27 lines
Quoted 27 lines
deleter and senders will sync on this lock, so we can also survive across unplug.
Signed-off-by Liu Ping Fan <[email protected]>--- include/net/net.h | 4 +++ include/net/queue.h | 1 + net/hub.c | 21 +++++++++++++- net/net.c | 72 ++++++++++++++++++++++++++++++++++++++++++++++++--- net/queue.c | 15 +++++++++- 5 files changed, 105 insertions(+), 8 deletions(-)include/net/net.h +2
-0@@ -63,6 +63,8 @@ typedef struct NetClientInfo { } NetClientInfo;struct NetClientState { +/* protect peer's send_queue */ +QemuMutex transfer_lock;NetClientInfo *info;int link_down;QTAILQ_ENTRY (NetClientState) next;@@ -78,6 +80,7 @@ struct NetClientState {
Quoted 7 lines
include/net/net.h struct NetClientState { · line 83
Quoted 3 lines
include/net/net.h +1
-0typedef struct NICState { NetClientState ncs[MAX_QUEUE_NUM]; + NetClientState *pending_peer[MAX_QUEUE_NUM];Please rebase onto github.com/stefanha/qemu.git net. ncs[] is no longer statically sized to
MAX_QUEUE_NUM.
OK
Quoted 22 lines
Quoted 22 lines
include/net/net.h +1
-0NICConf *conf;void *opaque;bool peer_deleted;@@ -105,6 +108,7 @@ NetClientState *qemu_find_vlan_client_by_name(Monitor *mon, int vlan_id, const char *client_str);typedef void (*qemu_nic_foreach )(NICState *nic,void *opaque);void qemu_foreach_nic (qemu_nic_foreach func,void *opaque); +int qemu_can_send_packet_nolock (NetClientState *sender);int qemu_can_send_packet (NetClientState *nc);ssize_t qemu_sendv_packet (NetClientState *nc,const struct iovec *iov,int iovcnt);include/net/queue.h +1
-0@@ -67,6 +67,7 @@ ssize_t qemu_net_queue_send_iov(NetQueue *queue, NetPacketSent *sent_cb);void qemu_net_queue_purge (NetQueue *queue,NetClientState *from); +void qemu_net_queue_purge_all (NetQueue *queue);bool qemu_net_queue_flush (NetQueue *queue);
Quoted 16 lines
net/hub.c static ssize_t net_hub_receive(NetHub *hub, NetHubPort *source_port, · line 57
Quoted 12 lines
include/net/queue.h +0
-0#endif /* QEMU_NET_QUEUE_H */ net/hub.c +2
-1@@ -53,9 +53,14 @@ static ssize_t net_hub_receive(NetHub *hub, NetHubPort *source_port,if (port == source_port) {continue ; } - +qemu_mutex_lock (&port->nc.transfer_lock); +if (!port->nc.peer) {.peer is protected by
transfer_locktoo? This was not documented above and I think it's not necessary to protect .peer?
The transfer_lock has two aims:
to protect the send path against remove path. (lock for nc->peer)
to protect among the senders (lock for nc->peer->send_queue)Quoted 52 lines
Quoted 52 lines
net/hub.c +17
-1+qemu_mutex_unlock (&port->nc.transfer_lock); +continue ; + }qemu_net_queue_append (port->nc.peer->send_queue, &port->nc,QEMU_NET_PACKET_FLAG_NONE , buf, len,NULL ); +qemu_mutex_unlock (&port->nc.transfer_lock);event_notifier_set (&port->e); }return len;@@ -65,7 +70,13 @@ static void hub_port_deliver_packet(void *opaque) {NetHubPort *port = (NetHubPort *)opaque; +qemu_mutex_lock (&port->nc.transfer_lock); +if (!port->nc.peer) { +qemu_mutex_unlock (&port->nc.transfer_lock); +return ; + }qemu_net_queue_flush (port->nc.peer->send_queue); +qemu_mutex_unlock (&port->nc.transfer_lock); } static ssize_t net_hub_receive_iov(NetHub *hub, NetHubPort *source_port,@@ -78,10 +89,16 @@ static ssize_t net_hub_receive_iov(NetHub *hub, NetHubPort *source_port,if (port == source_port) {continue ; } - +qemu_mutex_lock (&port->nc.transfer_lock); +if (!port->nc.peer) { +qemu_mutex_unlock (&port->nc.transfer_lock); +continue ; + }qemu_net_queue_append_iov (port->nc.peer->send_queue, &port->nc,QEMU_NET_PACKET_FLAG_NONE , iov, iovcnt,NULL ); +qemu_mutex_unlock (&port->nc.transfer_lock);event_notifier_set (&port->e); + }return len; }net/net.c +1
-0@@ -207,6 +207,7 @@ static void qemu_net_client_setup(NetClientState *nc, nc->peer = peer; peer->peer = nc; } +qemu_mutex_init (&nc->transfer_lock);QTAILQ_INSERT_TAIL (&net_clients, nc, next);
Quoted 9 lines
net/net.c void *qemu_get_nic_opaque(NetClientState *nc) · line 289
Quoted 6 lines
net/net.c +1
-0nc->send_queue =qemu_new_net_queue (nc);@@ -285,6 +286,7 @@ void *qemu_get_nic_opaque(NetClientState *nc)static void qemu_cleanup_net_client (NetClientState *nc) { +/* This is the place where may be out of big lock, when dev finalized */ I don't understand this comment.
Will remove. I had recorded it to remind myself that extra lock is needed here.
Quoted 7 lines
Quoted 7 lines
net/net.c +0
-0QTAILQ_REMOVE (&net_clients, nc, next);if (nc->info->cleanup) {@@ -307,6 +309,28 @@ static void qemu_free_net_client(NetClientState *nc) } }
Quoted 5 lines
net/net.c static void qemu_free_net_client(NetClientState *nc) · line 313
Quoted 2 lines
net/net.c +2
-0+/* exclude race with rx/tx path, flush out peer's queue */ +static void qemu_flushout_net_client (NetClientState *nc)This function detaches the peer, the name should reflect that.
OK.
Quoted 5 lines
Quoted 5 lines
net/net.c +5
-0+{ +NetClientState *peer; + +/* sync on receive path */ + peer = nc->peer;
Quoted 8 lines
net/net.c static void qemu_free_net_client(NetClientState *nc) · line 323
Quoted 5 lines
net/net.c +5
-0+if (peer) { +qemu_mutex_lock (&peer->transfer_lock); + peer->peer =NULL ; +qemu_mutex_unlock (&peer->transfer_lock); + }This is weird. You don't lock to read
nc->peerbut you do lock to writepeer->peer. If you use a lock it must be used consistently.
Because removal is the only code path to assign nc->peer = NULL, so the reader and updater is serial here. But as for peer->peer, it must run against sender.
Thanks and regards,
Pingfan
Re: [Qemu-devel] [PATCH 2/3] net: introduce lock to protect NetClientState's send_queue
Quoted 174 lines · liu ping fan
Quoted 8 lines · Stefan Hajnoczi
Quoted 3 lines · Liu Ping Fan
From: Liu Ping Fan <[email protected]>
Use
nc->transfer_lockto protect thenc->peer->send_queue. All of thePlease use consistent names: the lock protects ->send_queue so it's best called
send_queue_lockorsend_lock.OK.
Quoted 27 lines
Quoted 27 lines
deleter and senders will sync on this lock, so we can also survive across unplug.
Signed-off-by Liu Ping Fan <[email protected]>--- include/net/net.h | 4 +++ include/net/queue.h | 1 + net/hub.c | 21 +++++++++++++- net/net.c | 72 ++++++++++++++++++++++++++++++++++++++++++++++++--- net/queue.c | 15 +++++++++- 5 files changed, 105 insertions(+), 8 deletions(-)include/net/net.h +2
-0@@ -63,6 +63,8 @@ typedef struct NetClientInfo { } NetClientInfo;struct NetClientState { +/* protect peer's send_queue */ +QemuMutex transfer_lock;NetClientInfo *info;int link_down;QTAILQ_ENTRY (NetClientState) next;@@ -78,6 +80,7 @@ struct NetClientState {Quoted 7 lines
include/net/net.h struct NetClientState { · line 83
Quoted 3 lines
include/net/net.h +1
-0typedef struct NICState { NetClientState ncs[MAX_QUEUE_NUM]; + NetClientState *pending_peer[MAX_QUEUE_NUM];Please rebase onto github.com/stefanha/qemu.git net. ncs[] is no longer statically sized to
MAX_QUEUE_NUM.OK
Quoted 22 lines
Quoted 22 lines
include/net/net.h +1
-0NICConf *conf;void *opaque;bool peer_deleted;@@ -105,6 +108,7 @@ NetClientState *qemu_find_vlan_client_by_name(Monitor *mon, int vlan_id, const char *client_str);typedef void (*qemu_nic_foreach )(NICState *nic,void *opaque);void qemu_foreach_nic (qemu_nic_foreach func,void *opaque); +int qemu_can_send_packet_nolock (NetClientState *sender);int qemu_can_send_packet (NetClientState *nc);ssize_t qemu_sendv_packet (NetClientState *nc,const struct iovec *iov,int iovcnt);include/net/queue.h +1
-0@@ -67,6 +67,7 @@ ssize_t qemu_net_queue_send_iov(NetQueue *queue, NetPacketSent *sent_cb);void qemu_net_queue_purge (NetQueue *queue,NetClientState *from); +void qemu_net_queue_purge_all (NetQueue *queue);bool qemu_net_queue_flush (NetQueue *queue);Quoted 16 lines
net/hub.c static ssize_t net_hub_receive(NetHub *hub, NetHubPort *source_port, · line 57
Quoted 12 lines
include/net/queue.h +0
-0#endif /* QEMU_NET_QUEUE_H */ net/hub.c +2
-1@@ -53,9 +53,14 @@ static ssize_t net_hub_receive(NetHub *hub, NetHubPort *source_port,if (port == source_port) {continue ; } - +qemu_mutex_lock (&port->nc.transfer_lock); +if (!port->nc.peer) {.peer is protected by
transfer_locktoo? This was not documented above and I think it's not necessary to protect .peer?The
transfer_lockhas two aims:to protect the send path against remove path. (lock for nc->peer) to protect among the senders (lock for nc->peer->send_queue)Quoted 52 lines
Quoted 52 lines
net/hub.c +17
-1+qemu_mutex_unlock (&port->nc.transfer_lock); +continue ; + }qemu_net_queue_append (port->nc.peer->send_queue, &port->nc,QEMU_NET_PACKET_FLAG_NONE , buf, len,NULL ); +qemu_mutex_unlock (&port->nc.transfer_lock);event_notifier_set (&port->e); }return len;@@ -65,7 +70,13 @@ static void hub_port_deliver_packet(void *opaque) {NetHubPort *port = (NetHubPort *)opaque; +qemu_mutex_lock (&port->nc.transfer_lock); +if (!port->nc.peer) { +qemu_mutex_unlock (&port->nc.transfer_lock); +return ; + }qemu_net_queue_flush (port->nc.peer->send_queue); +qemu_mutex_unlock (&port->nc.transfer_lock); } static ssize_t net_hub_receive_iov(NetHub *hub, NetHubPort *source_port,@@ -78,10 +89,16 @@ static ssize_t net_hub_receive_iov(NetHub *hub, NetHubPort *source_port,if (port == source_port) {continue ; } - +qemu_mutex_lock (&port->nc.transfer_lock); +if (!port->nc.peer) { +qemu_mutex_unlock (&port->nc.transfer_lock); +continue ; + }qemu_net_queue_append_iov (port->nc.peer->send_queue, &port->nc,QEMU_NET_PACKET_FLAG_NONE , iov, iovcnt,NULL ); +qemu_mutex_unlock (&port->nc.transfer_lock);event_notifier_set (&port->e); + }return len; }net/net.c +1
-0@@ -207,6 +207,7 @@ static void qemu_net_client_setup(NetClientState *nc, nc->peer = peer; peer->peer = nc; } +qemu_mutex_init (&nc->transfer_lock);QTAILQ_INSERT_TAIL (&net_clients, nc, next);Quoted 9 lines
net/net.c void *qemu_get_nic_opaque(NetClientState *nc) · line 289
Quoted 6 lines
net/net.c +1
-0nc->send_queue =qemu_new_net_queue (nc);@@ -285,6 +286,7 @@ void *qemu_get_nic_opaque(NetClientState *nc)static void qemu_cleanup_net_client (NetClientState *nc) { +/* This is the place where may be out of big lock, when dev finalized */ I don't understand this comment.
Will remove. I had recorded it to remind myself that extra lock is needed here.
Quoted 7 lines
Quoted 7 lines
net/net.c +0
-0QTAILQ_REMOVE (&net_clients, nc, next);if (nc->info->cleanup) {@@ -307,6 +309,28 @@ static void qemu_free_net_client(NetClientState *nc) } }Quoted 5 lines
net/net.c static void qemu_free_net_client(NetClientState *nc) · line 313
Quoted 2 lines
net/net.c +2
-0+/* exclude race with rx/tx path, flush out peer's queue */ +static void qemu_flushout_net_client (NetClientState *nc)This function detaches the peer, the name should reflect that.
OK.
Quoted 5 lines
Quoted 5 lines
net/net.c +5
-0+{ +NetClientState *peer; + +/* sync on receive path */ + peer = nc->peer;Quoted 8 lines
net/net.c static void qemu_free_net_client(NetClientState *nc) · line 323
Quoted 5 lines
net/net.c +5
-0+if (peer) { +qemu_mutex_lock (&peer->transfer_lock); + peer->peer =NULL ; +qemu_mutex_unlock (&peer->transfer_lock); + }This is weird. You don't lock to read
nc->peerbut you do lock to writepeer->peer. If you use a lock it must be used consistently.
Quoted 4 lines
Because removal is the only code path to assign
nc->peer= NULL, so the reader and updater is serial here. But as forpeer->peer, it must run against sender.
The race between removers is excluded by big lock in hot-unplug path.
Quoted 3 lines
Thanks and regards,
Pingfan
Re: [Qemu-devel] [PATCH 2/3] net: introduce lock to protect NetClientState's send_queue
Quoted 27 lines · liu ping fan
Quoted 26 lines · liu ping fan
Quoted 11 lines · Stefan Hajnoczi
net/net.c static void qemu_free_net_client(NetClientState *nc) · line 313
Quoted 7 lines · Liu Ping Fan
From: Liu Ping Fan <[email protected]>
net/net.c +2
-0@@ -307,6 +309,28 @@ static void qemu_free_net_client(NetClientState *nc) } } +/* exclude race with rx/tx path, flush out peer's queue */ +static void qemu_flushout_net_client (NetClientState *nc)This function detaches the peer, the name should reflect that.
OK.
Quoted 13 lines
net/net.c static void qemu_free_net_client(NetClientState *nc) · line 323
Quoted 10 lines
net/net.c +10
-0+{ +NetClientState *peer; + +/* sync on receive path */ + peer = nc->peer; +if (peer) { +qemu_mutex_lock (&peer->transfer_lock); + peer->peer =NULL ; +qemu_mutex_unlock (&peer->transfer_lock); + }This is weird. You don't lock to read
nc->peerbut you do lock to writepeer->peer. If you use a lock it must be used consistently.
Quoted 5 lines
Quoted 4 lines
Because removal is the only code path to assign
nc->peer= NULL, so the reader and updater is serial here. But as forpeer->peer, it must run against sender.The race between removers is excluded by big lock in hot-unplug path.
Please use the locks consistently instead of relying on the global mutex or other assumptions. The assumptions can change (plus you haven't documented them) so these patches make the net subsystem very brittle.
The next person wishing to modify net.c would have to reverse-engineer all the assumptions you've made and doesn't have clear examples of which locks to use.
Stefan
PATCH 3/3 net: make netclient re-entrant with refcnt
In reply to Liu Ping FanFrom: Liu Ping Fan <[email protected]>
With refcnt, NetClientState's caller can run agaist reclaimer.
--- hw/qdev-properties-system.c | 14 ++++++++++++++ include/net/net.h | 3 +++ net/hub.c | 29 ++++++++++++++++++++++++++++- net/net.c | 40 +++++++++++++++++++++++++++++++--------- net/slirp.c | 3 ++- 5 files changed, 78 insertions(+), 11 deletions(-)
hw/qdev-properties-system.c +14 -0
@@ -301,6 +301,7 @@ static void set_vlan(Object *obj, Visitor *v, void *opaque,return ; } +/* inc ref, released when unset property */ hubport =net_hub_port_find (id);if (!hubport) { error_set(errp, QERR_INVALID_PARAMETER_VALUE,
@@ -311,11 +312,24 @@ static void set_vlan(Object *obj, Visitor *v, void *opaque, *ptr = hubport; } +static void release_vlan (Object *obj,const char *name,void *opaque) +{ +DeviceState *dev =DEVICE (obj); +Property *prop = opaque; +NICPeers *peers_ptr =qdev_get_prop_ptr (dev, prop); +NetClientState **ptr = &peers_ptr->ncs[0 ]; + +if (*ptr) { +netclient_unref (*ptr); + } +} +PropertyInfo qdev_prop_vlan = { .name ="vlan" , .print = print_vlan, .get = get_vlan, .set = set_vlan, + .release = release_vlan, }; int qdev_prop_set_drive(DeviceState *dev, const char *name,
include/net/net.h +3 -0
@@ -63,6 +63,7 @@ typedef struct NetClientInfo { } NetClientInfo;struct NetClientState { +int ref;/* protect peer's send_queue */ QemuMutex transfer_lock;NetClientInfo *info;
@@ -89,6 +90,8 @@ typedef struct NICState {NetClientState *qemu_find_netdev (const char *id);int qemu_find_net_clients_except (const char *id,NetClientState **ncs,NetClientOptionsKind type,int max); +void netclient_ref (NetClientState *nc); +void netclient_unref (NetClientState *nc); NetClientState *qemu_new_net_client(NetClientInfo *info, NetClientState *peer, const char *model,
net/hub.c +28 -1
@@ -36,6 +36,7 @@ typedef struct NetHubPort { } NetHubPort;struct NetHub { +QemuMutex lock;int id;QLIST_ENTRY (NetHub) next;int num_ports;
@@ -49,6 +50,7 @@ static ssize_t net_hub_receive(NetHub *hub, NetHubPort *source_port, {NetHubPort *port; +qemu_mutex_lock (&hub->lock); QLIST_FOREACH(port, &hub->ports, next) {if (port == source_port) {continue ;
@@ -63,6 +65,7 @@ static ssize_t net_hub_receive(NetHub *hub, NetHubPort *source_port,qemu_mutex_unlock (&port->nc.transfer_lock);event_notifier_set (&port->e); } +qemu_mutex_unlock (&hub->lock);return len; }
@@ -85,6 +88,7 @@ static ssize_t net_hub_receive_iov(NetHub *hub, NetHubPort *source_port,NetHubPort *port;ssize_t len =iov_size (iov, iovcnt); +qemu_mutex_lock (&hub->lock); QLIST_FOREACH(port, &hub->ports, next) {if (port == source_port) {continue ;
@@ -100,6 +104,8 @@ static ssize_t net_hub_receive_iov(NetHub *hub, NetHubPort *source_port,event_notifier_set (&port->e); } +qemu_mutex_unlock (&hub->lock); +return len; }
@@ -111,7 +117,7 @@ static NetHub *net_hub_new(int id) hub->id = id; hub->num_ports =0 ;QLIST_INIT (&hub->ports); - +qemu_mutex_init (&hub->lock);QLIST_INSERT_HEAD (&hubs, hub, next);return hub;
@@ -123,15 +129,18 @@ static int net_hub_port_can_receive(NetClientState *nc)NetHubPort *src_port =DO_UPCAST (NetHubPort, nc, nc);NetHub *hub = src_port->hub; +qemu_mutex_lock (&hub->lock); QLIST_FOREACH(port, &hub->ports, next) {if (port == src_port) {continue ; }if (qemu_can_send_packet (&port->nc)) { +qemu_mutex_unlock (&hub->lock);return 1 ; } } +qemu_mutex_unlock (&hub->lock);return 0 ; }
@@ -160,7 +169,9 @@ static void net_hub_port_cleanup(NetClientState *nc)aio_set_fd_handler (port->ctx,event_notifier_get_fd (&port->e),NULL ,NULL ,NULL ,NULL ); } +qemu_mutex_lock (&port->hub->lock);QLIST_REMOVE (port, next); +qemu_mutex_unlock (&port->hub->lock); }static void net_hub_port_reside (NetClientState *nc,AioContext *ctx)
@@ -200,7 +211,9 @@ static NetHubPort *net_hub_port_new(NetHub *hub, const char *name) port->id = id; port->hub = hub;event_notifier_init (&port->e,0 ); +qemu_mutex_lock (&hub->lock);QLIST_INSERT_HEAD (&hub->ports, port, next); +qemu_mutex_unlock (&hub->lock);return port; }
@@ -241,13 +254,17 @@ NetClientState *net_hub_find_client_by_name(int hub_id, const char *name) QLIST_FOREACH(hub, &hubs, next) {if (hub->id == hub_id) { +qemu_mutex_lock (&hub->lock); QLIST_FOREACH(port, &hub->ports, next) { peer = port->nc.peer;if (peer &&strcmp (peer->name, name) ==0 ) { +netclient_ref (peer); +qemu_mutex_unlock (&hub->lock);return peer; } } +qemu_mutex_unlock (&hub->lock); } }return NULL ;
@@ -264,17 +281,22 @@ NetClientState *net_hub_port_find(int hub_id) QLIST_FOREACH(hub, &hubs, next) {if (hub->id == hub_id) { +qemu_mutex_lock (&hub->lock); QLIST_FOREACH(port, &hub->ports, next) { nc = port->nc.peer;if (!nc) { +netclient_ref (&port->nc); +qemu_mutex_unlock (&hub->lock);return &(port->nc); } } +qemu_mutex_unlock (&hub->lock);break ; } } nc =net_hub_add_port (hub_id,NULL ); +netclient_ref (nc);return nc; }
@@ -288,12 +310,14 @@ void net_hub_info(Monitor *mon) QLIST_FOREACH(hub, &hubs, next) {monitor_printf (mon,"hub %d\n" , hub->id); +qemu_mutex_lock (&hub->lock); QLIST_FOREACH(port, &hub->ports, next) {if (port->nc.peer) {monitor_printf (mon," \\ " );print_net_client (mon, port->nc.peer); } } +qemu_mutex_unlock (&hub->lock); } }
@@ -350,6 +374,7 @@ void net_hub_check_clients(void) QLIST_FOREACH(hub, &hubs, next) {int has_nic =0 , has_host_dev =0 ; +qemu_mutex_lock (&hub->lock); QLIST_FOREACH(port, &hub->ports, next) { peer = port->nc.peer;if (!peer) {
@@ -372,6 +397,8 @@ void net_hub_check_clients(void)break ; } } +qemu_mutex_unlock (&hub->lock); +if (has_host_dev && !has_nic) {fprintf (stderr,"Warning: vlan %d with no nics\n" , hub->id); }
net/net.c +31 -9
@@ -45,6 +45,7 @@# define CONFIG_NET_BRIDGE #endif +static QemuMutex net_clients_lock; static QTAILQ_HEAD(, NetClientState) net_clients;int default_net =1 ;
@@ -206,12 +207,16 @@ static void qemu_net_client_setup(NetClientState *nc,assert (!peer->peer); nc->peer = peer; peer->peer = nc; +netclient_ref (nc); +netclient_ref (peer); }qemu_mutex_init (&nc->transfer_lock); -QTAILQ_INSERT_TAIL (&net_clients, nc, next); - nc->send_queue =qemu_new_net_queue (nc); nc->destructor = destructor; + +qemu_mutex_lock (&net_clients_lock); +QTAILQ_INSERT_TAIL (&net_clients, nc, next); +qemu_mutex_unlock (&net_clients_lock); } NetClientState *qemu_new_net_client(NetClientInfo *info,
@@ -224,6 +229,7 @@ NetClientState *qemu_new_net_client(NetClientInfo *info,assert (info->size >=sizeof (NetClientState)); nc =g_malloc0 (info->size); +netclient_ref (nc);qemu_net_client_setup (nc, info, peer, model, name, qemu_net_client_destructor);
@@ -299,9 +305,6 @@ static void qemu_free_net_client(NetClientState *nc)if (nc->send_queue) {qemu_del_net_queue (nc->send_queue); } -if (nc->peer) { - nc->peer->peer =NULL ; - }g_free (nc->name);g_free (nc->model);if (nc->destructor) {
@@ -309,6 +312,18 @@ static void qemu_free_net_client(NetClientState *nc) } } +void netclient_ref (NetClientState *nc) +{ +__sync_add_and_fetch (&nc->ref,1 ); +} + +void netclient_unref (NetClientState *nc) +{ +if (__sync_sub_and_fetch (&nc->ref,1 ) ==0 ) { +qemu_free_net_client (nc); + } +} +/* exclude race with rx/tx path, flush out peer's queue */ static void qemu_flushout_net_client (NetClientState *nc) {
@@ -320,15 +335,17 @@ static void qemu_flushout_net_client(NetClientState *nc)qemu_mutex_lock (&peer->transfer_lock); peer->peer =NULL ;qemu_mutex_unlock (&peer->transfer_lock); +netclient_unref (nc); } -/* sync on send from this nc */ +/* sync on send, flush out packets on peer's queue */ qemu_mutex_lock (&nc->transfer_lock); nc->peer =NULL ;if (peer) {qemu_net_queue_purge (peer->send_queue, nc); }qemu_mutex_unlock (&nc->transfer_lock); +netclient_unref (peer); }void qemu_del_net_client (NetClientState *nc)
@@ -374,7 +391,7 @@ void qemu_del_net_client(NetClientState *nc)for (i =0 ; i < queues; i++) {qemu_flushout_net_client (ncs[i]);qemu_cleanup_net_client (ncs[i]); -qemu_free_net_client (ncs[i]); +netclient_unref (ncs[i]); } }
@@ -385,7 +402,7 @@ void qemu_del_nic(NICState *nic)/* If this is a peer NIC and peer has already been deleted, free it now. */ if (nic->peer_deleted) {for (i =0 ; i < queues; i++) { -qemu_free_net_client (nic->pending_peer[i]); +netclient_unref (nic->pending_peer[i]); } }
@@ -395,7 +412,7 @@ void qemu_del_nic(NICState *nic)qemu_flushout_net_client (nc);qemu_cleanup_net_client (nc); -qemu_free_net_client (nc); +netclient_unref (nc); } }
@@ -624,6 +641,8 @@ NetClientState *qemu_find_netdev(const char *id)if (nc->info->type ==NET_CLIENT_OPTIONS_KIND_NIC )continue ;if (!strcmp (nc->name, id)) { +netclient_ref (nc); +qemu_mutex_unlock (&net_clients_lock);return nc; } }
@@ -957,6 +976,7 @@ void net_host_device_remove(Monitor *mon, const QDict *qdict)return ; }qemu_del_net_client (nc); +netclient_unref (nc); }void netdev_add (QemuOpts *opts,Error **errp)
@@ -1012,6 +1032,7 @@ void qmp_netdev_del(const char *id, Error **errp) }qemu_del_net_client (nc); +netclient_unref (nc);qemu_opts_del (opts); }
@@ -1186,6 +1207,7 @@ int net_init_clients(void)#endif } +qemu_mutex_init (&net_clients_lock);QTAILQ_INIT (&net_clients);if (qemu_opts_foreach (qemu_find_opts ("netdev" ), net_init_netdev,NULL ,1 ) ==-1 )
net/slirp.c +2 -1
@@ -346,7 +346,7 @@ void net_slirp_hostfwd_remove(Monitor *mon, const QDict *qdict) err =slirp_remove_hostfwd (QTAILQ_FIRST (&slirp_stacks)->slirp, is_udp, host_addr, host_port); - +netclient_unref (&s->nc);monitor_printf (mon,"host forwarding rule for %s %s\n" , src_str, err ?"not found" :"removed" );return ;
@@ -437,6 +437,7 @@ void net_slirp_hostfwd_add(Monitor *mon, const QDict *qdict) }if (s) {slirp_hostfwd (s, redir_str,0 ); +netclient_unref (&s->nc); } }
Signature · 1 line
1.7.4.4
Re: [Qemu-devel] [PATCH 3/3] net: make netclient re-entrant with refcnt
net/hub.c typedef struct NetHubPort { · line 39
Quoted 9 lines · Liu Ping Fan
net/hub.c +1
-0@@ -36,6 +36,7 @@ typedef struct NetHubPort { } NetHubPort;struct NetHub { +QemuMutex lock;
Please document what this lock protects. I think it only protects the ports list.
net/net.c line 49
Quoted 10 lines
net/net.c +1
-0@@ -45,6 +45,7 @@# define CONFIG_NET_BRIDGE #endif +static QemuMutex net_clients_lock; static QTAILQ_HEAD(, NetClientState) net_clients;
I'm not sure what net_clients_lock protects since this lock is not taken by all net_clients users.
The lock should be destroyed in net_cleanup().
This should be a separate patch that fully explains the purpose of the lock and uses it consistently in net.c.
Re: [Qemu-devel] [PATCH 3/3] net: make netclient re-entrant with refcnt
Quoted 8 lines · Stefan Hajnoczi
Quoted 7 lines · Liu Ping Fan
net/hub.c +0
-0@@ -36,6 +36,7 @@ typedef struct NetHubPort { } NetHubPort;
Quoted 6 lines
net/hub.c typedef struct NetHubPort { · line 39
Quoted 2 lines
net/hub.c +1
-0struct NetHub { +QemuMutex lock;Please document what this lock protects. I think it only protects the ports list.
OK
Quoted 8 lines
Quoted 8 lines
net/net.c +0
-0@@ -45,6 +45,7 @@# define CONFIG_NET_BRIDGE #endif
Quoted 6 lines
net/net.c line 49
Quoted 2 lines
net/net.c +1
-0+static QemuMutex net_clients_lock; static QTAILQ_HEAD(, NetClientState) net_clients;I'm not sure what
net_clients_lockprotects since this lock is not taken by allnet_clientsusers.
Oh, some made mistake when rebasing. Will fix it.
Quoted 2 lines
The lock should be destroyed in
net_cleanup().
OK
Quoted 2 lines
This should be a separate patch that fully explains the purpose of the lock and uses it consistently in
net.c.
OK
Thanks and regards,
Pingfan
Quoted 4 lines · Liu Ping Fan
From: Liu Ping Fan <[email protected]>
This series aim to make netlayer re-entrant, so netlayer can run out of biglock safely.
I think most of the locking considerations are still applicable either way, but this series seems to be written under the assumption that we'll be associating hubs/ports with separate AioContexts to facilitate driving the event handling outside of the iothread. Is this the case?
Quoted 1 line
From what I gathered from the other thread, the path forward was to
replace the global iohandler list that we currently use to drive NetClient events and replace it with a GSource and GMainContext, rather than relying on AioContexts.
I do agree that the event handlers currently grouped under iohandler.c:io_handlers look like a nice fit for AioContexts, but other things like slirp and chardevs seem better served by a more general mechanism like GSources/GMainContexts. The chardev flow control patches seem to be doing something similar already as well.
Quoted 20 lines
Liu Ping Fan (3):
net: spread hub on AioContexts
net: introduce lock to protect NetClientState'ssend_queue
net: make netclient re-entrant with refcnthw/qdev-properties-system.c | 15 ++++++ include/block/aio.h | 1 + include/net/net.h | 12 +++++ include/net/queue.h | 15 ++++++ main-loop.c | 5 ++ net/hub.c | 81 ++++++++++++++++++++++++++++++-- net/net.c | 109 ++++++++++++++++++++++++++++++++++++++---- net/queue.c | 19 ++++++-- net/slirp.c | 3 +- 9 files changed, 239 insertions(+), 21 deletions(-)Signature · 1 line
1.7.4.4
Quoted 11 lines · mdroth
Quoted 4 lines · Liu Ping Fan
From: Liu Ping Fan <[email protected]>
This series aim to make netlayer re-entrant, so netlayer can run out of biglock safely.
I think most of the locking considerations are still applicable either way, but this series seems to be written under the assumption that we'll be associating hubs/ports with separate AioContexts to facilitate driving the event handling outside of the iothread. Is this the case?
Yes.
Quoted 5 lines
From what I gathered from the other thread, the path forward was to replace the global iohandler list that we currently use to drive NetClient events and replace it with a GSource and GMainContext, rather than relying on AioContexts.
Not quite sure about it. Seems that AioContext is built on GSource, so I think they are similar, and AioContext is easy to reuse.
Quoted 6 lines
I do agree that the event handlers currently grouped under iohandler.c:io_handlers look like a nice fit for AioContexts, but other things like slirp and chardevs seem better served by a more general mechanism like GSources/GMainContexts. The chardev flow control patches seem to be doing something similar already as well.
I have made some fix for this series, apart from the concert about GSource/ AioContext. Hope to discuss it clearly in next version and fix it too. BTW what can we benefit from AioContext besides those from GSource
Thanks and regards,
Pingfan
Quoted 20 lines
Quoted 20 lines
Liu Ping Fan (3):
net: spread hub on AioContexts
net: introduce lock to protect NetClientState'ssend_queue
net: make netclient re-entrant with refcnthw/qdev-properties-system.c | 15 ++++++ include/block/aio.h | 1 + include/net/net.h | 12 +++++ include/net/queue.h | 15 ++++++ main-loop.c | 5 ++ net/hub.c | 81 ++++++++++++++++++++++++++++++-- net/net.c | 109 ++++++++++++++++++++++++++++++++++++++---- net/queue.c | 19 ++++++-- net/slirp.c | 3 +- 9 files changed, 239 insertions(+), 21 deletions(-)--
1.7.4.4
Quoted 27 lines · liu ping fan
Quoted 11 lines · mdroth
Quoted 4 lines · Liu Ping Fan
From: Liu Ping Fan <[email protected]>
This series aim to make netlayer re-entrant, so netlayer can run out of biglock safely.
I think most of the locking considerations are still applicable either way, but this series seems to be written under the assumption that we'll be associating hubs/ports with separate AioContexts to facilitate driving the event handling outside of the iothread. Is this the case?
Yes.
Quoted 5 lines
From what I gathered from the other thread, the path forward was to replace the global iohandler list that we currently use to drive NetClient events and replace it with a GSource and GMainContext, rather than relying on AioContexts.
Not quite sure about it. Seems that AioContext is built on GSource, so I think they are similar, and AioContext is easy to reuse.
Quoted 6 lines
I do agree that the event handlers currently grouped under iohandler.c:io_handlers look like a nice fit for AioContexts, but other things like slirp and chardevs seem better served by a more general mechanism like GSources/GMainContexts. The chardev flow control patches seem to be doing something similar already as well.
Quoted 4 lines
I have made some fix for this series, apart from the concert about GSource/ AioContext. Hope to discuss it clearly in next version and fix it too. BTW what can we benefit from AioContext besides those from GSource
This is a good discussion. I'd like to hear more about using glib event loop concepts instead of rolling our own. Here are my thoughts after exploring the glib main loop vs AioContext.
AioContext supports two things:
1. BH which is similar to GIdle for scheduling functions that get called from the event loop.
Note that GIdle doesn't work in aio_poll() because we don't run integrate the glib event loop.
2. aio_poll() which goes a step beyond iohandlers because the ->io_flush() handler signals whether there are pending aio requests. This way aio_poll() can be called in a loop until all pending requests have finished.
Imagine block/iscsi.c which has a TCP socket to the iSCSI target. We're ready to receive from the socket but we only want to wait until all pending requests complete. That means the socket fd is always looking for G_IO_IN events but we shouldn't wait unless there are actually iSCSI requests pending.
This feature is important for the synchronous (nested event loop) functionality that QEMU relies on for bdrv_drain_all() and synchronous I/O (bdrv_read(), bdrv_write(), bdrv_flush()).
The glib equivalent to aio_poll() is g_main_context_iteration():
https://developer.gnome.org/glib/2.30/glib-The-Main-Event-Loop.html#g-main-context-iteration
But note that the return value is different. g_main_context_iteration() tells you if any event handlers were called. aio_poll() tells you whether more calls are necessary in order to reach a quiescent state (all requests completed).
I guess it's time to look at the chardev flow control patches to see how glib event loop concepts are being used.
Stefan
Quoted 27 lines · liu ping fan
Quoted 11 lines · mdroth
Quoted 4 lines · Liu Ping Fan
From: Liu Ping Fan <[email protected]>
This series aim to make netlayer re-entrant, so netlayer can run out of biglock safely.
I think most of the locking considerations are still applicable either way, but this series seems to be written under the assumption that we'll be associating hubs/ports with separate AioContexts to facilitate driving the event handling outside of the iothread. Is this the case?
Yes.
Quoted 5 lines
From what I gathered from the other thread, the path forward was to replace the global iohandler list that we currently use to drive NetClient events and replace it with a GSource and GMainContext, rather than relying on AioContexts.
Not quite sure about it. Seems that AioContext is built on GSource, so I think they are similar, and AioContext is easy to reuse.
Quoted 6 lines
I do agree that the event handlers currently grouped under iohandler.c:io_handlers look like a nice fit for AioContexts, but other things like slirp and chardevs seem better served by a more general mechanism like GSources/GMainContexts. The chardev flow control patches seem to be doing something similar already as well.
Quoted 4 lines
I have made some fix for this series, apart from the concert about GSource/ AioContext. Hope to discuss it clearly in next version and fix it too. BTW what can we benefit from AioContext besides those from GSource
One thing I forgot to add: net/ doesn't use BH or qemu_aio_set_fd_handler() so AioContext isn't strictly necessary.
Stefan
Il 07/03/2013 10:33, Stefan Hajnoczi ha scritto:
Quoted 6 lines
Quoted 4 lines
Quoted 4 lines
I have made some fix for this series, apart from the concert about GSource/ AioContext. Hope to discuss it clearly in next version and fix it too. BTW what can we benefit from AioContext besides those from GSource
One thing I forgot to add:
net/doesn't use BH orqemu_aio_set_fd_handler()so AioContext isn't strictly necessary.
No, it's not. However, if we want to share ioeventfd handlers between virtio-blk and virtio-net, it becomes more important.
Paolo
Il 07/03/2013 03:06, liu ping fan ha scritto:
Quoted 7 lines
Quoted 4 lines
From what I gathered from the other thread, the path forward was to replace the global iohandler list that we currently use to drive NetClient events and replace it with a GSource and GMainContext, rather than relying on AioContexts.
Not quite sure about it. Seems that AioContext is built on GSource, so I think they are similar, and AioContext is easy to reuse.
AioContext is designed so that it can be used either independently (as virtio-blk-dataplane does now) or as a GSource (as we do for the "main" AioContext, which is used for the block layer and where qemu_bh_new places bottom halves).
Paolo