From: Alexander Aring Date: Wed, 23 Jun 2021 11:14:52 -0400 Subject: [Cluster-devel] [RFC dlm/next 13/15] fs: dlm: move writequeue init to sendcon only In-Reply-To: <20210623151454.176649-1-aahringo@redhat.com> References: <20210623151454.176649-1-aahringo@redhat.com> Message-ID: <20210623151454.176649-14-aahringo@redhat.com> List-Id: To: cluster-devel.redhat.com MIME-Version: 1.0 Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: 7bit This patch inits only sendconf functionality to the sendcon and not othercon. If we have make a mistake by accident we can see it when the kernel crashes because the othercon was queued for transmitting which should never be the case. Also add a comment about the othercon handling and why it's there and how we could possible remove it with breaking backwards compatibility. Signed-off-by: Alexander Aring --- fs/dlm/lowcomms.c | 61 ++++++++++++++++++++++++++++++++++------------- 1 file changed, 44 insertions(+), 17 deletions(-) diff --git a/fs/dlm/lowcomms.c b/fs/dlm/lowcomms.c index ddf3c0c98386..e858453b4eb7 100644 --- a/fs/dlm/lowcomms.c +++ b/fs/dlm/lowcomms.c @@ -40,6 +40,29 @@ * cluster-wide mechanism as it must be the same on all nodes of the cluster * for the DLM to function. * + * TODO: + * + * Special note about the "othercon" field in connection structure. This is + * only set if we hit a race between two peers do connect() and accept() at + * the same time. If we don't accept the connection and close it the other + * peer will disconnect on it's non "othercon" connection which is known as + * "sendcon". It's named "sendcon" because it is a must that we only do + * sending over this connection. If we hit the race then the "othercon" is + * be used for receiving only. Calling a "ss -t" shows two connections in + * this case. + * + * Overall it makes this code a lot of confusion and the code tries to use + * only the "sendcon" as resource e.g. mutexes. As the race is only sometimes + * there moving "othercon" in it's own struct e.g. "struct connection_other" + * makes it difficult to deal with it when we don't hit the race and the + * receiving is done by "sendcon". + * + * There exists an idea by Steve Whitehouse to get rid of this race by + * introducing a priotize accept() rule e.g. $OWN_NODEID < $PEER_NODEID. + * If the condition is true we accept the node otherwise we trigger a reconnect + * to this peer (because the peer wants connect again). However this is not + * backwards compatible and will break the connection handling with the + * "othercon" handling. */ #include @@ -283,16 +306,15 @@ static struct connection *__find_con(int nodeid, int r) static bool tcp_eof_condition(struct connection *con) { + if (test_bit(CF_IS_OTHERCON, &con->flags)) + return false; + return atomic_read(&con->writequeue_cnt); } static void dlm_con_init(struct connection *con, int nodeid) { con->nodeid = nodeid; - INIT_LIST_HEAD(&con->writequeue); - spin_lock_init(&con->writequeue_lock); - atomic_set(&con->writequeue_cnt, 0); - INIT_DELAYED_WORK(&con->swork, process_send_sockets); INIT_WORK(&con->rwork, process_recv_sockets); INIT_WORK(&con->cwork, process_close_sockets); init_waitqueue_head(&con->shutdown_wait); @@ -320,7 +342,12 @@ static struct connection *nodeid2con(int nodeid, gfp_t alloc) mutex_init(&con->rwork_lock); mutex_init(&con->swork_lock); + mutex_init(&con->wq_alloc); + INIT_LIST_HEAD(&con->writequeue); + spin_lock_init(&con->writequeue_lock); + atomic_set(&con->writequeue_cnt, 0); + INIT_DELAYED_WORK(&con->swork, process_send_sockets); mutex_init(&con->process_lock); INIT_LIST_HEAD(&con->processqueue); @@ -813,20 +840,23 @@ static void close_connection(struct connection *con, bool and_other) * our policy is to start on a clean state when disconnects, we don't * know what's send/received on transport layer in this case. */ - spin_lock(&con->writequeue_lock); - if (!list_empty(&con->writequeue)) { - e = list_first_entry(&con->writequeue, struct writequeue_entry, - list); - if (e->dirty) - free_entry(e); + if (!test_bit(CF_IS_OTHERCON, &con->flags)) { + spin_lock(&con->writequeue_lock); + if (!list_empty(&con->writequeue)) { + e = list_first_entry(&con->writequeue, struct writequeue_entry, + list); + if (e->dirty) + free_entry(e); + } + spin_unlock(&con->writequeue_lock); + + con->retries = 0; + clear_bit(CF_APP_LIMITED, &con->flags); + clear_bit(CF_EOF, &con->flags); } - spin_unlock(&con->writequeue_lock); con->rx_leftover = 0; - con->retries = 0; - clear_bit(CF_APP_LIMITED, &con->flags); clear_bit(CF_CONNECTED, &con->flags); - clear_bit(CF_EOF, &con->flags); /* handling for tcp shutdown */ clear_bit(CF_SHUTDOWN, &con->flags); @@ -1544,8 +1574,6 @@ int dlm_lowcomms_close(int nodeid) set_bit(CF_CLOSE, &con->flags); cancel_io_work(con, true); clean_one_writequeue(con); - if (con->othercon) - clean_one_writequeue(con->othercon); } srcu_read_unlock(&connections_srcu, idx); @@ -1781,7 +1809,6 @@ static void free_conn(struct connection *con) hlist_del_rcu(&con->list); spin_unlock(&connections_lock); if (con->othercon) { - clean_one_writequeue(con->othercon); call_srcu(&connections_srcu, &con->othercon->rcu, connection_release); } -- 2.26.3