Skip to contentSkip to conversation

fs: dlm: introduce dlm retransmission layer

RFC PATCH New Alexander Aring 1 participant 21 messages Nov 13, 2020 – Nov 24, 2020 Distributed lock manager (DLM)

lore
Share & tools
lore

Alexander Aring

author maintainer
#0 lore raw

cover fs: dlm: introduce dlm retransmission layer

Hi,

this is the final patch-series to make dlm reliable when reconnection occurs. You can easily generate a couple of reconnections by running:

tcpkill -9 -i $IFACE port 21064

on your own to test these patches. At some time dlm will detect message drops and will retransmit messages if necessary. It introduces a new dlm protocol behaviour and increases the dlm protocol version. I tested it with SCTP as well and tried to be backwardscompatible with dlm protocol version 3.1. However I don't recommend at all to not mix these versions in a setup since dlm version 3.2 fixes long-term issues.

- Alex

Alexander Aring (16):
fs: dlm: add errno handling to check callback
fs: dlm: add check if dlm is currently running
fs: dlm: add check for minimum allocation length
fs: dlm: public utils header utils
fs: dlm: use GFP_ZERO for page buffer
fs: dlm: simplify writequeue handling
fs: dlm: add more midcomms hooks
fs: dlm: make buffer handling per msg
fs: dlm: make new buffer handling softirq ready
fs: dlm: add functionality to retransmit a message
fs: dlm: move out some hash functionality
fs: dlm: remove unaligned memory access handling
fs: dlm: check on minimum header size
fs: dlm: add union in dlm header for lockspace id
fs: dlm: add reliable connection if reconnect
fs: dlm: don't allow half transmitted messages

 fs/dlm/config.c       |   60 ++-
 fs/dlm/dlm_internal.h |   32 +-
 fs/dlm/lock.c         |   16 +-
 fs/dlm/lockspace.c    |    5 +-
 fs/dlm/lowcomms.c     |  273 +++++++---
 fs/dlm/lowcomms.h     |   25 +-
 fs/dlm/member.c       |   16 +
 fs/dlm/member.h       |    1 +
 fs/dlm/midcomms.c     | 1094 +++++++++++++++++++++++++++++++++++++++--
 fs/dlm/midcomms.h     |   10 +
 fs/dlm/rcom.c         |   61 ++-
 fs/dlm/recoverd.c     |    4 +
 fs/dlm/util.c         |   10 +-
 fs/dlm/util.h         |    2 +
 14 files changed, 1438 insertions(+), 171 deletions(-)
Signature · 1 line
2.26.2
author maintainer
#1 lore raw

RFC PATCH 1/16 fs: dlm: add errno handling to check callback

This allows to return invidiual errno values for the config attribute check callback instead of returning invalid argument only.

---
 fs/dlm/config.c | 23 ++++++++++++++++-------
 1 file changed, 16 insertions(+), 7 deletions(-)
fs/dlm/config.c +16 -7
index 49c5f9407098..73e6643903af 100644
@@ -125,7 +125,7 @@ static ssize_t cluster_cluster_name_store(struct config_item *item,
 CONFIGFS_ATTR(cluster_, cluster_name);
 
 static ssize_t cluster_set(struct dlm_cluster *cl, unsigned int *cl_field,
-			   int *info_field, bool (*check_cb)(unsigned int x),
+			   int *info_field, int (*check_cb)(unsigned int x),
 			   const char *buf, size_t len)
 {
 	unsigned int x;
@@ -137,8 +137,11 @@ static ssize_t cluster_set(struct dlm_cluster *cl, unsigned int *cl_field,
 	if (rc)
 		return rc;
 
-	if (check_cb && check_cb(x))
-		return -EINVAL;
+	if (check_cb) {
+		rc = check_cb(x);
+		if (rc)
+			return rc;
+	}
 
 	*cl_field = x;
 	*info_field = x;
@@ -161,14 +164,20 @@ static ssize_t cluster_##name##_show(struct config_item *item, char *buf)     \
 }                                                                             \
 CONFIGFS_ATTR(cluster_, name);
 
-static bool dlm_check_zero(unsigned int x)
+static int dlm_check_zero(unsigned int x)
 {
-	return !x;
+	if (!x)
+		return -EINVAL;
+
+	return 0;
 }
 
-static bool dlm_check_buffer_size(unsigned int x)
+static int dlm_check_buffer_size(unsigned int x)
 {
-	return (x < DEFAULT_BUFFER_SIZE);
+	if (x < DEFAULT_BUFFER_SIZE)
+		return -EINVAL;
+
+	return 0;
 }
 
 CLUSTER_ATTR(tcp_port, dlm_check_zero);
Signature · 1 line
2.26.2
author maintainer
#2 lore raw

RFC PATCH 2/16 fs: dlm: add check if dlm is currently running

In reply to Alexander Aring

This patch adds checks for dlm config attributes regarding to protocol parameters as it makes only sense to change them when dlm is not running. It also adds a check for valid protocol specifiers and return invalid argument if they are not supported.

---
 fs/dlm/config.c   | 34 ++++++++++++++++++++++++++++++++--
 fs/dlm/lowcomms.c |  2 +-
 fs/dlm/lowcomms.h |  3 +++
 3 files changed, 36 insertions(+), 3 deletions(-)
fs/dlm/config.c +32 -2
index 73e6643903af..ab26cf135710 100644
@@ -164,6 +164,36 @@ static ssize_t cluster_##name##_show(struct config_item *item, char *buf)     \
 }                                                                             \
 CONFIGFS_ATTR(cluster_, name);
 
+static int dlm_check_protocol_and_dlm_running(unsigned int x)
+{
+	switch (x) {
+	case 0:
+		/* TCP */
+		break;
+	case 1:
+		/* SCTP */
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	if (dlm_allow_conn)
+		return -EBUSY;
+
+	return 0;
+}
+
+static int dlm_check_zero_and_dlm_running(unsigned int x)
+{
+	if (!x)
+		return -EINVAL;
+
+	if (dlm_allow_conn)
+		return -EBUSY;
+
+	return 0;
+}
+
 static int dlm_check_zero(unsigned int x)
 {
 	if (!x)
@@ -180,7 +210,7 @@ static int dlm_check_buffer_size(unsigned int x)
 	return 0;
 }
 
-CLUSTER_ATTR(tcp_port, dlm_check_zero);
+CLUSTER_ATTR(tcp_port, dlm_check_zero_and_dlm_running);
 CLUSTER_ATTR(buffer_size, dlm_check_buffer_size);
 CLUSTER_ATTR(rsbtbl_size, dlm_check_zero);
 CLUSTER_ATTR(recover_timer, dlm_check_zero);
@@ -188,7 +218,7 @@ CLUSTER_ATTR(toss_secs, dlm_check_zero);
 CLUSTER_ATTR(scan_secs, dlm_check_zero);
 CLUSTER_ATTR(log_debug, NULL);
 CLUSTER_ATTR(log_info, NULL);
-CLUSTER_ATTR(protocol, NULL);
+CLUSTER_ATTR(protocol, dlm_check_protocol_and_dlm_running);
 CLUSTER_ATTR(mark, NULL);
 CLUSTER_ATTR(timewarn_cs, dlm_check_zero);
 CLUSTER_ATTR(waitwarn_us, NULL);
fs/dlm/lowcomms.c +1 -1
index 372c34ff8594..ffea14c50d46 100644
@@ -134,7 +134,7 @@ static DEFINE_SPINLOCK(dlm_node_addrs_spin);
 static struct listen_connection listen_con;
 static struct sockaddr_storage *dlm_local_addr[DLM_MAX_ADDR_COUNT];
 static int dlm_local_count;
-static int dlm_allow_conn;
+int dlm_allow_conn;
 
 /* Work queues */
 static struct workqueue_struct *recv_workqueue;
fs/dlm/lowcomms.h +3 -0
index 0918f9376489..f74888ed43b4 100644
@@ -14,6 +14,9 @@
 
 #define LOWCOMMS_MAX_TX_BUFFER_LEN	4096
 
+/* switch to check if dlm is running */
+extern int dlm_allow_conn;
+
 int dlm_lowcomms_start(void);
 void dlm_lowcomms_stop(void);
 void dlm_lowcomms_exit(void);
Signature · 1 line
2.26.2
author maintainer
#3 lore raw

RFC PATCH 3/16 fs: dlm: add check for minimum allocation length

In reply to Alexander Aring

While running tcpkill I experienced invalid header length values while receiving to check that a node doesn't try to send a invalid dlm message we also check on applications minimum allocation limit.

---
 fs/dlm/lowcomms.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)
fs/dlm/lowcomms.c +2 -1
index ffea14c50d46..3ab56272ee2d 100644
@@ -1355,7 +1355,8 @@ void *dlm_lowcomms_get_buffer(int nodeid, int len, gfp_t allocation, char **ppc)
 	struct writequeue_entry *e;
 	int offset = 0;
 
-	if (len > LOWCOMMS_MAX_TX_BUFFER_LEN) {
+	if (len > LOWCOMMS_MAX_TX_BUFFER_LEN ||
+	    len < sizeof(struct dlm_header)) {
 		BUILD_BUG_ON(PAGE_SIZE < LOWCOMMS_MAX_TX_BUFFER_LEN);
 		log_print("failed to allocate a buffer of size %d", len);
 		return NULL;
Signature · 1 line
2.26.2
author maintainer
#4 lore raw

RFC PATCH 4/16 fs: dlm: public utils header utils

In reply to Alexander Aring

This patch allows to use header_out() and header_in() outside of dlm util functionality.

---
 fs/dlm/util.c | 4 ++--
 fs/dlm/util.h | 2 ++
 2 files changed, 4 insertions(+), 2 deletions(-)
fs/dlm/util.c +2 -2
index cfd0d00b19ae..74a8c5bfe9b5 100644
@@ -20,7 +20,7 @@
 #define DLM_ERRNO_ETIMEDOUT	       110
 #define DLM_ERRNO_EINPROGRESS	       115
 
-static void header_out(struct dlm_header *hd)
+void header_out(struct dlm_header *hd)
 {
 	hd->h_version		= cpu_to_le32(hd->h_version);
 	hd->h_lockspace		= cpu_to_le32(hd->h_lockspace);
@@ -28,7 +28,7 @@ static void header_out(struct dlm_header *hd)
 	hd->h_length		= cpu_to_le16(hd->h_length);
 }
 
-static void header_in(struct dlm_header *hd)
+void header_in(struct dlm_header *hd)
 {
 	hd->h_version		= le32_to_cpu(hd->h_version);
 	hd->h_lockspace		= le32_to_cpu(hd->h_lockspace);
fs/dlm/util.h +2 -0
index cc719ca9397e..d46f23c7a6a0 100644
@@ -15,6 +15,8 @@ void dlm_message_out(struct dlm_message *ms);
 void dlm_message_in(struct dlm_message *ms);
 void dlm_rcom_out(struct dlm_rcom *rc);
 void dlm_rcom_in(struct dlm_rcom *rc);
+void header_out(struct dlm_header *hd);
+void header_in(struct dlm_header *hd);
 
 #endif
 
Signature · 1 line
2.26.2
author maintainer
#5 lore raw

RFC PATCH 5/16 fs: dlm: use GFP_ZERO for page buffer

In reply to Alexander Aring

This patch uses GFP_ZERO for allocate a page for the internal dlm sending buffer allocator instead of calling memset zero after every allocation. An already allocated space will never be reused again.

---
 fs/dlm/lock.c     | 2 --
 fs/dlm/lowcomms.c | 2 +-
 fs/dlm/rcom.c     | 2 --
 3 files changed, 1 insertion(+), 5 deletions(-)
fs/dlm/lock.c +0 -2
index 002123efc6b0..b93df39d0915 100644
@@ -3541,8 +3541,6 @@ static int _create_message(struct dlm_ls *ls, int mb_len,
 	if (!mh)
 		return -ENOBUFS;
 
-	memset(mb, 0, mb_len);
-
 	ms = (struct dlm_message *) mb;
 
 	ms->m_header.h_version = (DLM_HEADER_MAJOR | DLM_HEADER_MINOR);
fs/dlm/lowcomms.c +1 -1
index 3ab56272ee2d..6d3183ea97f8 100644
@@ -1334,7 +1334,7 @@ static struct writequeue_entry *new_writequeue_entry(struct connection *con,
 	if (!entry)
 		return NULL;
 
-	entry->page = alloc_page(allocation);
+	entry->page = alloc_page(allocation | __GFP_ZERO);
 	if (!entry->page) {
 		kfree(entry);
 		return NULL;
fs/dlm/rcom.c +0 -2
index 73ddee5159d7..f5b1bd65728d 100644
@@ -41,7 +41,6 @@ static int create_rcom(struct dlm_ls *ls, int to_nodeid, int type, int len,
 			  to_nodeid, type, len);
 		return -ENOBUFS;
 	}
-	memset(mb, 0, mb_len);
 
 	rc = (struct dlm_rcom *) mb;
 
@@ -462,7 +461,6 @@ int dlm_send_ls_not_ready(int nodeid, struct dlm_rcom *rc_in)
 	mh = dlm_lowcomms_get_buffer(nodeid, mb_len, GFP_NOFS, &mb);
 	if (!mh)
 		return -ENOBUFS;
-	memset(mb, 0, mb_len);
 
 	rc = (struct dlm_rcom *) mb;
 
Signature · 1 line
2.26.2
author maintainer
#6 lore raw

RFC PATCH 6/16 fs: dlm: simplify writequeue handling

In reply to Alexander Aring

This patch cleans up the current dlm sending allocator handling by using some named macros, list functionality and removes some goto.

---
 fs/dlm/lowcomms.c | 83 ++++++++++++++++++++++++-----------------------
 1 file changed, 43 insertions(+), 40 deletions(-)
fs/dlm/lowcomms.c +43 -40
index 6d3183ea97f8..2fc1198105ac 100644
@@ -102,6 +102,9 @@ struct listen_connection {
 	struct work_struct rwork;
 };
 
+#define DLM_WQ_REMAIN_BYTES(e) (PAGE_SIZE - e->end)
+#define DLM_WQ_LENGTH_BYTES(e) (e->end - e->offset)
+
 /* An entry waiting to be sent */
 struct writequeue_entry {
 	struct list_head list;
@@ -1330,7 +1333,7 @@ static struct writequeue_entry *new_writequeue_entry(struct connection *con,
 {
 	struct writequeue_entry *entry;
 
-	entry = kmalloc(sizeof(struct writequeue_entry), allocation);
+	entry = kzalloc(sizeof(*entry), allocation);
 	if (!entry)
 		return NULL;
 
@@ -1340,20 +1343,48 @@ static struct writequeue_entry *new_writequeue_entry(struct connection *con,
 		return NULL;
 	}
 
-	entry->offset = 0;
-	entry->len = 0;
-	entry->end = 0;
-	entry->users = 0;
 	entry->con = con;
+	entry->users = 1;
 
 	return entry;
 }
 
+static struct writequeue_entry *new_wq_entry(struct connection *con, int len,
+					     gfp_t allocation, char **ppc)
+{
+	struct writequeue_entry *e;
+
+	spin_lock(&con->writequeue_lock);
+	if (!list_empty(&con->writequeue)) {
+		e = list_last_entry(&con->writequeue, struct writequeue_entry, list);
+		if (DLM_WQ_REMAIN_BYTES(e) >= len) {
+			*ppc = page_address(e->page) + e->end;
+			e->end += len;
+			e->users++;
+			spin_unlock(&con->writequeue_lock);
+
+			return e;
+		}
+	}
+	spin_unlock(&con->writequeue_lock);
+
+	e = new_writequeue_entry(con, allocation);
+	if (!e)
+		return NULL;
+
+	*ppc = page_address(e->page);
+	e->end += len;
+
+	spin_lock(&con->writequeue_lock);
+	list_add_tail(&e->list, &con->writequeue);
+	spin_unlock(&con->writequeue_lock);
+
+	return e;
+};
+
 void *dlm_lowcomms_get_buffer(int nodeid, int len, gfp_t allocation, char **ppc)
 {
 	struct connection *con;
-	struct writequeue_entry *e;
-	int offset = 0;
 
 	if (len > LOWCOMMS_MAX_TX_BUFFER_LEN ||
 	    len < sizeof(struct dlm_header)) {
@@ -1366,35 +1397,7 @@ void *dlm_lowcomms_get_buffer(int nodeid, int len, gfp_t allocation, char **ppc)
 	if (!con)
 		return NULL;
 
-	spin_lock(&con->writequeue_lock);
-	e = list_entry(con->writequeue.prev, struct writequeue_entry, list);
-	if ((&e->list == &con->writequeue) ||
-	    (PAGE_SIZE - e->end < len)) {
-		e = NULL;
-	} else {
-		offset = e->end;
-		e->end += len;
-		e->users++;
-	}
-	spin_unlock(&con->writequeue_lock);
-
-	if (e) {
-	got_one:
-		*ppc = page_address(e->page) + offset;
-		return e;
-	}
-
-	e = new_writequeue_entry(con, allocation);
-	if (e) {
-		spin_lock(&con->writequeue_lock);
-		offset = e->end;
-		e->end += len;
-		e->users++;
-		list_add_tail(&e->list, &con->writequeue);
-		spin_unlock(&con->writequeue_lock);
-		goto got_one;
-	}
-	return NULL;
+	return new_wq_entry(con, len, allocation, ppc);
 }
 
 void dlm_lowcomms_commit_buffer(void *mh)
@@ -1407,7 +1410,8 @@ void dlm_lowcomms_commit_buffer(void *mh)
 	users = --e->users;
 	if (users)
 		goto out;
-	e->len = e->end - e->offset;
+
+	e->len = DLM_WQ_LENGTH_BYTES(e);
 	spin_unlock(&con->writequeue_lock);
 
 	queue_work(send_workqueue, &con->swork);
@@ -1433,11 +1437,10 @@ static void send_to_sock(struct connection *con)
 
 	spin_lock(&con->writequeue_lock);
 	for (;;) {
-		e = list_entry(con->writequeue.next, struct writequeue_entry,
-			       list);
-		if ((struct list_head *) e == &con->writequeue)
+		if (list_empty(&con->writequeue))
 			break;
 
+		e = list_first_entry(&con->writequeue, struct writequeue_entry, list);
 		len = e->len;
 		offset = e->offset;
 		BUG_ON(len == 0 && e->users == 0);
Signature · 1 line
2.26.2
author maintainer
#7 lore raw

RFC PATCH 7/16 fs: dlm: add more midcomms hooks

In reply to Alexander Aring

This patch prepares hooks to redirect to the midcomms layer which will be used by the midcomms retransmit handling.

There exists the new concept of stateless buffers allocation and commits. This can be used to bypass the midcomms retransmit handling. It is used by RCOM_STATUS and RCOM_NAMES messages, because they have their own ping-like retransmit handling. As well these two messages will be used to determine the DLM version per node, because these two messages are per observation the first messages which are exchanged.

The midcomms_remove_member() hook should be called when there is nothing to send to the other node and the other node is still capable to transmit dlm messages to the other node which called midcomms_remove_member(). I experienced that the dlm protocol has a lack of support for synchronize this event on protocol level. The result was that there was still something to transmit but the other node was already gone. This hook can be used to provide such synchronization. Although I am not totally sure about the placement of this hook, I did not observed issues yet when providing such synchronization on protocol layer.

---
 fs/dlm/config.c    |  3 ++-
 fs/dlm/lock.c      |  6 ++---
 fs/dlm/lockspace.c |  5 +++--
 fs/dlm/member.c    | 16 ++++++++++++++
 fs/dlm/member.h    |  1 +
 fs/dlm/midcomms.c  | 44 +++++++++++++++++++++++++++++++++++++
 fs/dlm/midcomms.h  | 10 +++++++++
 fs/dlm/rcom.c      | 55 +++++++++++++++++++++++++++-------------------
 fs/dlm/recoverd.c  |  4 ++++
 9 files changed, 116 insertions(+), 28 deletions(-)
fs/dlm/config.c +2 -1
index ab26cf135710..ba8b1f104df3 100644
@@ -20,6 +20,7 @@
 #include <net/sock.h>
 
 #include "config.h"
+#include "midcomms.h"
 #include "lowcomms.h"
 
 /*
@@ -532,7 +533,7 @@ static void drop_comm(struct config_group *g, struct config_item *i)
 	struct dlm_comm *cm = config_item_to_comm(i);
 	if (local_comm == cm)
 		local_comm = NULL;
-	dlm_lowcomms_close(cm->nodeid);
+	dlm_midcomms_close(cm->nodeid);
 	while (cm->addr_count--)
 		kfree(cm->addr[cm->addr_count]);
 	config_item_put(i);
fs/dlm/lock.c +3 -3
index b93df39d0915..b3fd823009f4 100644
@@ -59,7 +59,7 @@
 #include "dlm_internal.h"
 #include <linux/dlm_device.h>
 #include "memory.h"
-#include "lowcomms.h"
+#include "midcomms.h"
 #include "requestqueue.h"
 #include "util.h"
 #include "dir.h"
@@ -3537,7 +3537,7 @@ static int _create_message(struct dlm_ls *ls, int mb_len,
 	   pass into lowcomms_commit and a message buffer (mb) that we
 	   write our data into */
 
-	mh = dlm_lowcomms_get_buffer(to_nodeid, mb_len, GFP_NOFS, &mb);
+	mh = dlm_midcomms_get_buffer(to_nodeid, mb_len, GFP_NOFS, &mb);
 	if (!mh)
 		return -ENOBUFS;
 
@@ -3589,7 +3589,7 @@ static int create_message(struct dlm_rsb *r, struct dlm_lkb *lkb,
 static int send_message(struct dlm_mhandle *mh, struct dlm_message *ms)
 {
 	dlm_message_out(ms);
-	dlm_lowcomms_commit_buffer(mh);
+	dlm_midcomms_commit_buffer(mh);
 	return 0;
 }
 
fs/dlm/lockspace.c +3 -2
index 561dcad08ad6..c9e0f5ac9f9a 100644
@@ -16,6 +16,7 @@
 #include "member.h"
 #include "recoverd.h"
 #include "dir.h"
+#include "midcomms.h"
 #include "lowcomms.h"
 #include "config.h"
 #include "memory.h"
@@ -390,7 +391,7 @@ static int threads_start(void)
 	}
 
 	/* Thread for sending/receiving messages for all lockspace's */
-	error = dlm_lowcomms_start();
+	error = dlm_midcomms_start();
 	if (error) {
 		log_print("cannot start dlm lowcomms %d", error);
 		goto scand_fail;
@@ -407,7 +408,7 @@ static int threads_start(void)
 static void threads_stop(void)
 {
 	dlm_scand_stop();
-	dlm_lowcomms_stop();
+	dlm_midcomms_stop();
 }
 
 static int new_lockspace(const char *name, const char *cluster,
fs/dlm/member.c +16 -0
index ceef3f2074ff..8291566766f3 100644
@@ -15,6 +15,7 @@
 #include "recover.h"
 #include "rcom.h"
 #include "config.h"
+#include "midcomms.h"
 #include "lowcomms.h"
 
 int dlm_slots_version(struct dlm_header *h)
@@ -521,6 +522,20 @@ static struct dlm_config_node *find_config_node(struct dlm_recover *rv,
 	return NULL;
 }
 
+void dlm_flush_removed_members(struct dlm_ls *ls, struct dlm_recover *rv)
+{
+	const struct dlm_config_node *node;
+	const struct dlm_member *memb;
+
+	list_for_each_entry(memb, &ls->ls_nodes, list) {
+		node = find_config_node(rv, memb->nodeid);
+		if (node && !node->new)
+			continue;
+
+		midcomms_remove_member(memb->nodeid);
+	}
+}
+
 int dlm_recover_members(struct dlm_ls *ls, struct dlm_recover *rv, int *neg_out)
 {
 	struct dlm_member *memb, *safe;
@@ -563,6 +578,7 @@ int dlm_recover_members(struct dlm_ls *ls, struct dlm_recover *rv, int *neg_out)
 		if (dlm_is_member(ls, node->nodeid))
 			continue;
 		dlm_add_member(ls, node);
+		midcomms_add_member(node->nodeid);
 		log_rinfo(ls, "add member %d", node->nodeid);
 	}
 
fs/dlm/member.h +1 -0
index 433b2fac9f4a..3a6dd2324283 100644
@@ -26,6 +26,7 @@ int dlm_slots_copy_in(struct dlm_ls *ls);
 int dlm_slots_assign(struct dlm_ls *ls, int *num_slots, int *slots_size,
 		     struct dlm_slot **slots_out, uint32_t *gen_out);
 void dlm_lsop_recover_done(struct dlm_ls *ls);
+void dlm_flush_removed_members(struct dlm_ls *ls, struct dlm_recover *rv);
 
 #endif                          /* __MEMBER_DOT_H__ */
 
fs/dlm/midcomms.c +44 -0

50 lines not on this page · raw message

fs/dlm/midcomms.h +10 -0
index 61e90a921849..134c57430cdd 100644
@@ -13,6 +13,16 @@
 #define __MIDCOMMS_DOT_H__
 
 int dlm_process_incoming_buffer(int nodeid, unsigned char *buf, int buflen);
+void *dlm_midcomms_get_buffer(int nodeid, int len, gfp_t allocation, char **ppc);
+void dlm_midcomms_commit_buffer(void *mh);
+void *dlm_midcomms_stateless_get_buffer(int nodeid, int len, gfp_t allocation,
+					char **ppc);
+void dlm_midcomms_stateless_commit_buffer(void *mh);
+void midcomms_add_member(int nodeid);
+void midcomms_remove_member(int nodeid);
+int dlm_midcomms_close(int nodeid);
+int dlm_midcomms_start(void);
+void dlm_midcomms_stop(void);
 
 #endif				/* __MIDCOMMS_DOT_H__ */
 
fs/dlm/rcom.c +33 -22

182 lines not on this page · raw message

fs/dlm/recoverd.c +4 -0
index 85e245392715..763d129aa741 100644
@@ -16,6 +16,7 @@
 #include "ast.h"
 #include "recover.h"
 #include "lowcomms.h"
+#include "midcomms.h"
 #include "lock.h"
 #include "requestqueue.h"
 #include "recoverd.h"
@@ -53,6 +54,8 @@ static int ls_recover(struct dlm_ls *ls, struct dlm_recover *rv)
 	unsigned long start;
 	int error, neg = 0;
 
+	dlm_flush_removed_members(ls, rv);
+
 	log_rinfo(ls, "dlm_recover %llu", (unsigned long long)rv->seq);
 
 	mutex_lock(&ls->ls_recoverd_active);
@@ -239,6 +242,7 @@ static int ls_recover(struct dlm_ls *ls, struct dlm_recover *rv)
 	mutex_unlock(&ls->ls_recoverd_active);
 
 	dlm_lsop_recover_done(ls);
+
 	return 0;
 
  fail:
Signature · 1 line
2.26.2
author maintainer
#8 lore raw

RFC PATCH 8/16 fs: dlm: make buffer handling per msg

In reply to Alexander Aring

This patch makes the void pointer handle for lowcomms functionality per message and not per page allocation entry. A refcount handling for the handle was added to keep the message alive until the user doesn't need it anymore.

There exists now a per message callback which will be called when allocating a new buffer. This callback will be guaranted to be called according the order of the sending buffer, which can be used that the caller increments a sequence number.

---
 fs/dlm/lowcomms.c | 100 +++++++++++++++++++++++++++++++++++++++++-----
 fs/dlm/lowcomms.h |   5 ++-
 fs/dlm/midcomms.c |   8 +++-
 3 files changed, 101 insertions(+), 12 deletions(-)
fs/dlm/lowcomms.c +91 -9
index 2fc1198105ac..3e1ddcb3f69b 100644
@@ -114,6 +114,17 @@ struct writequeue_entry {
 	int end;
 	int users;
 	struct connection *con;
+	struct list_head msgs;
+	struct kref ref;
+};
+
+struct dlm_msg {
+	struct writequeue_entry *entry;
+	void *ppc;
+	int len;
+
+	struct list_head list;
+	struct kref ref;
 };
 
 struct dlm_node_addr {
@@ -954,12 +965,36 @@ static int accept_from_sock(struct listen_connection *con)
 	return result;
 }
 
-static void free_entry(struct writequeue_entry *e)
+static void dlm_page_release(struct kref *kref)
 {
+	struct writequeue_entry *e = container_of(kref, struct writequeue_entry,
+						  ref);
+
 	__free_page(e->page);
 	kfree(e);
 }
 
+static void dlm_msg_release(struct kref *kref)
+{
+	struct dlm_msg *msg = container_of(kref, struct dlm_msg, ref);
+
+	kref_put(&msg->entry->ref, dlm_page_release);
+	kfree(msg);
+}
+
+static void free_entry(struct writequeue_entry *e)
+{
+	struct dlm_msg *msg, *tmp;
+
+	list_for_each_entry_safe(msg, tmp, &e->msgs, list) {
+		list_del(&msg->list);
+		kref_put(&msg->ref, dlm_msg_release);
+	}
+
+	list_del(&e->list);
+	kref_put(&e->ref, dlm_page_release);
+}
+
 /*
  * writequeue_entry_complete - try to delete and free write queue entry
  * @e: write queue entry to try to delete
@@ -972,10 +1007,8 @@ static void writequeue_entry_complete(struct writequeue_entry *e, int completed)
 	e->offset += completed;
 	e->len -= completed;
 
-	if (e->len == 0 && e->users == 0) {
-		list_del(&e->list);
+	if (e->len == 0 && e->users == 0)
 		free_entry(e);
-	}
 }
 
 /*
@@ -1345,12 +1378,16 @@ static struct writequeue_entry *new_writequeue_entry(struct connection *con,
 
 	entry->con = con;
 	entry->users = 1;
+	kref_init(&entry->ref);
+	INIT_LIST_HEAD(&entry->msgs);
 
 	return entry;
 }
 
 static struct writequeue_entry *new_wq_entry(struct connection *con, int len,
-					     gfp_t allocation, char **ppc)
+					     gfp_t allocation, char **ppc,
+					     void (*cb)(void *buf, void *priv),
+					     void *priv)
 {
 	struct writequeue_entry *e;
 
@@ -1358,7 +1395,12 @@ static struct writequeue_entry *new_wq_entry(struct connection *con, int len,
 	if (!list_empty(&con->writequeue)) {
 		e = list_last_entry(&con->writequeue, struct writequeue_entry, list);
 		if (DLM_WQ_REMAIN_BYTES(e) >= len) {
+			kref_get(&e->ref);
+
 			*ppc = page_address(e->page) + e->end;
+			if (cb)
+				cb(*ppc, priv);
+
 			e->end += len;
 			e->users++;
 			spin_unlock(&con->writequeue_lock);
@@ -1372,19 +1414,26 @@ static struct writequeue_entry *new_wq_entry(struct connection *con, int len,
 	if (!e)
 		return NULL;
 
+	kref_get(&e->ref);
 	*ppc = page_address(e->page);
 	e->end += len;
 
 	spin_lock(&con->writequeue_lock);
+	if (cb)
+		cb(*ppc, priv);
+
 	list_add_tail(&e->list, &con->writequeue);
 	spin_unlock(&con->writequeue_lock);
 
 	return e;
 };
 
-void *dlm_lowcomms_get_buffer(int nodeid, int len, gfp_t allocation, char **ppc)
+void *dlm_lowcomms_new_buffer(int nodeid, int len, gfp_t allocation, char **ppc,
+			      void (*cb)(void *buf, void *priv), void *priv)
 {
+	struct writequeue_entry *e;
 	struct connection *con;
+	struct dlm_msg *msg;
 
 	if (len > LOWCOMMS_MAX_TX_BUFFER_LEN ||
 	    len < sizeof(struct dlm_header)) {
@@ -1397,16 +1446,36 @@ void *dlm_lowcomms_get_buffer(int nodeid, int len, gfp_t allocation, char **ppc)
 	if (!con)
 		return NULL;
 
-	return new_wq_entry(con, len, allocation, ppc);
+	msg = kzalloc(sizeof(*msg), allocation);
+	if (!msg)
+		return NULL;
+
+	kref_init(&msg->ref);
+
+	e = new_wq_entry(con, len, allocation, ppc, cb, priv);
+	if (!e) {
+		kfree(msg);
+		return NULL;
+	}
+
+	msg->ppc = *ppc;
+	msg->len = len;
+	msg->entry = e;
+
+	return msg;
 }
 
 void dlm_lowcomms_commit_buffer(void *mh)
 {
-	struct writequeue_entry *e = (struct writequeue_entry *)mh;
+	struct dlm_msg *msg = mh;
+	struct writequeue_entry *e = msg->entry;
 	struct connection *con = e->con;
 	int users;
 
 	spin_lock(&con->writequeue_lock);
+	list_add(&msg->list, &e->msgs);
+	kref_get(&msg->ref);
+
 	users = --e->users;
 	if (users)
 		goto out;
@@ -1422,6 +1491,20 @@ void dlm_lowcomms_commit_buffer(void *mh)
 	return;
 }
 
+void dlm_lowcomms_put_buffer(void *mh)
+{
+	struct dlm_msg *msg = mh;
+
+	kref_put(&msg->ref, dlm_msg_release);
+}
+
+void dlm_lowcomms_get_buffer(void *mh)
+{
+	struct dlm_msg *msg = mh;
+
+	kref_get(&msg->ref);
+}
+
 /* Send a message */
 static void send_to_sock(struct connection *con)
 {
@@ -1500,7 +1583,6 @@ static void clean_one_writequeue(struct connection *con)
 
 	spin_lock(&con->writequeue_lock);
 	list_for_each_entry_safe(e, safe, &con->writequeue, list) {
-		list_del(&e->list);
 		free_entry(e);
 	}
 	spin_unlock(&con->writequeue_lock);
fs/dlm/lowcomms.h +4 -1
index f74888ed43b4..6417c5fca215 100644
@@ -21,10 +21,13 @@ int dlm_lowcomms_start(void);
 void dlm_lowcomms_stop(void);
 void dlm_lowcomms_exit(void);
 int dlm_lowcomms_close(int nodeid);
-void *dlm_lowcomms_get_buffer(int nodeid, int len, gfp_t allocation, char **ppc);
+void *dlm_lowcomms_new_buffer(int nodeid, int len, gfp_t allocation, char **ppc,
+			      void (*cb)(void *buf, void *priv), void *priv);
 void dlm_lowcomms_commit_buffer(void *mh);
 int dlm_lowcomms_connect_node(int nodeid);
 int dlm_lowcomms_addr(int nodeid, struct sockaddr_storage *addr, int len);
+void dlm_lowcomms_put_buffer(void *mh);
+void dlm_lowcomms_get_buffer(void *mh);
 
 #endif				/* __LOWCOMMS_DOT_H__ */
 
fs/dlm/midcomms.c +6 -2
index 717a3ed34f6a..e058e017c77d 100644
@@ -32,23 +32,27 @@
 
 void *dlm_midcomms_get_buffer(int nodeid, int len, gfp_t allocation, char **ppc)
 {
-	return dlm_lowcomms_get_buffer(nodeid, len, allocation, ppc);
+	return dlm_lowcomms_new_buffer(nodeid, len, allocation, ppc, NULL,
+				       NULL);
 }
 
 void dlm_midcomms_commit_buffer(void *mh)
 {
 	dlm_lowcomms_commit_buffer(mh);
+	dlm_lowcomms_put_buffer(mh);
 }
 
 void *dlm_midcomms_stateless_get_buffer(int nodeid, int len, gfp_t allocation,
 					char **ppc)
 {
-	return dlm_lowcomms_get_buffer(nodeid, len, allocation, ppc);
+	return dlm_lowcomms_new_buffer(nodeid, len, allocation, ppc, NULL,
+				       NULL);
 }
 
 void dlm_midcomms_stateless_commit_buffer(void *mh)
 {
 	dlm_lowcomms_commit_buffer(mh);
+	dlm_lowcomms_put_buffer(mh);
 }
 
 void midcomms_add_member(int nodeid)
Signature · 1 line
2.26.2
author maintainer
#9 lore raw

RFC PATCH 9/16 fs: dlm: make new buffer handling softirq ready

In reply to Alexander Aring

This patch makes the writequeue and message handling ready to be called from a softirq by using spinlock handling to stop software interrupts on local cpu while they are hold. The coming midcomms retransmit handling will introduce a timer which is using this functionality when the timer expires.

---
 fs/dlm/lowcomms.c | 28 ++++++++++++++--------------
 1 file changed, 14 insertions(+), 14 deletions(-)
fs/dlm/lowcomms.c +14 -14
index 3e1ddcb3f69b..3c133cd2ff22 100644
@@ -1391,7 +1391,7 @@ static struct writequeue_entry *new_wq_entry(struct connection *con, int len,
 {
 	struct writequeue_entry *e;
 
-	spin_lock(&con->writequeue_lock);
+	spin_lock_bh(&con->writequeue_lock);
 	if (!list_empty(&con->writequeue)) {
 		e = list_last_entry(&con->writequeue, struct writequeue_entry, list);
 		if (DLM_WQ_REMAIN_BYTES(e) >= len) {
@@ -1403,12 +1403,12 @@ static struct writequeue_entry *new_wq_entry(struct connection *con, int len,
 
 			e->end += len;
 			e->users++;
-			spin_unlock(&con->writequeue_lock);
+			spin_unlock_bh(&con->writequeue_lock);
 
 			return e;
 		}
 	}
-	spin_unlock(&con->writequeue_lock);
+	spin_unlock_bh(&con->writequeue_lock);
 
 	e = new_writequeue_entry(con, allocation);
 	if (!e)
@@ -1418,12 +1418,12 @@ static struct writequeue_entry *new_wq_entry(struct connection *con, int len,
 	*ppc = page_address(e->page);
 	e->end += len;
 
-	spin_lock(&con->writequeue_lock);
+	spin_lock_bh(&con->writequeue_lock);
 	if (cb)
 		cb(*ppc, priv);
 
 	list_add_tail(&e->list, &con->writequeue);
-	spin_unlock(&con->writequeue_lock);
+	spin_unlock_bh(&con->writequeue_lock);
 
 	return e;
 };
@@ -1472,7 +1472,7 @@ void dlm_lowcomms_commit_buffer(void *mh)
 	struct connection *con = e->con;
 	int users;
 
-	spin_lock(&con->writequeue_lock);
+	spin_lock_bh(&con->writequeue_lock);
 	list_add(&msg->list, &e->msgs);
 	kref_get(&msg->ref);
 
@@ -1481,13 +1481,13 @@ void dlm_lowcomms_commit_buffer(void *mh)
 		goto out;
 
 	e->len = DLM_WQ_LENGTH_BYTES(e);
-	spin_unlock(&con->writequeue_lock);
+	spin_unlock_bh(&con->writequeue_lock);
 
 	queue_work(send_workqueue, &con->swork);
 	return;
 
 out:
-	spin_unlock(&con->writequeue_lock);
+	spin_unlock_bh(&con->writequeue_lock);
 	return;
 }
 
@@ -1518,7 +1518,7 @@ static void send_to_sock(struct connection *con)
 	if (con->sock == NULL)
 		goto out_connect;
 
-	spin_lock(&con->writequeue_lock);
+	spin_lock_bh(&con->writequeue_lock);
 	for (;;) {
 		if (list_empty(&con->writequeue))
 			break;
@@ -1527,7 +1527,7 @@ static void send_to_sock(struct connection *con)
 		len = e->len;
 		offset = e->offset;
 		BUG_ON(len == 0 && e->users == 0);
-		spin_unlock(&con->writequeue_lock);
+		spin_unlock_bh(&con->writequeue_lock);
 
 		ret = 0;
 		if (len) {
@@ -1555,10 +1555,10 @@ static void send_to_sock(struct connection *con)
 			count = 0;
 		}
 
-		spin_lock(&con->writequeue_lock);
+		spin_lock_bh(&con->writequeue_lock);
 		writequeue_entry_complete(e, ret);
 	}
-	spin_unlock(&con->writequeue_lock);
+	spin_unlock_bh(&con->writequeue_lock);
 out:
 	mutex_unlock(&con->sock_mutex);
 	return;
@@ -1581,11 +1581,11 @@ static void clean_one_writequeue(struct connection *con)
 {
 	struct writequeue_entry *e, *safe;
 
-	spin_lock(&con->writequeue_lock);
+	spin_lock_bh(&con->writequeue_lock);
 	list_for_each_entry_safe(e, safe, &con->writequeue, list) {
 		free_entry(e);
 	}
-	spin_unlock(&con->writequeue_lock);
+	spin_unlock_bh(&con->writequeue_lock);
 }
 
 /* Called from recovery when it knows that a node has
Signature · 1 line
2.26.2
author maintainer
#10 lore raw

RFC PATCH 10/16 fs: dlm: add functionality to retransmit a message

In reply to Alexander Aring

This patch introduces a irqsafe retransmit functionality for a lowcomms message handle. It's just allocates a new buffer and transmit it again. To avoid another connection look some refactor was done to make a new buffer allocation with a preexisting connection pointer.

---
 fs/dlm/lowcomms.c | 54 +++++++++++++++++++++++++++++++++++------------
 fs/dlm/lowcomms.h |  1 +
 2 files changed, 41 insertions(+), 14 deletions(-)
fs/dlm/lowcomms.c +40 -14
index 3c133cd2ff22..0ef1a1ffd301 100644
@@ -1428,24 +1428,14 @@ static struct writequeue_entry *new_wq_entry(struct connection *con, int len,
 	return e;
 };
 
-void *dlm_lowcomms_new_buffer(int nodeid, int len, gfp_t allocation, char **ppc,
-			      void (*cb)(void *buf, void *priv), void *priv)
+static void *dlm_lowcomms_new_buffer_con(struct connection *con, int len,
+					 gfp_t allocation, char **ppc,
+					 void (*cb)(void *buf, void *priv),
+					 void *priv)
 {
 	struct writequeue_entry *e;
-	struct connection *con;
 	struct dlm_msg *msg;
 
-	if (len > LOWCOMMS_MAX_TX_BUFFER_LEN ||
-	    len < sizeof(struct dlm_header)) {
-		BUILD_BUG_ON(PAGE_SIZE < LOWCOMMS_MAX_TX_BUFFER_LEN);
-		log_print("failed to allocate a buffer of size %d", len);
-		return NULL;
-	}
-
-	con = nodeid2con(nodeid, allocation);
-	if (!con)
-		return NULL;
-
 	msg = kzalloc(sizeof(*msg), allocation);
 	if (!msg)
 		return NULL;
@@ -1465,6 +1455,25 @@ void *dlm_lowcomms_new_buffer(int nodeid, int len, gfp_t allocation, char **ppc,
 	return msg;
 }
 
+void *dlm_lowcomms_new_buffer(int nodeid, int len, gfp_t allocation, char **ppc,
+			      void (*cb)(void *buf, void *priv), void *priv)
+{
+	struct connection *con;
+
+	if (len > LOWCOMMS_MAX_TX_BUFFER_LEN ||
+	    len < sizeof(struct dlm_header)) {
+		BUILD_BUG_ON(PAGE_SIZE < LOWCOMMS_MAX_TX_BUFFER_LEN);
+		log_print("failed to allocate a buffer of size %d", len);
+		return NULL;
+	}
+
+	con = nodeid2con(nodeid, allocation);
+	if (!con)
+		return NULL;
+
+	return dlm_lowcomms_new_buffer_con(con, len, GFP_ATOMIC, ppc, cb, priv);
+}
+
 void dlm_lowcomms_commit_buffer(void *mh)
 {
 	struct dlm_msg *msg = mh;
@@ -1505,6 +1514,23 @@ void dlm_lowcomms_get_buffer(void *mh)
 	kref_get(&msg->ref);
 }
 
+/* irqsafe */
+void dlm_lowcomms_resend_buffer(void *mh)
+{
+	struct dlm_msg *msg = mh;
+	void *mh_new;
+	char *ppc;
+
+	mh_new = dlm_lowcomms_new_buffer_con(msg->entry->con, msg->len, GFP_ATOMIC,
+					     &ppc, NULL, NULL);
+	if (!mh_new)
+		return;
+
+	memcpy(ppc, msg->ppc, msg->len);
+	dlm_lowcomms_commit_buffer(mh_new);
+	dlm_lowcomms_put_buffer(mh_new);
+}
+
 /* Send a message */
 static void send_to_sock(struct connection *con)
 {
fs/dlm/lowcomms.h +1 -0
index 6417c5fca215..30924b6f03e1 100644
@@ -28,6 +28,7 @@ int dlm_lowcomms_connect_node(int nodeid);
 int dlm_lowcomms_addr(int nodeid, struct sockaddr_storage *addr, int len);
 void dlm_lowcomms_put_buffer(void *mh);
 void dlm_lowcomms_get_buffer(void *mh);
+void dlm_lowcomms_resend_buffer(void *mh);
 
 #endif				/* __LOWCOMMS_DOT_H__ */
 
Signature · 1 line
2.26.2
author maintainer
#11 lore raw

RFC PATCH 11/16 fs: dlm: move out some hash functionality

In reply to Alexander Aring

This patch moves out some lowcomms hash functionality into lowcomms header to provide them to other layers like midcomms as well.

---
 fs/dlm/lowcomms.c |  9 ---------
 fs/dlm/lowcomms.h | 10 ++++++++++
 2 files changed, 10 insertions(+), 9 deletions(-)
fs/dlm/lowcomms.c +0 -9
index 0ef1a1ffd301..a90999e1f306 100644
@@ -59,7 +59,6 @@
 #include "config.h"
 
 #define NEEDED_RMEM (4*1024*1024)
-#define CONN_HASH_SIZE 32
 
 /* Number of messages to send before rescheduling */
 #define MAX_SEND_MSG_COUNT 25
@@ -165,14 +164,6 @@ static void sctp_connect_to_sock(struct connection *con);
 static void tcp_connect_to_sock(struct connection *con);
 static void dlm_tcp_shutdown(struct connection *con);
 
-/* This is deliberately very simple because most clusters have simple
-   sequential nodeids, so we should be able to go straight to a connection
-   struct in the array */
-static inline int nodeid_hash(int nodeid)
-{
-	return nodeid & (CONN_HASH_SIZE-1);
-}
-
 static struct connection *__find_con(int nodeid)
 {
 	int r, idx;
fs/dlm/lowcomms.h +10 -0
index 30924b6f03e1..8286531f9a9e 100644
@@ -13,6 +13,16 @@
 #define __LOWCOMMS_DOT_H__
 
 #define LOWCOMMS_MAX_TX_BUFFER_LEN	4096
+#define CONN_HASH_SIZE 32
+
+/* This is deliberately very simple because most clusters have simple
+ * sequential nodeids, so we should be able to go straight to a connection
+ * struct in the array
+ */
+static inline int nodeid_hash(int nodeid)
+{
+	return nodeid & (CONN_HASH_SIZE-1);
+}
 
 /* switch to check if dlm is running */
 extern int dlm_allow_conn;
Signature · 1 line
2.26.2
author maintainer
#12 lore raw

RFC PATCH 12/16 fs: dlm: remove unaligned memory access handling

In reply to Alexander Aring

This patch removes unaligned memory access handling for receiving midcomms messages. The allocated receive buffer is always memory aligned as the code shows, but each dlm message length and their structure fields are always aligned to 4 bytes addresses so it should be fine to remove this special handling.

---
 fs/dlm/midcomms.c | 12 +-----------
 1 file changed, 1 insertion(+), 11 deletions(-)
fs/dlm/midcomms.c +1 -11
index e058e017c77d..b146842be54a 100644
@@ -22,8 +22,6 @@
  * into packets and sends them to the comms layer.
  */
 
-#include <asm/unaligned.h>
-
 #include "dlm_internal.h"
 #include "lowcomms.h"
 #include "config.h"
@@ -96,7 +94,7 @@ int dlm_process_incoming_buffer(int nodeid, unsigned char *buf, int len)
 		/* no message should be more than this otherwise we
 		 * cannot deliver this message to upper layers
 		 */
-		msglen = get_unaligned_le16(&hd->h_length);
+		msglen = le16_to_cpu(hd->h_length);
 		if (msglen > DEFAULT_BUFFER_SIZE) {
 			log_print("received invalid length header: %u, will abort message parsing",
 				  msglen);
@@ -132,14 +130,6 @@ int dlm_process_incoming_buffer(int nodeid, unsigned char *buf, int len)
 			goto skip;
 		}
 
-		/* for aligned memory access, we just copy current message
-		 * to begin of the buffer which contains already parsed buffer
-		 * data and should provide align access for upper layers
-		 * because the start address of the buffer has a aligned
-		 * address. This memmove can be removed when the upperlayer
-		 * is capable of unaligned memory access.
-		 */
-		memmove(buf, ptr, msglen);
 		dlm_receive_buffer((union dlm_packet *)buf, nodeid);
 
 skip:
Signature · 1 line
2.26.2
author maintainer
#13 lore raw

RFC PATCH 12/16 fs: dlm: remove unaligned memory access handling

Hi,

Quoted 26 lines · Alexander Aring

This patch removes unaligned memory access handling for receiving midcomms messages. The allocated receive buffer is always memory aligned as the code shows, but each dlm message length and their structure fields are always aligned to 4 bytes addresses so it should be fine to remove this special handling.

---
 fs/dlm/midcomms.c | 12 +-----------
 1 file changed, 1 insertion(+), 11 deletions(-)
fs/dlm/midcomms.c +0 -2
index e058e017c77d..b146842be54a 100644
@@ -22,8 +22,6 @@
  * into packets and sends them to the comms layer.
  */
 
-#include <asm/unaligned.h>
-
 #include "dlm_internal.h"
 #include "lowcomms.h"
 #include "config.h"

fs/dlm/midcomms.c int dlm_process_incoming_buffer(int nodeid, unsigned char *buf, int len) · line 98

Quoted 6 lines
fs/dlm/midcomms.c +1 -1
@@ -96,7 +94,7 @@ int dlm_process_incoming_buffer(int nodeid, unsigned char *buf, int len)
                /* no message should be more than this otherwise we
                 * cannot deliver this message to upper layers
                 */
-               msglen = get_unaligned_le16(&hd->h_length);
+               msglen = le16_to_cpu(hd->h_length);
                if (msglen > DEFAULT_BUFFER_SIZE) {

I will change this condition to:

                if (msglen > DEFAULT_BUFFER_SIZE ||
                    msglen < sizeof(struct dlm_header) ||
                    DLM_MSGLEN_IS_NOT_ALIGNED(msglen)) {

As these must always be true for a valid dlm message, we disconnect when seeing such a message. We cannot skip it because skipping requires a valid msglen.

- Alex

author maintainer
#14 lore raw

RFC PATCH 13/16 fs: dlm: check on minimum header size

In reply to Alexander Aring

As I experienced invalid header length e.g. worst-case zero triggers an endless loop inside the receive handling, because we don't increment any consumed bytes. This patch will add also a check for the minimum size of a dlm message inside the dlm header length field.

---
 fs/dlm/midcomms.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)
fs/dlm/midcomms.c +2 -1
index b146842be54a..139d87f1ba17 100644
@@ -95,7 +95,8 @@ int dlm_process_incoming_buffer(int nodeid, unsigned char *buf, int len)
 		 * cannot deliver this message to upper layers
 		 */
 		msglen = le16_to_cpu(hd->h_length);
-		if (msglen > DEFAULT_BUFFER_SIZE) {
+		if (msglen > DEFAULT_BUFFER_SIZE ||
+		    msglen < sizeof(struct dlm_header)) {
 			log_print("received invalid length header: %u, will abort message parsing",
 				  msglen);
 			return -EBADMSG;
Signature · 1 line
2.26.2
author maintainer
#15 lore raw

RFC PATCH 14/16 fs: dlm: add union in dlm header for lockspace id

In reply to Alexander Aring

This patch adds union inside the lockspace id to handle it also for another use case for a different dlm command.

---
 fs/dlm/dlm_internal.h | 5 ++++-
 fs/dlm/lock.c         | 8 ++++----
 fs/dlm/midcomms.c     | 1 -
 fs/dlm/rcom.c         | 4 ++--
 fs/dlm/util.c         | 6 ++++--
 5 files changed, 14 insertions(+), 10 deletions(-)
fs/dlm/dlm_internal.h +4 -1
index 04fe9f525ac7..917de7367a32 100644
@@ -377,7 +377,10 @@ static inline int rsb_flag(struct dlm_rsb *r, enum rsb_flags flag)
 
 struct dlm_header {
 	uint32_t		h_version;
-	uint32_t		h_lockspace;
+	union {
+		/* for DLM_MSG and DLM_RCOM */
+		uint32_t	h_lockspace;
+	} u;
 	uint32_t		h_nodeid;	/* nodeid of sender */
 	uint16_t		h_length;
 	uint8_t			h_cmd;		/* DLM_MSG, DLM_RCOM */
fs/dlm/lock.c +4 -4
index b3fd823009f4..daa5747c8556 100644
@@ -3544,7 +3544,7 @@ static int _create_message(struct dlm_ls *ls, int mb_len,
 	ms = (struct dlm_message *) mb;
 
 	ms->m_header.h_version = (DLM_HEADER_MAJOR | DLM_HEADER_MINOR);
-	ms->m_header.h_lockspace = ls->ls_global_id;
+	ms->m_header.u.h_lockspace = ls->ls_global_id;
 	ms->m_header.h_nodeid = dlm_our_nodeid();
 	ms->m_header.h_length = mb_len;
 	ms->m_header.h_cmd = DLM_MSG;
@@ -5038,16 +5038,16 @@ void dlm_receive_buffer(union dlm_packet *p, int nodeid)
 
 	if (hd->h_nodeid != nodeid) {
 		log_print("invalid h_nodeid %d from %d lockspace %x",
-			  hd->h_nodeid, nodeid, hd->h_lockspace);
+			  hd->h_nodeid, nodeid, hd->u.h_lockspace);
 		return;
 	}
 
-	ls = dlm_find_lockspace_global(hd->h_lockspace);
+	ls = dlm_find_lockspace_global(hd->u.h_lockspace);
 	if (!ls) {
 		if (dlm_config.ci_log_debug) {
 			printk_ratelimited(KERN_DEBUG "dlm: invalid lockspace "
 				"%u from %d cmd %d type %d\n",
-				hd->h_lockspace, nodeid, hd->h_cmd, type);
+				hd->u.h_lockspace, nodeid, hd->h_cmd, type);
 		}
 
 		if (hd->h_cmd == DLM_RCOM && type == DLM_RCOM_STATUS)
fs/dlm/midcomms.c +0 -1
index 139d87f1ba17..f157d8077b76 100644
@@ -141,4 +141,3 @@ int dlm_process_incoming_buffer(int nodeid, unsigned char *buf, int len)
 
 	return ret;
 }
-
fs/dlm/rcom.c +2 -2
index 7a7d4a8e4706..06f7a5f1d99d 100644
@@ -49,7 +49,7 @@ static int create_rcom(struct dlm_ls *ls, int to_nodeid, int type, int len,
 	rc = (struct dlm_rcom *) mb;
 
 	rc->rc_header.h_version = (DLM_HEADER_MAJOR | DLM_HEADER_MINOR);
-	rc->rc_header.h_lockspace = ls->ls_global_id;
+	rc->rc_header.u.h_lockspace = ls->ls_global_id;
 	rc->rc_header.h_nodeid = dlm_our_nodeid();
 	rc->rc_header.h_length = mb_len;
 	rc->rc_header.h_cmd = DLM_RCOM;
@@ -476,7 +476,7 @@ int dlm_send_ls_not_ready(int nodeid, struct dlm_rcom *rc_in)
 	rc = (struct dlm_rcom *) mb;
 
 	rc->rc_header.h_version = (DLM_HEADER_MAJOR | DLM_HEADER_MINOR);
-	rc->rc_header.h_lockspace = rc_in->rc_header.h_lockspace;
+	rc->rc_header.u.h_lockspace = rc_in->rc_header.u.h_lockspace;
 	rc->rc_header.h_nodeid = dlm_our_nodeid();
 	rc->rc_header.h_length = mb_len;
 	rc->rc_header.h_cmd = DLM_RCOM;
fs/dlm/util.c +4 -2
index 74a8c5bfe9b5..58acbcc2081a 100644
@@ -23,7 +23,8 @@
 void header_out(struct dlm_header *hd)
 {
 	hd->h_version		= cpu_to_le32(hd->h_version);
-	hd->h_lockspace		= cpu_to_le32(hd->h_lockspace);
+	/* does it for others u32 in union as well */
+	hd->u.h_lockspace	= cpu_to_le32(hd->u.h_lockspace);
 	hd->h_nodeid		= cpu_to_le32(hd->h_nodeid);
 	hd->h_length		= cpu_to_le16(hd->h_length);
 }
@@ -31,7 +32,8 @@ void header_out(struct dlm_header *hd)
 void header_in(struct dlm_header *hd)
 {
 	hd->h_version		= le32_to_cpu(hd->h_version);
-	hd->h_lockspace		= le32_to_cpu(hd->h_lockspace);
+	/* does it for others u32 in union as well */
+	hd->u.h_lockspace	= le32_to_cpu(hd->u.h_lockspace);
 	hd->h_nodeid		= le32_to_cpu(hd->h_nodeid);
 	hd->h_length		= le16_to_cpu(hd->h_length);
 }
Signature · 1 line
2.26.2
author maintainer
#16 lore raw

RFC PATCH 15/16 fs: dlm: add reliable connection if reconnect

In reply to Alexander Aring

This patch introduce to make a tcp lowcomms connection reliable even if reconnects occurs. This is done by an application layer retransmission handling and sequence numbers in dlm protocols. There are three new dlm commands:

DLM_OPTS:

This will encapsulate an exisiting dlm message (and rcom message if they don't have an own application side retransmission handling). As optional handling additional tlv's (type length fields) can be appended. This can be for example a sequence number field. However because in DLM_OPTS the lockspace field is unused and a sequence number is a mandatory field it isn't made as a tlv and we put the sequence number inside the lockspace id. The possibilty to add optional options are still there for future purposes.

DLM_ACK:

Just a dlm header to ackknowledge the receipe of a DLM_OPTS message to it's sender.

DLM_FIN:

A new DLM message to synchronize pending message to the other dlm end if the node want to disconnects. Each side waits until it receives this message and disconnects. It's important that this message has nothing to do with the application logik because it might run in a timeout if ackknowledge messages are dropped.

To explain the basic functionality take a look into the dlm_midcomms_receive_buffer() function. This function will take care that dlm messages are delivered according to their sequence numbers and request retransmission via sending ackknowledge messages. However there exists three cases:

1. sequence number is the one which is expected. That means everything is working fine. Additional there is always a check if the next message was already queued for future, this will occur when there was some messages drops before.

2. A sequence number is in the future, in this case we queue it for might future delivery, see case 1.

3. A sequence number is in the past, in this case we drop this message because it was already delivered.

To send ackknowledge we always send the sequence number which is expected, if the other node sends multiple ackknowledge for the same sequence numbers it will trigger a retransmission. In case no ackknowledge is send back, a timer with a timeout handling is running and will trigger a retranmission as well. Sending multiple acks with the same sequence or messages with the same sequence should not have any effects that breaks dlm. Only messages in the far future can break dlm, that's why important that the closing connection is right synchronized with DLM_FIN which also resets the sequence numbers.

As RCOM_STATUS and RCOM_NAMES messages are the first messages which are exchanged and they have they own retransmission handling, there exists logic that these messages must be first. If these messages arrives we store the dlm version field. This handling is on DLM 3.1 and after this patch 3.2 the same. A backwards compatibility handling has been added which seems to work on tests without tcpkill, however it's not recommended to use DLM 3.1 and 3.2 at the same time, because DLM 3.2 tries to fix long term bugs in the DLM protocol.

---
 fs/dlm/dlm_internal.h |   27 +-
 fs/dlm/lowcomms.h     |    6 +-
 fs/dlm/midcomms.c     | 1074 +++++++++++++++++++++++++++++++++++++++--
 3 files changed, 1053 insertions(+), 54 deletions(-)
fs/dlm/dlm_internal.h +25 -2
index 917de7367a32..7ae95ab59774 100644
@@ -368,18 +368,26 @@ static inline int rsb_flag(struct dlm_rsb *r, enum rsb_flags flag)
 /* dlm_header is first element of all structs sent between nodes */
 
 #define DLM_HEADER_MAJOR	0x00030000
-#define DLM_HEADER_MINOR	0x00000001
+#define DLM_HEADER_MINOR	0x00000002
+
+#define DLM_VERSION_3_1		0x00030001
+#define DLM_VERSION_3_2		0x00030002
 
 #define DLM_HEADER_SLOTS	0x00000001
 
 #define DLM_MSG			1
 #define DLM_RCOM		2
+#define DLM_OPTS		3
+#define DLM_ACK			4
+#define DLM_FIN			5
 
 struct dlm_header {
 	uint32_t		h_version;
 	union {
 		/* for DLM_MSG and DLM_RCOM */
 		uint32_t	h_lockspace;
+		/* for DLM_ACK and DLM_OPTS */
+		uint32_t	h_seq;
 	} u;
 	uint32_t		h_nodeid;	/* nodeid of sender */
 	uint16_t		h_length;
@@ -387,7 +395,6 @@ struct dlm_header {
 	uint8_t			h_pad;
 };
 
-
 #define DLM_MSG_REQUEST		1
 #define DLM_MSG_CONVERT		2
 #define DLM_MSG_UNLOCK		3
@@ -455,10 +462,26 @@ struct dlm_rcom {
 	char			rc_buf[];
 };
 
+struct dlm_opt_header {
+	uint16_t	t_type;
+	uint16_t	t_length;
+	char		t_value[];
+};
+
+/* encapsulation header */
+struct dlm_opts {
+	struct dlm_header	o_header;
+	uint8_t			o_nextcmd;
+	uint8_t			o_pad;
+	uint16_t		o_optlen;
+	char			o_opts[];
+};
+
 union dlm_packet {
 	struct dlm_header	header;		/* common to other two */
 	struct dlm_message	message;
 	struct dlm_rcom		rcom;
+	struct dlm_opts		opts;
 };
 
 #define DLM_RSF_NEED_SLOTS	0x00000001
fs/dlm/lowcomms.h +5 -1
index 8286531f9a9e..b61538e1be86 100644
@@ -12,7 +12,11 @@
 #ifndef __LOWCOMMS_DOT_H__
 #define __LOWCOMMS_DOT_H__
 
-#define LOWCOMMS_MAX_TX_BUFFER_LEN	4096
+#include "dlm_internal.h"
+
+#define DLM_MIDCOMMS_OPT_LEN		sizeof(struct dlm_opts)
+#define LOWCOMMS_MAX_TX_BUFFER_LEN	(4096 - DLM_MIDCOMMS_OPT_LEN)
+
 #define CONN_HASH_SIZE 32
 
 /* This is deliberately very simple because most clusters have simple
fs/dlm/midcomms.c +1023 -51

1,133 lines not on this page · raw message

Signature · 1 line
2.26.2
author maintainer
#17 lore raw

RFC PATCH 15/16 fs: dlm: add reliable connection if reconnect

Hi,

Quoted 22 lines · Alexander Aring

This patch introduce to make a tcp lowcomms connection reliable even if reconnects occurs. This is done by an application layer retransmission handling and sequence numbers in dlm protocols. There are three new dlm commands:

DLM_OPTS:

This will encapsulate an exisiting dlm message (and rcom message if they don't have an own application side retransmission handling). As optional handling additional tlv's (type length fields) can be appended. This can be for example a sequence number field. However because in DLM_OPTS the lockspace field is unused and a sequence number is a mandatory field it isn't made as a tlv and we put the sequence number inside the lockspace id. The possibilty to add optional options are still there for future purposes.

DLM_ACK:

Just a dlm header to ackknowledge the receipe of a DLM_OPTS message to it's sender.

Quoted 6 lines

DLM_FIN:

A new DLM message to synchronize pending message to the other dlm end if the node want to disconnects. Each side waits until it receives this message and disconnects. It's important that this message has nothing to do with the application logik because it might run in a timeout if

s/logik/logic/

Quoted 13 lines

ackknowledge messages are dropped.

To explain the basic functionality take a look into the dlm_midcomms_receive_buffer() function. This function will take care that dlm messages are delivered according to their sequence numbers and request retransmission via sending ackknowledge messages. However there exists three cases:

1. sequence number is the one which is expected. That means everything is working fine. Additional there is always a check if the next message was already queued for future, this will occur when there was some messages drops before.

Quoted 8 lines

2. A sequence number is in the future, in this case we queue it for might future delivery, see case 1.

3. A sequence number is in the past, in this case we drop this message because it was already delivered.

To send ackknowledge we always send the sequence number which is expected, if the other node sends multiple ackknowledge for the same

s/sends/receives/

Quoted 7 lines

sequence numbers it will trigger a retransmission. In case no ackknowledge is send back, a timer with a timeout handling is running and will trigger a retranmission as well. Sending multiple acks with the same sequence or messages with the same sequence should not have any effects that breaks dlm. Only messages in the far future can break dlm, that's why important that the closing connection is right synchronized with DLM_FIN which also resets the sequence numbers.

s/ackknowledge/acknowledge/ everywhere and s/retranmission/retransmission/

sorry, I will run aspell on my commit message (I thought checkpatch is doing that).

Quoted 1 line

...

fs/dlm/midcomms.c line 617

Quoted 11 lines
fs/dlm/midcomms.c +11 -0
+
+               dlm_receive_buffer(p, nodeid);
+               return;
+       case DLM_OPTS:
+               seq = le32_to_cpu(p->header.u.h_seq);
+
+               ret = dlm_opts_check_msglen(p, msglen, nodeid);
+               if (ret < 0)
+                       return;
+#if 0
+               ret = dlm_parse_opts(p->opts.o_opts, p->opts.o_optlen);

le16_to_cpu() is missing in optlen.

fs/dlm/midcomms.c line 623

Quoted 6 lines
fs/dlm/midcomms.c +6 -0
+               if (ret < 0)
+                       return;
+#endif
+
+               p = (union dlm_packet *)((unsigned char *)p->opts.o_opts + ret);
+

mhh, this + ret is a leftover of the commented code above. I commented it in because as so far we don't have any opts, we simply ignore it, we still can make changes to the tlv header e.g. I wasn't sure about one byte or 2 byte fields. 2 bytes is fine or we have a lot of pads which we may never use? We have then a lot of space in types and lengths, but we probably never have a length above 255 for full messages less than 4096, also 255 for types is okay as well, may the 4096 bytes limit can be changed in future...

- Alex

author maintainer
#18 lore raw

RFC PATCH 15/16 fs: dlm: add reliable connection if reconnect

In reply to Alexander Aring

Hi,

Quoted 8 lines · Alexander Aring

This patch introduce to make a tcp lowcomms connection reliable even if reconnects occurs. This is done by an application layer retransmission handling and sequence numbers in dlm protocols. There are three new dlm commands:

DLM_OPTS:

..

Quoted 4 lines
fs/dlm/midcomms.c +4 -0
+                       /* we only alloc a new node at receiving for the above
+                        * RCOM messages. It can be that the other side is
+                        * already gone and we cannot ack FIN messages anymore,
+                        * we ignore it until the other side runs into an

fs/dlm/midcomms.c line 458

Quoted 10 lines
fs/dlm/midcomms.c +10 -0
+                        * timeout. FIN messages are application stateless and
+                        * it's not imortant to be acked since it is the last
+                        * message before disconnect.
+                        *
+                        * we don't print a warning in this case.
+                        */
+                       switch (p->opts.o_nextcmd) {
+                       case DLM_ACK:
+                               /* ignore ACK as well */
+                               fallthrough;

This can't happen, DLM_ACK is never encapsulated by DLM_OPTS. I think I saw once DLM_ACK messages arriving but the node was already disconnected. I think we should print warnings in this case, the warning is here when we are already disconnected and we don't see one of the initial dlm messages as a new connection. This should always be the case and if not something weird is going on.

One of my last changes in this patchset was to change the hook for the remove member function, may that fixed the issue why I saw the DLM_ACK but no node was "active".

I will remove this case.

- Alex

author maintainer
#19 lore raw

RFC PATCH 15/16 fs: dlm: add reliable connection if reconnect

In reply to Alexander Aring

Hi,

Quoted 1 line · Alexander Aring

...

fs/dlm/midcomms.c line 623

Quoted 3 lines
fs/dlm/midcomms.c +3 -0
+
+               p = (union dlm_packet *)((unsigned char *)p->opts.o_opts + ret);
+

actually this needs to be:

p = (union dlm_packet *)((unsigned char *)p->opts.o_opts + le16_to_cpu(p->opts.o_optlen));

otherwise it would break if we add options because we would not "jump over" the header.

- Alex

author maintainer
#20 lore raw

RFC PATCH 16/16 fs: dlm: don't allow half transmitted messages

In reply to Alexander Aring

This patch will clean a dirty page buffer if a reconnect occurs. If a page buffer was half transmitted we cannot start inside the middle of a dlm message if a node connects again. I observed invalid length receiptions errors and was guessing that this behaviour occurs, after this patch I never saw an invalid message length again. This patch might drops more messages for dlm version 3.1 but 3.1 can't deal with half messages as well, for 3.2 it might trigger more retransmissions but will not leave dlm in a broken state.

---
 fs/dlm/lowcomms.c | 84 ++++++++++++++++++++++++++++++-----------------
 1 file changed, 54 insertions(+), 30 deletions(-)
fs/dlm/lowcomms.c +54 -30
index a90999e1f306..294999e0c4f3 100644
@@ -112,6 +112,7 @@ struct writequeue_entry {
 	int len;
 	int end;
 	int users;
+	bool dirty;
 	struct connection *con;
 	struct list_head msgs;
 	struct kref ref;
@@ -647,6 +648,36 @@ static void make_sockaddr(struct sockaddr_storage *saddr, uint16_t port,
 	memset((char *)saddr + *addr_len, 0, sizeof(struct sockaddr_storage) - *addr_len);
 }
 
+static void dlm_page_release(struct kref *kref)
+{
+	struct writequeue_entry *e = container_of(kref, struct writequeue_entry,
+						  ref);
+
+	__free_page(e->page);
+	kfree(e);
+}
+
+static void dlm_msg_release(struct kref *kref)
+{
+	struct dlm_msg *msg = container_of(kref, struct dlm_msg, ref);
+
+	kref_put(&msg->entry->ref, dlm_page_release);
+	kfree(msg);
+}
+
+static void free_entry(struct writequeue_entry *e)
+{
+	struct dlm_msg *msg, *tmp;
+
+	list_for_each_entry_safe(msg, tmp, &e->msgs, list) {
+		list_del(&msg->list);
+		kref_put(&msg->ref, dlm_msg_release);
+	}
+
+	list_del(&e->list);
+	kref_put(&e->ref, dlm_page_release);
+}
+
 static void dlm_close_sock(struct socket **sock)
 {
 	if (*sock) {
@@ -661,6 +692,7 @@ static void close_connection(struct connection *con, bool and_other,
 			     bool tx, bool rx)
 {
 	bool closing = test_and_set_bit(CF_CLOSING, &con->flags);
+	struct writequeue_entry *e;
 
 	if (tx && !closing && cancel_work_sync(&con->swork)) {
 		log_print("canceled swork for node %d", con->nodeid);
@@ -679,6 +711,26 @@ static void close_connection(struct connection *con, bool and_other,
 		close_connection(con->othercon, false, true, true);
 	}
 
+	/* if we send a writequeue entry only a half way, we drop the
+	 * whole entry because reconnection and that we not start of the
+	 * middle of a msg which will confuse the other end.
+	 *
+	 * we can always drop messages because retransmits, but what we
+	 * cannot allow is to transmit half messages which may be processed
+	 * at the other side.
+	 *
+	 * our policy is to start on a clean state when disconnects, we don't
+	 * know what's send/received on transport layer in this case.
+	 */
+	spin_lock_bh(&con->writequeue_lock);
+	if (!list_empty(&con->writequeue)) {
+		e = list_first_entry(&con->writequeue, struct writequeue_entry,
+				     list);
+		if (e->dirty)
+			free_entry(e);
+	}
+	spin_unlock_bh(&con->writequeue_lock);
+
 	con->rx_leftover = 0;
 	con->retries = 0;
 	clear_bit(CF_CONNECTED, &con->flags);
@@ -956,36 +1008,6 @@ static int accept_from_sock(struct listen_connection *con)
 	return result;
 }
 
-static void dlm_page_release(struct kref *kref)
-{
-	struct writequeue_entry *e = container_of(kref, struct writequeue_entry,
-						  ref);
-
-	__free_page(e->page);
-	kfree(e);
-}
-
-static void dlm_msg_release(struct kref *kref)
-{
-	struct dlm_msg *msg = container_of(kref, struct dlm_msg, ref);
-
-	kref_put(&msg->entry->ref, dlm_page_release);
-	kfree(msg);
-}
-
-static void free_entry(struct writequeue_entry *e)
-{
-	struct dlm_msg *msg, *tmp;
-
-	list_for_each_entry_safe(msg, tmp, &e->msgs, list) {
-		list_del(&msg->list);
-		kref_put(&msg->ref, dlm_msg_release);
-	}
-
-	list_del(&e->list);
-	kref_put(&e->ref, dlm_page_release);
-}
-
 /*
  * writequeue_entry_complete - try to delete and free write queue entry
  * @e: write queue entry to try to delete
@@ -997,6 +1019,8 @@ static void writequeue_entry_complete(struct writequeue_entry *e, int completed)
 {
 	e->offset += completed;
 	e->len -= completed;
+	/* signal that page was half way transmitted */
+	e->dirty = true;
 
 	if (e->len == 0 && e->users == 0)
 		free_entry(e);
Signature · 1 line
2.26.2