Skip to contentSkip to conversation

ocfs2: resend master request when lost connection with someone

PATCH New Xue jiufei 3 participants 4 messages Dec 22, 2012 – May 31, 2013 OCFS2

lore
Share & tools
lore

Xue jiufei

author
#0 lore raw

PATCH 1/1 ocfs2: resend master request when lost connection with someone

Function dlm_get_lock_resource() sends master request to all nodes in
domain_map and waits for their responses when the node(say nodeA) doesn't known who the master is.
When nodeA sends the master request, it happened that network of
nodeB down for a while, and then restore. The master request from nodeA does not reach nodeB. NodeA may wait again and again in dlm_wait_for_lock_mastery() and never returns.
This patch resend the mater request when a node lost connection with
some other nodes.

Signed-off-by xuejiufei <[email protected]>
---
 fs/ocfs2/dlm/dlmmaster.c |   41 +++++++++++++++++++++++++++++++++++------
 1 files changed, 35 insertions(+), 6 deletions(-)
fs/ocfs2/dlm/dlmmaster.c +35 -6
index c491f97..2a99a95 100644
@@ -106,7 +106,7 @@ static int dlm_do_master_request(struct dlm_lock_resource *res,
 static int dlm_wait_for_lock_mastery(struct dlm_ctxt *dlm,
 				     struct dlm_lock_resource *res,
 				     struct dlm_master_list_entry *mle,
-				     int *blocked);
+				     int *blocked, int *retry, int host_down);
 static int dlm_restart_lock_mastery(struct dlm_ctxt *dlm,
 				    struct dlm_lock_resource *res,
 				    struct dlm_master_list_entry *mle,

@@ -712,6 +712,8 @@ struct dlm_lock_resource * dlm_get_lock_resource(struct dlm_ctxt *dlm,
 	unsigned int hash;
 	int tries = 0;
 	int bit, wait_on_recovery = 0;
+	int retry = 0;
+	unsigned long down_nodemap[BITS_TO_LONGS(O2NM_MAX_NODES)];
 
 	BUG_ON(!lockid);
 

@@ -910,11 +912,25 @@ redo_request:
 		goto wait;
 
 	ret = -EINVAL;
-	dlm_node_iter_init(mle->vote_map, &iter);
+	if (!retry)
+		dlm_node_iter_init(mle->vote_map, &iter);
+	else {
+		mlog(0, "%s:%.*s: retrying, send master request to maybe down node\n",
+				dlm->name, res->lockname.len, res->lockname.name);
+		dlm_node_iter_init(down_nodemap, &iter);
+	}
+	memset(down_nodemap, 0, sizeof(down_nodemap));
+
 	while ((nodenum = dlm_node_iter_next(&iter)) >= 0) {
 		ret = dlm_do_master_request(res, mle, nodenum);
-		if (ret < 0)
+		if (ret < 0) {
 			mlog_errno(ret);
+			if (dlm_is_host_down(ret)) {
+				mlog(0, "%s:%.*s: node %u maybe dead, set down_nodemap\n",
+						dlm->name, res->lockname.len, res->lockname.name, nodenum);
+				set_bit(nodenum, down_nodemap);
+			}
+		}
 		if (mle->master != O2NM_MAX_NODES) {
 			/* found a master ! */
 			if (mle->master <= nodenum)

@@ -931,9 +947,11 @@ redo_request:
 
 wait:
 	/* keep going until the response map includes all nodes */
-	ret = dlm_wait_for_lock_mastery(dlm, res, mle, &blocked);
+	ret = dlm_wait_for_lock_mastery(dlm, res, mle, &blocked, &retry,
+			find_next_bit(down_nodemap, O2NM_MAX_NODES, 0) < O2NM_MAX_NODES);
 	if (ret < 0) {
-		wait_on_recovery = 1;
+		if (!retry)
+			wait_on_recovery = 1;
 		mlog(0, "%s: res %.*s, Node map changed, redo the master "
 		     "request now, blocked=%d\n", dlm->name, res->lockname.len,
 		     res->lockname.name, blocked);

@@ -980,7 +998,7 @@ leave:
 static int dlm_wait_for_lock_mastery(struct dlm_ctxt *dlm,
 				     struct dlm_lock_resource *res,
 				     struct dlm_master_list_entry *mle,
-				     int *blocked)
+				     int *blocked, int *retry, int host_down)
 {
 	u8 m;
 	int ret, bit;

@@ -990,6 +1008,7 @@ static int dlm_wait_for_lock_mastery(struct dlm_ctxt *dlm,
 recheck:
 	ret = 0;
 	assert = 0;
+	*retry = 0;
 
 	/* check if another node has already become the owner */
 	spin_lock(&res->spinlock);

@@ -1043,6 +1062,16 @@ recheck:
 		     res->lockname.name);
 		goto recheck;
 	} else {
+		if (host_down && (m == O2NM_MAX_NODES)) {
+			mlog(0, "map not changed but some one may lost connection, "
+					"rechecking\n");
+			*retry = 1;
+			spin_unlock(&mle->spinlock);
+			msleep(DLM_NODE_DEATH_WAIT_MAX);
+			ret = -EAGAIN;
+			goto leave;
+		}
+
 		if (!voting_done) {
 			mlog(0, "map not changed and voting not done "
 			     "for %s:%.*s\n", dlm->name, res->lockname.len,
Signature · 1 line
1.7.8.6
#1 lore raw

PATCH 1/1 ocfs2: resend master request when lost connection with someone

Hi,

I reviewed this patch , it did could fix a temp lost connection problem, but a few questions:

1. since we don't need to know the node numbers of down nodes, if simply replace the down_nodemap[BITS_TO_LONGS(O2NM_MAX_NODES)], with a int named for example mreq_msg_send_fail ?

2.since the final work is to return -EAGAIN, the resend all master requests. How about we simply do this?:

while ((nodenum = dlm_node_iter_next(&iter)) >= 0) {
ret = dlm_do_master_request(res, mle, nodenum);
- if (ret < 0)
+ if (ret < 0) {
mlog_errno(ret);
+ wait_on_recovery = 1;
+ msleep(DLM_NODE_DEATH_WAIT_MAX);
+ goto redo_request;
+ }

Am I missing something?

Thanks,
Xiaowei

Quoted 113 lines · Xue jiufei

Function dlm_get_lock_resource() sends master request to all nodes in
domain_map and waits for their responses when the node(say nodeA) doesn't known who the master is.
When nodeA sends the master request, it happened that network of
nodeB down for a while, and then restore. The master request from nodeA does not reach nodeB. NodeA may wait again and again in dlm_wait_for_lock_mastery() and never returns.
This patch resend the mater request when a node lost connection with
some other nodes.

Signed-off-by xuejiufei <[email protected]>
---
  fs/ocfs2/dlm/dlmmaster.c |   41 +++++++++++++++++++++++++++++++++++------
  1 files changed, 35 insertions(+), 6 deletions(-)
fs/ocfs2/dlm/dlmmaster.c +35 -6
index c491f97..2a99a95 100644
@@ -106,7 +106,7 @@ static int dlm_do_master_request(struct dlm_lock_resource *res,
  static int dlm_wait_for_lock_mastery(struct dlm_ctxt *dlm,
  				     struct dlm_lock_resource *res,
  				     struct dlm_master_list_entry *mle,
-				     int *blocked);
+				     int *blocked, int *retry, int host_down);
  static int dlm_restart_lock_mastery(struct dlm_ctxt *dlm,
  				    struct dlm_lock_resource *res,
  				    struct dlm_master_list_entry *mle,

@@ -712,6 +712,8 @@ struct dlm_lock_resource * dlm_get_lock_resource(struct dlm_ctxt *dlm,
  	unsigned int hash;
  	int tries = 0;
  	int bit, wait_on_recovery = 0;
+	int retry = 0;
+	unsigned long down_nodemap[BITS_TO_LONGS(O2NM_MAX_NODES)];
 
  	BUG_ON(!lockid);
 

@@ -910,11 +912,25 @@ redo_request:
  		goto wait;
 
  	ret = -EINVAL;
-	dlm_node_iter_init(mle->vote_map, &iter);
+	if (!retry)
+		dlm_node_iter_init(mle->vote_map, &iter);
+	else {
+		mlog(0, "%s:%.*s: retrying, send master request to maybe down node\n",
+				dlm->name, res->lockname.len, res->lockname.name);
+		dlm_node_iter_init(down_nodemap, &iter);
+	}
+	memset(down_nodemap, 0, sizeof(down_nodemap));
+
  	while ((nodenum = dlm_node_iter_next(&iter)) >= 0) {
  		ret = dlm_do_master_request(res, mle, nodenum);
-		if (ret < 0)
+		if (ret < 0) {
  			mlog_errno(ret);
+			if (dlm_is_host_down(ret)) {
+				mlog(0, "%s:%.*s: node %u maybe dead, set down_nodemap\n",
+						dlm->name, res->lockname.len, res->lockname.name, nodenum);
+				set_bit(nodenum, down_nodemap);
+			}
+		}
  		if (mle->master != O2NM_MAX_NODES) {
  			/* found a master ! */
  			if (mle->master <= nodenum)

@@ -931,9 +947,11 @@ redo_request:
 
  wait:
  	/* keep going until the response map includes all nodes */
-	ret = dlm_wait_for_lock_mastery(dlm, res, mle, &blocked);
+	ret = dlm_wait_for_lock_mastery(dlm, res, mle, &blocked, &retry,
+			find_next_bit(down_nodemap, O2NM_MAX_NODES, 0) < O2NM_MAX_NODES);
  	if (ret < 0) {
-		wait_on_recovery = 1;
+		if (!retry)
+			wait_on_recovery = 1;
  		mlog(0, "%s: res %.*s, Node map changed, redo the master "
  		     "request now, blocked=%d\n", dlm->name, res->lockname.len,
  		     res->lockname.name, blocked);

@@ -980,7 +998,7 @@ leave:
  static int dlm_wait_for_lock_mastery(struct dlm_ctxt *dlm,
  				     struct dlm_lock_resource *res,
  				     struct dlm_master_list_entry *mle,
-				     int *blocked)
+				     int *blocked, int *retry, int host_down)
  {
  	u8 m;
  	int ret, bit;

@@ -990,6 +1008,7 @@ static int dlm_wait_for_lock_mastery(struct dlm_ctxt *dlm,
  recheck:
  	ret = 0;
  	assert = 0;
+	*retry = 0;
 
  	/* check if another node has already become the owner */
  	spin_lock(&res->spinlock);

@@ -1043,6 +1062,16 @@ recheck:
  		     res->lockname.name);
  		goto recheck;
  	} else {
+		if (host_down && (m == O2NM_MAX_NODES)) {
+			mlog(0, "map not changed but some one may lost connection, "
+					"rechecking\n");
+			*retry = 1;
+			spin_unlock(&mle->spinlock);
+			msleep(DLM_NODE_DEATH_WAIT_MAX);
+			ret = -EAGAIN;
+			goto leave;
+		}
+
  		if (!voting_done) {
  			mlog(0, "map not changed and voting not done "
  			     "for %s:%.*s\n", dlm->name, res->lockname.len,
author
#2 lore raw

PATCH 1/1 ocfs2: resend master request when lost connection with someone

Hi, Xiaowei
It's OK to simlify the patch just as you did. But we don't want to resend master request to all others nodes in consideration of network traffic. So we record those maybe down nodes in down_nodemap.

? 2013/5/28 14:12, xiaowei.hu ??:

Quoted 135 lines

Hi,

I reviewed this patch , it did could fix a temp lost connection problem, but a few questions:

1. since we don't need to know the node numbers of down nodes, if simply replace the down_nodemap[BITS_TO_LONGS(O2NM_MAX_NODES)], with a int named for example mreq_msg_send_fail ?

2.since the final work is to return -EAGAIN, the resend all master requests. How about we simply do this?:

     while ((nodenum = dlm_node_iter_next(&iter)) >= 0) {
         ret = dlm_do_master_request(res, mle, nodenum);
fs/ocfs2/dlm/dlmmaster.c +5 -1
-        if (ret < 0)
+        if (ret < 0) {
             mlog_errno(ret);
+            wait_on_recovery = 1;
+            msleep(DLM_NODE_DEATH_WAIT_MAX);
+            goto redo_request;
+        }

Am I missing something?

Thanks,
Xiaowei

Quoted 98 lines · Xue jiufei

Function dlm_get_lock_resource() sends master request to all nodes in
domain_map and waits for their responses when the node(say nodeA) doesn't known who the master is.
When nodeA sends the master request, it happened that network of
nodeB down for a while, and then restore. The master request from nodeA does not reach nodeB. NodeA may wait again and again in dlm_wait_for_lock_mastery() and never returns.
This patch resend the mater request when a node lost connection with
some other nodes.

Signed-off-by xuejiufei <[email protected]>
---
  fs/ocfs2/dlm/dlmmaster.c |   41 +++++++++++++++++++++++++++++++++++------
  1 files changed, 35 insertions(+), 6 deletions(-)
fs/ocfs2/dlm/dlmmaster.c +3 -1
index c491f97..2a99a95 100644
@@ -106,7 +106,7 @@ static int dlm_do_master_request(struct dlm_lock_resource *res,
  static int dlm_wait_for_lock_mastery(struct dlm_ctxt *dlm,
                       struct dlm_lock_resource *res,
                       struct dlm_master_list_entry *mle,
-                     int *blocked);
+                     int *blocked, int *retry, int host_down);
  static int dlm_restart_lock_mastery(struct dlm_ctxt *dlm,
                      struct dlm_lock_resource *res,
                      struct dlm_master_list_entry *mle,

@@ -712,6 +712,8 @@ struct dlm_lock_resource * dlm_get_lock_resource(struct dlm_ctxt *dlm,
      unsigned int hash;
      int tries = 0;
      int bit, wait_on_recovery = 0;
+    int retry = 0;
+    unsigned long down_nodemap[BITS_TO_LONGS(O2NM_MAX_NODES)];
        BUG_ON(!lockid);
  @@ -910,11 +912,25 @@ redo_request:
          goto wait;

        ret = -EINVAL;
fs/ocfs2/dlm/dlmmaster.c +25 -5
-    dlm_node_iter_init(mle->vote_map, &iter);
+    if (!retry)
+        dlm_node_iter_init(mle->vote_map, &iter);
+    else {
+        mlog(0, "%s:%.*s: retrying, send master request to maybe down node\n",
+                dlm->name, res->lockname.len, res->lockname.name);
+        dlm_node_iter_init(down_nodemap, &iter);
+    }
+    memset(down_nodemap, 0, sizeof(down_nodemap));
+
      while ((nodenum = dlm_node_iter_next(&iter)) >= 0) {
          ret = dlm_do_master_request(res, mle, nodenum);
-        if (ret < 0)
+        if (ret < 0) {
              mlog_errno(ret);
+            if (dlm_is_host_down(ret)) {
+                mlog(0, "%s:%.*s: node %u maybe dead, set down_nodemap\n",
+                        dlm->name, res->lockname.len, res->lockname.name, nodenum);
+                set_bit(nodenum, down_nodemap);
+            }
+        }
          if (mle->master != O2NM_MAX_NODES) {
              /* found a master ! */
              if (mle->master <= nodenum)

@@ -931,9 +947,11 @@ redo_request:
    wait:
      /* keep going until the response map includes all nodes */
-    ret = dlm_wait_for_lock_mastery(dlm, res, mle, &blocked);
+    ret = dlm_wait_for_lock_mastery(dlm, res, mle, &blocked, &retry,
+            find_next_bit(down_nodemap, O2NM_MAX_NODES, 0) < O2NM_MAX_NODES);
      if (ret < 0) {
-        wait_on_recovery = 1;
+        if (!retry)
+            wait_on_recovery = 1;
          mlog(0, "%s: res %.*s, Node map changed, redo the master "
               "request now, blocked=%d\n", dlm->name, res->lockname.len,
               res->lockname.name, blocked);

@@ -980,7 +998,7 @@ leave:
  static int dlm_wait_for_lock_mastery(struct dlm_ctxt *dlm,
                       struct dlm_lock_resource *res,
                       struct dlm_master_list_entry *mle,
-                     int *blocked)
+                     int *blocked, int *retry, int host_down)
  {
      u8 m;
      int ret, bit;

@@ -990,6 +1008,7 @@ static int dlm_wait_for_lock_mastery(struct dlm_ctxt *dlm,
  recheck:
      ret = 0;
      assert = 0;
+    *retry = 0;
        /* check if another node has already become the owner */
      spin_lock(&res->spinlock);

@@ -1043,6 +1062,16 @@ recheck:
               res->lockname.name);
          goto recheck;
      } else {
+        if (host_down && (m == O2NM_MAX_NODES)) {
+            mlog(0, "map not changed but some one may lost connection, "
+                    "rechecking\n");

fs/ocfs2/dlm/dlmmaster.c recheck: · line 1077

Quoted 10 lines
fs/ocfs2/dlm/dlmmaster.c +7 -0
+            *retry = 1;
+            spin_unlock(&mle->spinlock);
+            msleep(DLM_NODE_DEATH_WAIT_MAX);
+            ret = -EAGAIN;
+            goto leave;
+        }
+
          if (!voting_done) {
              mlog(0, "map not changed and voting not done "
                   "for %s:%.*s\n", dlm->name, res->lockname.len,

.

#3 lore raw

PATCH 1/1 ocfs2: resend master request when lost connection with someone

Quoted 5 lines · Xue jiufei

Hi, Xiaowei
It's OK to simlify the patch just as you did. But we don't want to resend master request to all others nodes in consideration of network traffic. So we record those maybe down nodes in down_nodemap.

Quoted 34 lines

? 2013/5/28 14:12, xiaowei.hu ??:

Quoted 33 lines

Hi,

I reviewed this patch , it did could fix a temp lost connection problem, but a few questions:

1. since we don't need to know the node numbers of down nodes, if simply replace the down_nodemap[BITS_TO_LONGS(O2NM_MAX_NODES)], with a int named for example mreq_msg_send_fail ?

2.since the final work is to return -EAGAIN, the resend all master requests. How about we simply do this?:

      while ((nodenum = dlm_node_iter_next(&iter)) >= 0) {
          ret = dlm_do_master_request(res, mle, nodenum);
fs/ocfs2/dlm/dlmmaster.c +5 -1
-        if (ret < 0)
+        if (ret < 0) {
              mlog_errno(ret);
+            wait_on_recovery = 1;
+            msleep(DLM_NODE_DEATH_WAIT_MAX);
+            goto redo_request;
+        }

Am I missing something?

Thanks,
Xiaowei

Quoted 9 lines · Xue jiufei

Function dlm_get_lock_resource() sends master request to all nodes in
domain_map and waits for their responses when the node(say nodeA) doesn't known who the master is.
When nodeA sends the master request, it happened that network of
nodeB down for a while, and then restore. The master request from nodeA does not reach nodeB. NodeA may wait again and again in dlm_wait_for_lock_mastery() and never returns.
This patch resend the mater request when a node lost connection with
some other nodes.

Yes, with the current reconnect code there is a possibility of message loss and can happen to all kinds of messages and responses. You are assuming the message never received by node B. Message might have been received by node B but the response might have lost too. Currently there is no way of telling the difference. DLM layer shouldn't worry about nodes loosing connection temporarily. Right way to fix this is to fix o2net reconnect code.

Quoted 107 lines
Quoted 92 lines
Quoted 92 lines
Signed-off-by xuejiufei <[email protected]>
---
   fs/ocfs2/dlm/dlmmaster.c |   41 +++++++++++++++++++++++++++++++++++------
   1 files changed, 35 insertions(+), 6 deletions(-)
fs/ocfs2/dlm/dlmmaster.c +3 -1
index c491f97..2a99a95 100644
@@ -106,7 +106,7 @@ static int dlm_do_master_request(struct dlm_lock_resource *res,
   static int dlm_wait_for_lock_mastery(struct dlm_ctxt *dlm,
                        struct dlm_lock_resource *res,
                        struct dlm_master_list_entry *mle,
-                     int *blocked);
+                     int *blocked, int *retry, int host_down);
   static int dlm_restart_lock_mastery(struct dlm_ctxt *dlm,
                       struct dlm_lock_resource *res,
                       struct dlm_master_list_entry *mle,

@@ -712,6 +712,8 @@ struct dlm_lock_resource * dlm_get_lock_resource(struct dlm_ctxt *dlm,
       unsigned int hash;
       int tries = 0;
       int bit, wait_on_recovery = 0;
+    int retry = 0;
+    unsigned long down_nodemap[BITS_TO_LONGS(O2NM_MAX_NODES)];
         BUG_ON(!lockid);
   @@ -910,11 +912,25 @@ redo_request:
           goto wait;

         ret = -EINVAL;
fs/ocfs2/dlm/dlmmaster.c +28 -5
-    dlm_node_iter_init(mle->vote_map, &iter);
+    if (!retry)
+        dlm_node_iter_init(mle->vote_map, &iter);
+    else {
+        mlog(0, "%s:%.*s: retrying, send master request to maybe down node\n",
+                dlm->name, res->lockname.len, res->lockname.name);
+        dlm_node_iter_init(down_nodemap, &iter);
+    }
+    memset(down_nodemap, 0, sizeof(down_nodemap));
+
       while ((nodenum = dlm_node_iter_next(&iter)) >= 0) {
           ret = dlm_do_master_request(res, mle, nodenum);
-        if (ret < 0)
+        if (ret < 0) {
               mlog_errno(ret);
+            if (dlm_is_host_down(ret)) {
+                mlog(0, "%s:%.*s: node %u maybe dead, set down_nodemap\n",
+                        dlm->name, res->lockname.len, res->lockname.name, nodenum);
+                set_bit(nodenum, down_nodemap);
+            }
+        }
           if (mle->master != O2NM_MAX_NODES) {
               /* found a master ! */
               if (mle->master <= nodenum)

@@ -931,9 +947,11 @@ redo_request:
     wait:
       /* keep going until the response map includes all nodes */
-    ret = dlm_wait_for_lock_mastery(dlm, res, mle, &blocked);
+    ret = dlm_wait_for_lock_mastery(dlm, res, mle, &blocked, &retry,
+            find_next_bit(down_nodemap, O2NM_MAX_NODES, 0) < O2NM_MAX_NODES);
       if (ret < 0) {
-        wait_on_recovery = 1;
+        if (!retry)
+            wait_on_recovery = 1;
           mlog(0, "%s: res %.*s, Node map changed, redo the master "
                "request now, blocked=%d\n", dlm->name, res->lockname.len,
                res->lockname.name, blocked);

@@ -980,7 +998,7 @@ leave:
   static int dlm_wait_for_lock_mastery(struct dlm_ctxt *dlm,
                        struct dlm_lock_resource *res,
                        struct dlm_master_list_entry *mle,
-                     int *blocked)
+                     int *blocked, int *retry, int host_down)
   {
       u8 m;
       int ret, bit;

@@ -990,6 +1008,7 @@ static int dlm_wait_for_lock_mastery(struct dlm_ctxt *dlm,
   recheck:
       ret = 0;
       assert = 0;
+    *retry = 0;
         /* check if another node has already become the owner */
       spin_lock(&res->spinlock);

@@ -1043,6 +1062,16 @@ recheck:
                res->lockname.name);
           goto recheck;
       } else {
+        if (host_down && (m == O2NM_MAX_NODES)) {
+            mlog(0, "map not changed but some one may lost connection, "
+                    "rechecking\n");
+            *retry = 1;
+            spin_unlock(&mle->spinlock);
+            msleep(DLM_NODE_DEATH_WAIT_MAX);
Quoted 9 lines

fs/ocfs2/dlm/dlmmaster.c recheck: · line 1077

Quoted 7 lines
fs/ocfs2/dlm/dlmmaster.c +4 -0
+            ret = -EAGAIN;
+            goto leave;
+        }
+
           if (!voting_done) {
               mlog(0, "map not changed and voting not done "
                    "for %s:%.*s\n", dlm->name, res->lockname.len,

.

_______________________________________________
Ocfs2-devel mailing list
Ocfs2-devel at oss.oracle.com
https://oss.oracle.com/mailman/listinfo/ocfs2-devel