Skip to contentSkip to conversation

ocfs2/dlm: fix race between convert and recovery

PATCH Superseded Joseph Qi 2 participants 10 messages Sep 18, 2015 – Sep 23, 2015 OCFS2

v2 of 3 Superseded Latest is v3 →

lore
Share & tools
lore

Joseph Qi

author
#0 lore raw

PATCH v2 1/1 ocfs2/dlm: fix race between convert and recovery

There is a race window between dlmconvert_remote and dlm_move_lockres_to_recovery_list, which will cause a lock with OCFS2_LOCK_BUSY in grant list, thus system hangs.

dlmconvert_remote

{
        spin_lock(&res->spinlock);
        list_move_tail(&lock->list, &res->converting);
        lock->convert_pending = 1;
        spin_unlock(&res->spinlock);

        status = dlm_send_remote_convert_request();
        >>>>>> race window, master has queued ast and return DLM_NORMAL,
               and then down before sending ast.
               this node detects master down and calls
               dlm_move_lockres_to_recovery_list, which will revert the
               lock to grant list.
               Then OCFS2_LOCK_BUSY won't be cleared as new master won't
               send ast any more because it thinks already be authorized.

        spin_lock(&res->spinlock);
        lock->convert_pending = 0;
        if (status != DLM_NORMAL)
                dlm_revert_pending_convert(res, lock);
        spin_unlock(&res->spinlock);
}

In this case, just leave it in convert list and new master will take care of it after recovery. And if convert request returns other than DLM_NORMAL, convert thread will do the revert itself. So remove the revert logic in dlm_move_lockres_to_recovery_list.

changelog since v1:
Clean up convert_pending since it is now useless.

---
 fs/ocfs2/dlm/dlmcommon.h   |  1 -
 fs/ocfs2/dlm/dlmconvert.c  |  2 --
 fs/ocfs2/dlm/dlmdebug.c    |  7 +++----
 fs/ocfs2/dlm/dlmlock.c     |  1 -
 fs/ocfs2/dlm/dlmrecovery.c | 10 +---------
 5 files changed, 4 insertions(+), 17 deletions(-)
fs/ocfs2/dlm/dlmcommon.h +0 -1
index e88ccf8..25853d7 100644
@@ -369,7 +369,6 @@ struct dlm_lock
 	struct dlm_lockstatus *lksb;
 	unsigned ast_pending:1,
 		 bast_pending:1,
-		 convert_pending:1,
 		 lock_pending:1,
 		 cancel_pending:1,
 		 unlock_pending:1,
fs/ocfs2/dlm/dlmconvert.c +0 -2
index e36d63f..82ba8d5 100644
@@ -291,7 +291,6 @@ enum dlm_status dlmconvert_remote(struct dlm_ctxt *dlm,
 	/* move lock to local convert queue */
 	/* do not alter lock refcount.  switching lists. */
 	list_move_tail(&lock->list, &res->converting);
-	lock->convert_pending = 1;
 	lock->ml.convert_type = type;
 
 	if (flags & LKM_VALBLK) {
@@ -315,7 +314,6 @@ enum dlm_status dlmconvert_remote(struct dlm_ctxt *dlm,
 
 	spin_lock(&res->spinlock);
 	res->state &= ~DLM_LOCK_RES_IN_PROGRESS;
-	lock->convert_pending = 0;
 	/* if it failed, move it back to granted queue */
 	if (status != DLM_NORMAL) {
 		if (status != DLM_NOTQUEUED)
fs/ocfs2/dlm/dlmdebug.c +3 -4
index 8251360..710e94c 100644
@@ -77,7 +77,7 @@ static void __dlm_print_lock(struct dlm_lock *lock)
 
 	printk("    type=%d, conv=%d, node=%u, cookie=%u:%llu, "
 	       "ref=%u, ast=(empty=%c,pend=%c), bast=(empty=%c,pend=%c), "
-	       "pending=(conv=%c,lock=%c,cancel=%c,unlock=%c)\n",
+	       "pending=(lock=%c,cancel=%c,unlock=%c)\n",
 	       lock->ml.type, lock->ml.convert_type, lock->ml.node,
 	       dlm_get_lock_cookie_node(be64_to_cpu(lock->ml.cookie)),
 	       dlm_get_lock_cookie_seq(be64_to_cpu(lock->ml.cookie)),
@@ -86,7 +86,6 @@ static void __dlm_print_lock(struct dlm_lock *lock)
 	       (lock->ast_pending ? 'y' : 'n'),
 	       (list_empty(&lock->bast_list) ? 'y' : 'n'),
 	       (lock->bast_pending ? 'y' : 'n'),
-	       (lock->convert_pending ? 'y' : 'n'),
 	       (lock->lock_pending ? 'y' : 'n'),
 	       (lock->cancel_pending ? 'y' : 'n'),
 	       (lock->unlock_pending ? 'y' : 'n'));
@@ -502,7 +501,7 @@ static int dump_lock(struct dlm_lock *lock, int list_type, char *buf, int len)
 
 #define DEBUG_LOCK_VERSION	1
 	spin_lock(&lock->spinlock);
-	out = snprintf(buf, len, "LOCK:%d,%d,%d,%d,%d,%d:%lld,%d,%d,%d,%d,%d,"
+	out = snprintf(buf, len, "LOCK:%d,%d,%d,%d,%d,%d:%lld,%d,%d,%d,%d,"
 		       "%d,%d,%d,%d\n",
 		       DEBUG_LOCK_VERSION,
 		       list_type, lock->ml.type, lock->ml.convert_type,
@@ -512,7 +511,7 @@ static int dump_lock(struct dlm_lock *lock, int list_type, char *buf, int len)
 		       !list_empty(&lock->ast_list),
 		       !list_empty(&lock->bast_list),
 		       lock->ast_pending, lock->bast_pending,
-		       lock->convert_pending, lock->lock_pending,
+		       lock->lock_pending,
 		       lock->cancel_pending, lock->unlock_pending,
 		       atomic_read(&lock->lock_refs.refcount));
 	spin_unlock(&lock->spinlock);
fs/ocfs2/dlm/dlmlock.c +0 -1
index 66c2a49..857dd15 100644
@@ -411,7 +411,6 @@ static void dlm_init_lock(struct dlm_lock *newlock, int type,
 	newlock->ml.cookie = cpu_to_be64(cookie);
 	newlock->ast_pending = 0;
 	newlock->bast_pending = 0;
-	newlock->convert_pending = 0;
 	newlock->lock_pending = 0;
 	newlock->unlock_pending = 0;
 	newlock->cancel_pending = 0;
fs/ocfs2/dlm/dlmrecovery.c +1 -9
index 3d90ad7..2402ef7 100644
@@ -2062,15 +2062,7 @@ void dlm_move_lockres_to_recovery_list(struct dlm_ctxt *dlm,
 		queue = dlm_list_idx_to_ptr(res, i);
 		list_for_each_entry_safe(lock, next, queue, list) {
 			dlm_lock_get(lock);
-			if (lock->convert_pending) {
-				/* move converting lock back to granted */
-				BUG_ON(i != DLM_CONVERTING_LIST);
-				mlog(0, "node died with convert pending "
-				     "on %.*s. move back to granted list.\n",
-				     res->lockname.len, res->lockname.name);
-				dlm_revert_pending_convert(res, lock);
-				lock->convert_pending = 0;
-			} else if (lock->lock_pending) {
+			if (lock->lock_pending) {
 				/* remove pending lock requests completely */
 				BUG_ON(i != DLM_BLOCKED_LIST);
 				mlog(0, "node died with lock pending "
Signature · 1 line
1.8.4.3
#1 lore raw

PATCH v2 1/1 ocfs2/dlm: fix race between convert and recovery

Quoted 156 lines

? 2015?9?18????7:25?Joseph Qi <[email protected]> ???

There is a race window between dlmconvert_remote and dlm_move_lockres_to_recovery_list, which will cause a lock with OCFS2_LOCK_BUSY in grant list, thus system hangs.

dlmconvert_remote

{
       spin_lock(&res->spinlock);
       list_move_tail(&lock->list, &res->converting);
       lock->convert_pending = 1;
       spin_unlock(&res->spinlock);

       status = dlm_send_remote_convert_request();
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line

race window, master has queued ast and return DLM_NORMAL,

and then down before sending ast.
this node detects master down and calls dlm_move_lockres_to_recovery_list, which will revert the lock to grant list.
Then OCFS2_LOCK_BUSY won't be cleared as new master won't send ast any more because it thinks already be authorized.

spin_lock(&res->spinlock); lock->convert_pending = 0;
if (status != DLM_NORMAL)
dlm_revert_pending_convert(res, lock);
spin_unlock(&res->spinlock);
}

In this case, just leave it in convert list and new master will take care of it after recovery. And if convert request returns other than DLM_NORMAL, convert thread will do the revert itself. So remove the revert logic in dlm_move_lockres_to_recovery_list.

changelog since v1:
Clean up convert_pending since it is now useless.

---

fs/ocfs2/dlm/dlmcommon.h   |  1 -
fs/ocfs2/dlm/dlmconvert.c  |  2 --
fs/ocfs2/dlm/dlmdebug.c    |  7 +++----
fs/ocfs2/dlm/dlmlock.c     |  1 -
fs/ocfs2/dlm/dlmrecovery.c | 10 +---------
5 files changed, 4 insertions(+), 17 deletions(-)
fs/ocfs2/dlm/dlmcommon.h +0 -1
index e88ccf8..25853d7 100644
@@ -369,7 +369,6 @@ struct dlm_lock
 	struct dlm_lockstatus *lksb;
 	unsigned ast_pending:1,
 		 bast_pending:1,
-		 convert_pending:1,
 		 lock_pending:1,
 		 cancel_pending:1,
 		 unlock_pending:1,
fs/ocfs2/dlm/dlmconvert.c +0 -2
index e36d63f..82ba8d5 100644
@@ -291,7 +291,6 @@ enum dlm_status dlmconvert_remote(struct dlm_ctxt *dlm,
 	/* move lock to local convert queue */
 	/* do not alter lock refcount.  switching lists. */
 	list_move_tail(&lock->list, &res->converting);
-	lock->convert_pending = 1;
 	lock->ml.convert_type = type;
 
 	if (flags & LKM_VALBLK) {
@@ -315,7 +314,6 @@ enum dlm_status dlmconvert_remote(struct dlm_ctxt *dlm,
 
 	spin_lock(&res->spinlock);
 	res->state &= ~DLM_LOCK_RES_IN_PROGRESS;
-	lock->convert_pending = 0;
 	/* if it failed, move it back to granted queue */
 	if (status != DLM_NORMAL) {
 		if (status != DLM_NOTQUEUED)
fs/ocfs2/dlm/dlmdebug.c +3 -4
index 8251360..710e94c 100644
@@ -77,7 +77,7 @@ static void __dlm_print_lock(struct dlm_lock *lock)
 
 	printk("    type=%d, conv=%d, node=%u, cookie=%u:%llu, "
 	       "ref=%u, ast=(empty=%c,pend=%c), bast=(empty=%c,pend=%c), "
-	       "pending=(conv=%c,lock=%c,cancel=%c,unlock=%c)\n",
+	       "pending=(lock=%c,cancel=%c,unlock=%c)\n",
 	       lock->ml.type, lock->ml.convert_type, lock->ml.node,
 	       dlm_get_lock_cookie_node(be64_to_cpu(lock->ml.cookie)),
 	       dlm_get_lock_cookie_seq(be64_to_cpu(lock->ml.cookie)),
@@ -86,7 +86,6 @@ static void __dlm_print_lock(struct dlm_lock *lock)
 	       (lock->ast_pending ? 'y' : 'n'),
 	       (list_empty(&lock->bast_list) ? 'y' : 'n'),
 	       (lock->bast_pending ? 'y' : 'n'),
-	       (lock->convert_pending ? 'y' : 'n'),
 	       (lock->lock_pending ? 'y' : 'n'),
 	       (lock->cancel_pending ? 'y' : 'n'),
 	       (lock->unlock_pending ? 'y' : 'n'));
@@ -502,7 +501,7 @@ static int dump_lock(struct dlm_lock *lock, int list_type, char *buf, int len)
 
 #define DEBUG_LOCK_VERSION	1
 	spin_lock(&lock->spinlock);
-	out = snprintf(buf, len, "LOCK:%d,%d,%d,%d,%d,%d:%lld,%d,%d,%d,%d,%d,"
+	out = snprintf(buf, len, "LOCK:%d,%d,%d,%d,%d,%d:%lld,%d,%d,%d,%d,"
 		       "%d,%d,%d,%d\n",
 		       DEBUG_LOCK_VERSION,
 		       list_type, lock->ml.type, lock->ml.convert_type,
@@ -512,7 +511,7 @@ static int dump_lock(struct dlm_lock *lock, int list_type, char *buf, int len)
 		       !list_empty(&lock->ast_list),
 		       !list_empty(&lock->bast_list),
 		       lock->ast_pending, lock->bast_pending,
-		       lock->convert_pending, lock->lock_pending,
+		       lock->lock_pending,
 		       lock->cancel_pending, lock->unlock_pending,
 		       atomic_read(&lock->lock_refs.refcount));
 	spin_unlock(&lock->spinlock);
fs/ocfs2/dlm/dlmlock.c +0 -1
index 66c2a49..857dd15 100644
@@ -411,7 +411,6 @@ static void dlm_init_lock(struct dlm_lock *newlock, int type,
 	newlock->ml.cookie = cpu_to_be64(cookie);
 	newlock->ast_pending = 0;
 	newlock->bast_pending = 0;
-	newlock->convert_pending = 0;
 	newlock->lock_pending = 0;
 	newlock->unlock_pending = 0;
 	newlock->cancel_pending = 0;
fs/ocfs2/dlm/dlmrecovery.c +1 -9
index 3d90ad7..2402ef7 100644
@@ -2062,15 +2062,7 @@ void dlm_move_lockres_to_recovery_list(struct dlm_ctxt *dlm,
 		queue = dlm_list_idx_to_ptr(res, i);
 		list_for_each_entry_safe(lock, next, queue, list) {
 			dlm_lock_get(lock);
-			if (lock->convert_pending) {
-				/* move converting lock back to granted */
-				BUG_ON(i != DLM_CONVERTING_LIST);
-				mlog(0, "node died with convert pending "
-				     "on %.*s. move back to granted list.\n",
-				     res->lockname.len, res->lockname.name);
-				dlm_revert_pending_convert(res, lock);
-				lock->convert_pending = 0;
-			} else if (lock->lock_pending) {
+			if (lock->lock_pending) {
 				/* remove pending lock requests completely */
 				BUG_ON(i != DLM_BLOCKED_LIST);
 				mlog(0, "node died with lock pending "
Signature · 1 line
1.8.4.3
author
#2 lore raw

PATCH v2 1/1 ocfs2/dlm: fix race between convert and recovery

Hi Junxiao,
This solution may have problem in the following scenario:
Consider dlm_send_remote_convert_request has taken too much time, and dlm_move_lockres_to_recovery_list runs first and new master will see this node is currently in convert list after recovery.
Then dlm_send_remote_convert_request returns other than DLM_NORMAL and it will revert it to grant list, then retry convert. This will makes this node and master inconsistent.
I will try to find another solution to resolve the race issue.

Quoted 38 lines · Junxiao Bi
Quoted 36 lines
Quoted 36 lines

? 2015?9?18????7:25?Joseph Qi <[email protected]> ???

There is a race window between dlmconvert_remote and dlm_move_lockres_to_recovery_list, which will cause a lock with OCFS2_LOCK_BUSY in grant list, thus system hangs.

dlmconvert_remote

{
       spin_lock(&res->spinlock);
       list_move_tail(&lock->list, &res->converting);
       lock->convert_pending = 1;
       spin_unlock(&res->spinlock);

       status = dlm_send_remote_convert_request();
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line

race window, master has queued ast and return DLM_NORMAL,

and then down before sending ast.
this node detects master down and calls dlm_move_lockres_to_recovery_list, which will revert the lock to grant list.
Then OCFS2_LOCK_BUSY won't be cleared as new master won't send ast any more because it thinks already be authorized.

spin_lock(&res->spinlock); lock->convert_pending = 0;
if (status != DLM_NORMAL)
dlm_revert_pending_convert(res, lock);
spin_unlock(&res->spinlock);
}

In this case, just leave it in convert list and new master will take care of it after recovery. And if convert request returns other than DLM_NORMAL, convert thread will do the revert itself. So remove the revert logic in dlm_move_lockres_to_recovery_list.

changelog since v1:
Clean up convert_pending since it is now useless.

#3 lore raw

PATCH v2 1/1 ocfs2/dlm: fix race between convert and recovery

Quoted 9 lines · Joseph Qi

Hi Junxiao,
This solution may have problem in the following scenario:
Consider dlm_send_remote_convert_request has taken too much time, and dlm_move_lockres_to_recovery_list runs first and new master will see this node is currently in convert list after recovery.
Then dlm_send_remote_convert_request returns other than DLM_NORMAL and it will revert it to grant list, then retry convert. This will makes this node and master inconsistent.
I will try to find another solution to resolve the race issue.

If master is down, no need retry convert. May check the return value of dlm_send_remote_convert_request(), if DLM_RECOVERING, don't retry, otherwise retry?

Thanks,
Junxiao.

Quoted 42 lines · Junxiao Bi
Quoted 38 lines
Quoted 36 lines
Quoted 36 lines

? 2015?9?18????7:25?Joseph Qi <[email protected]> ???

There is a race window between dlmconvert_remote and dlm_move_lockres_to_recovery_list, which will cause a lock with OCFS2_LOCK_BUSY in grant list, thus system hangs.

dlmconvert_remote

{
       spin_lock(&res->spinlock);
       list_move_tail(&lock->list, &res->converting);
       lock->convert_pending = 1;
       spin_unlock(&res->spinlock);

       status = dlm_send_remote_convert_request();
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line

race window, master has queued ast and return DLM_NORMAL,

and then down before sending ast.
this node detects master down and calls dlm_move_lockres_to_recovery_list, which will revert the lock to grant list.
Then OCFS2_LOCK_BUSY won't be cleared as new master won't send ast any more because it thinks already be authorized.

spin_lock(&res->spinlock); lock->convert_pending = 0;
if (status != DLM_NORMAL)
dlm_revert_pending_convert(res, lock);
spin_unlock(&res->spinlock);
}

In this case, just leave it in convert list and new master will take care of it after recovery. And if convert request returns other than DLM_NORMAL, convert thread will do the revert itself. So remove the revert logic in dlm_move_lockres_to_recovery_list.

changelog since v1:
Clean up convert_pending since it is now useless.

author
#4 lore raw

PATCH v2 1/1 ocfs2/dlm: fix race between convert and recovery

Hi Junxiao,

Quoted 11 lines · Junxiao Bi
Quoted 9 lines · Joseph Qi

Hi Junxiao,
This solution may have problem in the following scenario:
Consider dlm_send_remote_convert_request has taken too much time, and dlm_move_lockres_to_recovery_list runs first and new master will see this node is currently in convert list after recovery.
Then dlm_send_remote_convert_request returns other than DLM_NORMAL and it will revert it to grant list, then retry convert. This will makes this node and master inconsistent.
I will try to find another solution to resolve the race issue.

Quoted 3 lines

If master is down, no need retry convert. May check the return value of dlm_send_remote_convert_request(), if DLM_RECOVERING, don't retry, otherwise retry?

I want to keep the original logic. And for fixing the race case I described, how about the following idea?

Check the status DLM_NORMAL. If res->state is currently DLM_LOCK_RES_RECOVERING (set in dlm_move_lockres_to_recovery_list, means still in recovery) or res master changed (means new master has finished the recovery), reset the status to DLM_RECOVERING, just like the check at the beginning of dlmconvert_remote. Then it is now in grant list and outer will retry.

Quoted 50 lines

Thanks,
Junxiao.

Quoted 33 lines · Junxiao Bi
Quoted 31 lines
Quoted 29 lines
Quoted 29 lines

? 2015?9?18????7:25?Joseph Qi <[email protected]> ???

There is a race window between dlmconvert_remote and dlm_move_lockres_to_recovery_list, which will cause a lock with OCFS2_LOCK_BUSY in grant list, thus system hangs.

dlmconvert_remote

{	
       spin_lock(&res->spinlock);
       list_move_tail(&lock->list, &res->converting);
       lock->convert_pending = 1;
       spin_unlock(&res->spinlock);

       status = dlm_send_remote_convert_request();
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line

race window, master has queued ast and return DLM_NORMAL,

and then down before sending ast.
this node detects master down and calls dlm_move_lockres_to_recovery_list, which will revert the lock to grant list.
Then OCFS2_LOCK_BUSY won't be cleared as new master won't send ast any more because it thinks already be authorized.

spin_lock(&res->spinlock); lock->convert_pending = 0;
if (status != DLM_NORMAL)
dlm_revert_pending_convert(res, lock);
spin_unlock(&res->spinlock);
}

Quoted 9 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines

In this case, just leave it in convert list and new master will take care of it after recovery. And if convert request returns other than DLM_NORMAL, convert thread will do the revert itself. So remove the revert logic in dlm_move_lockres_to_recovery_list.

changelog since v1:
Clean up convert_pending since it is now useless.

.

#5 lore raw

PATCH v2 1/1 ocfs2/dlm: fix race between convert and recovery

Quoted 21 lines · Joseph Qi

Hi Junxiao,

Quoted 11 lines · Junxiao Bi
Quoted 9 lines · Joseph Qi

Hi Junxiao,
This solution may have problem in the following scenario:
Consider dlm_send_remote_convert_request has taken too much time, and dlm_move_lockres_to_recovery_list runs first and new master will see this node is currently in convert list after recovery.
Then dlm_send_remote_convert_request returns other than DLM_NORMAL and it will revert it to grant list, then retry convert. This will makes this node and master inconsistent.
I will try to find another solution to resolve the race issue.

Quoted 3 lines

If master is down, no need retry convert. May check the return value of dlm_send_remote_convert_request(), if DLM_RECOVERING, don't retry, otherwise retry?

I want to keep the original logic. And for fixing the race case I described, how about the following idea?

Quoted 6 lines

Check the status DLM_NORMAL. If res->state is currently DLM_LOCK_RES_RECOVERING (set in dlm_move_lockres_to_recovery_list, means still in recovery) or res master changed (means new master has finished the recovery), reset the status to DLM_RECOVERING, just like the check at the beginning of dlmconvert_remote. Then it is now in grant list and outer will retry.

How this idea fix the race windows you described in patch log? Lock is reverted to granted list but dlm_send_remote_convert_request() return DLM_NORMAL.

Thanks,
Junxiao.

Quoted 53 lines
Quoted 51 lines

Thanks,
Junxiao.

Quoted 33 lines · Junxiao Bi
Quoted 31 lines
Quoted 29 lines
Quoted 29 lines

? 2015?9?18????7:25?Joseph Qi <[email protected]> ???

There is a race window between dlmconvert_remote and dlm_move_lockres_to_recovery_list, which will cause a lock with OCFS2_LOCK_BUSY in grant list, thus system hangs.

dlmconvert_remote

{	
       spin_lock(&res->spinlock);
       list_move_tail(&lock->list, &res->converting);
       lock->convert_pending = 1;
       spin_unlock(&res->spinlock);

       status = dlm_send_remote_convert_request();
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line

race window, master has queued ast and return DLM_NORMAL,

and then down before sending ast.
this node detects master down and calls dlm_move_lockres_to_recovery_list, which will revert the lock to grant list.
Then OCFS2_LOCK_BUSY won't be cleared as new master won't send ast any more because it thinks already be authorized.

spin_lock(&res->spinlock); lock->convert_pending = 0;
if (status != DLM_NORMAL)
dlm_revert_pending_convert(res, lock);
spin_unlock(&res->spinlock);
}

Quoted 9 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines

In this case, just leave it in convert list and new master will take care of it after recovery. And if convert request returns other than DLM_NORMAL, convert thread will do the revert itself. So remove the revert logic in dlm_move_lockres_to_recovery_list.

changelog since v1:
Clean up convert_pending since it is now useless.

.

author
#6 lore raw

PATCH v2 1/1 ocfs2/dlm: fix race between convert and recovery

Quoted 28 lines · Junxiao Bi
Quoted 21 lines · Joseph Qi

Hi Junxiao,

Quoted 11 lines · Junxiao Bi
Quoted 9 lines · Joseph Qi

Hi Junxiao,
This solution may have problem in the following scenario:
Consider dlm_send_remote_convert_request has taken too much time, and dlm_move_lockres_to_recovery_list runs first and new master will see this node is currently in convert list after recovery.
Then dlm_send_remote_convert_request returns other than DLM_NORMAL and it will revert it to grant list, then retry convert. This will makes this node and master inconsistent.
I will try to find another solution to resolve the race issue.

Quoted 3 lines

If master is down, no need retry convert. May check the return value of dlm_send_remote_convert_request(), if DLM_RECOVERING, don't retry, otherwise retry?

I want to keep the original logic. And for fixing the race case I described, how about the following idea?

Quoted 6 lines

Check the status DLM_NORMAL. If res->state is currently DLM_LOCK_RES_RECOVERING (set in dlm_move_lockres_to_recovery_list, means still in recovery) or res master changed (means new master has finished the recovery), reset the status to DLM_RECOVERING, just like the check at the beginning of dlmconvert_remote. Then it is now in grant list and outer will retry.

Quoted 4 lines

How this idea fix the race windows you described in patch log? Lock is reverted to granted list but dlm_send_remote_convert_request() return DLM_NORMAL.

As I described in the former mail, status is now reset to DLM_RECOVERING, caller will retry. And the original way will just wait forever.

Thanks
Joseph

Quoted 59 lines

Thanks,
Junxiao.

Quoted 43 lines
Quoted 43 lines

Thanks,
Junxiao.

Quoted 38 lines · Junxiao Bi
Quoted 36 lines
Quoted 34 lines
Quoted 34 lines

? 2015?9?18????7:25?Joseph Qi <[email protected]> ???

There is a race window between dlmconvert_remote and dlm_move_lockres_to_recovery_list, which will cause a lock with OCFS2_LOCK_BUSY in grant list, thus system hangs.

dlmconvert_remote

{	
       spin_lock(&res->spinlock);
       list_move_tail(&lock->list, &res->converting);
       lock->convert_pending = 1;
       spin_unlock(&res->spinlock);

       status = dlm_send_remote_convert_request();
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line

race window, master has queued ast and return DLM_NORMAL,

and then down before sending ast.
this node detects master down and calls dlm_move_lockres_to_recovery_list, which will revert the lock to grant list.
Then OCFS2_LOCK_BUSY won't be cleared as new master won't send ast any more because it thinks already be authorized.

spin_lock(&res->spinlock); lock->convert_pending = 0;
if (status != DLM_NORMAL)
dlm_revert_pending_convert(res, lock);
spin_unlock(&res->spinlock);
}

In this case, just leave it in convert list and new master will take care of it after recovery. And if convert request returns other than DLM_NORMAL, convert thread will do the revert itself. So remove the revert logic in dlm_move_lockres_to_recovery_list.

Quoted 10 lines
Quoted 8 lines
Quoted 4 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

changelog since v1:
Clean up convert_pending since it is now useless.

.

.

#7 lore raw

PATCH v2 1/1 ocfs2/dlm: fix race between convert and recovery

Quoted 29 lines · Joseph Qi
Quoted 28 lines · Junxiao Bi
Quoted 27 lines · Joseph Qi

Hi Junxiao,

Quoted 11 lines · Junxiao Bi
Quoted 9 lines · Joseph Qi

Hi Junxiao,
This solution may have problem in the following scenario:
Consider dlm_send_remote_convert_request has taken too much time, and dlm_move_lockres_to_recovery_list runs first and new master will see this node is currently in convert list after recovery.
Then dlm_send_remote_convert_request returns other than DLM_NORMAL and it will revert it to grant list, then retry convert. This will makes this node and master inconsistent.
I will try to find another solution to resolve the race issue.

Quoted 3 lines

If master is down, no need retry convert. May check the return value of dlm_send_remote_convert_request(), if DLM_RECOVERING, don't retry, otherwise retry?

I want to keep the original logic. And for fixing the race case I described, how about the following idea?

Check the status DLM_NORMAL. If res->state is currently DLM_LOCK_RES_RECOVERING (set in dlm_move_lockres_to_recovery_list, means still in recovery) or res master changed (means new master has finished the recovery), reset the status to DLM_RECOVERING, just like the check at the beginning of dlmconvert_remote. Then it is now in grant list and outer will retry.

Quoted 6 lines
Quoted 4 lines

How this idea fix the race windows you described in patch log? Lock is reverted to granted list but dlm_send_remote_convert_request() return DLM_NORMAL.

As I described in the former mail, status is now reset to DLM_RECOVERING, caller will retry. And the original way will just wait forever.

I mean convert request was sent to res master successfully, dlm_send_remote_convert_request() have return DLM_NORMAL, but before res master send ast it panic. Looks convert will not retry in this case.

Thanks,
Junxiao.

Quoted 65 lines

Thanks
Joseph

Quoted 59 lines

Thanks,
Junxiao.

Quoted 43 lines
Quoted 43 lines

Thanks,
Junxiao.

Quoted 38 lines · Junxiao Bi
Quoted 36 lines
Quoted 34 lines
Quoted 34 lines

? 2015?9?18????7:25?Joseph Qi <[email protected]> ???

There is a race window between dlmconvert_remote and dlm_move_lockres_to_recovery_list, which will cause a lock with OCFS2_LOCK_BUSY in grant list, thus system hangs.

dlmconvert_remote

{	
       spin_lock(&res->spinlock);
       list_move_tail(&lock->list, &res->converting);
       lock->convert_pending = 1;
       spin_unlock(&res->spinlock);

       status = dlm_send_remote_convert_request();
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line

race window, master has queued ast and return DLM_NORMAL,

and then down before sending ast.
this node detects master down and calls dlm_move_lockres_to_recovery_list, which will revert the lock to grant list.
Then OCFS2_LOCK_BUSY won't be cleared as new master won't send ast any more because it thinks already be authorized.

spin_lock(&res->spinlock); lock->convert_pending = 0;
if (status != DLM_NORMAL)
dlm_revert_pending_convert(res, lock);
spin_unlock(&res->spinlock);
}

In this case, just leave it in convert list and new master will take care of it after recovery. And if convert request returns other than DLM_NORMAL, convert thread will do the revert itself. So remove the revert logic in dlm_move_lockres_to_recovery_list.

Quoted 10 lines
Quoted 8 lines
Quoted 4 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

changelog since v1:
Clean up convert_pending since it is now useless.

.

.

author
#8 lore raw

PATCH v2 1/1 ocfs2/dlm: fix race between convert and recovery

Quoted 34 lines · Junxiao Bi
Quoted 33 lines · Joseph Qi
Quoted 32 lines · Junxiao Bi
Quoted 21 lines · Joseph Qi

Hi Junxiao,

Quoted 11 lines · Junxiao Bi
Quoted 9 lines · Joseph Qi

Hi Junxiao,
This solution may have problem in the following scenario:
Consider dlm_send_remote_convert_request has taken too much time, and dlm_move_lockres_to_recovery_list runs first and new master will see this node is currently in convert list after recovery.
Then dlm_send_remote_convert_request returns other than DLM_NORMAL and it will revert it to grant list, then retry convert. This will makes this node and master inconsistent.
I will try to find another solution to resolve the race issue.

Quoted 3 lines

If master is down, no need retry convert. May check the return value of dlm_send_remote_convert_request(), if DLM_RECOVERING, don't retry, otherwise retry?

I want to keep the original logic. And for fixing the race case I described, how about the following idea?

Quoted 6 lines

Check the status DLM_NORMAL. If res->state is currently DLM_LOCK_RES_RECOVERING (set in dlm_move_lockres_to_recovery_list, means still in recovery) or res master changed (means new master has finished the recovery), reset the status to DLM_RECOVERING, just like the check at the beginning of dlmconvert_remote. Then it is now in grant list and outer will retry.

How this idea fix the race windows you described in patch log? Lock is reverted to granted list but dlm_send_remote_convert_request() return DLM_NORMAL.

Quoted 6 lines
Quoted 2 lines

As I described in the former mail, status is now reset to DLM_RECOVERING, caller will retry. And the original way will just wait forever.

I mean convert request was sent to res master successfully, dlm_send_remote_convert_request() have return DLM_NORMAL, but before res master send ast it panic. Looks convert will not retry in this case.

The original way is just what you said.
And my idea is letting dlmlock retry by resetting status to DLM_RECOVERING in such a case.

Quoted 71 lines

Thanks,
Junxiao.

Quoted 65 lines

Thanks
Joseph

Quoted 59 lines

Thanks,
Junxiao.

Quoted 43 lines
Quoted 43 lines

Thanks,
Junxiao.

Quoted 38 lines · Junxiao Bi
Quoted 36 lines
Quoted 34 lines
Quoted 34 lines

? 2015?9?18????7:25?Joseph Qi <[email protected]> ???

There is a race window between dlmconvert_remote and dlm_move_lockres_to_recovery_list, which will cause a lock with OCFS2_LOCK_BUSY in grant list, thus system hangs.

dlmconvert_remote

{	
       spin_lock(&res->spinlock);
       list_move_tail(&lock->list, &res->converting);
       lock->convert_pending = 1;
       spin_unlock(&res->spinlock);

       status = dlm_send_remote_convert_request();
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line

race window, master has queued ast and return DLM_NORMAL,

and then down before sending ast.
this node detects master down and calls dlm_move_lockres_to_recovery_list, which will revert the lock to grant list.
Then OCFS2_LOCK_BUSY won't be cleared as new master won't send ast any more because it thinks already be authorized.

spin_lock(&res->spinlock); lock->convert_pending = 0;
if (status != DLM_NORMAL)
dlm_revert_pending_convert(res, lock);
spin_unlock(&res->spinlock);
}

In this case, just leave it in convert list and new master will take care of it after recovery. And if convert request returns other than DLM_NORMAL, convert thread will do the revert itself. So remove the revert logic in dlm_move_lockres_to_recovery_list.

Quoted 10 lines
Quoted 8 lines
Quoted 4 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

changelog since v1:
Clean up convert_pending since it is now useless.

.

.

.

#9 lore raw

PATCH v2 1/1 ocfs2/dlm: fix race between convert and recovery

Quoted 37 lines · Joseph Qi
Quoted 36 lines · Junxiao Bi
Quoted 35 lines · Joseph Qi
Quoted 28 lines · Junxiao Bi
Quoted 21 lines · Joseph Qi

Hi Junxiao,

Quoted 11 lines · Junxiao Bi
Quoted 9 lines · Joseph Qi

Hi Junxiao,
This solution may have problem in the following scenario:
Consider dlm_send_remote_convert_request has taken too much time, and dlm_move_lockres_to_recovery_list runs first and new master will see this node is currently in convert list after recovery.
Then dlm_send_remote_convert_request returns other than DLM_NORMAL and it will revert it to grant list, then retry convert. This will makes this node and master inconsistent.
I will try to find another solution to resolve the race issue.

Quoted 3 lines

If master is down, no need retry convert. May check the return value of dlm_send_remote_convert_request(), if DLM_RECOVERING, don't retry, otherwise retry?

I want to keep the original logic. And for fixing the race case I described, how about the following idea?

Quoted 6 lines

Check the status DLM_NORMAL. If res->state is currently DLM_LOCK_RES_RECOVERING (set in dlm_move_lockres_to_recovery_list, means still in recovery) or res master changed (means new master has finished the recovery), reset the status to DLM_RECOVERING, just like the check at the beginning of dlmconvert_remote. Then it is now in grant list and outer will retry.

Quoted 4 lines

How this idea fix the race windows you described in patch log? Lock is reverted to granted list but dlm_send_remote_convert_request() return DLM_NORMAL.

As I described in the former mail, status is now reset to DLM_RECOVERING, caller will retry. And the original way will just wait forever.

Quoted 7 lines
Quoted 4 lines

I mean convert request was sent to res master successfully, dlm_send_remote_convert_request() have return DLM_NORMAL, but before res master send ast it panic. Looks convert will not retry in this case.

The original way is just what you said.
And my idea is letting dlmlock retry by resetting status to DLM_RECOVERING in such a case.

So convert_pending is still dropped? If so when dlm_send_remote_convert_request() return DLM_RECOVERING, should reset it to DLM_NORMAL and didn't revert lock to granted list. Retry is not needed.

Thanks,
Junxiao.

Quoted 74 lines
Quoted 72 lines

Thanks,
Junxiao.

Quoted 65 lines

Thanks
Joseph

Quoted 59 lines

Thanks,
Junxiao.

Quoted 43 lines
Quoted 43 lines

Thanks,
Junxiao.

Quoted 38 lines · Junxiao Bi
Quoted 36 lines
Quoted 34 lines
Quoted 34 lines

? 2015?9?18????7:25?Joseph Qi <[email protected]> ???

There is a race window between dlmconvert_remote and dlm_move_lockres_to_recovery_list, which will cause a lock with OCFS2_LOCK_BUSY in grant list, thus system hangs.

dlmconvert_remote

{	
       spin_lock(&res->spinlock);
       list_move_tail(&lock->list, &res->converting);
       lock->convert_pending = 1;
       spin_unlock(&res->spinlock);

       status = dlm_send_remote_convert_request();
Quoted 1 line
Quoted 1 line
Quoted 1 line

race window, master has queued ast and return DLM_NORMAL,

and then down before sending ast.
this node detects master down and calls dlm_move_lockres_to_recovery_list, which will revert the lock to grant list.
Then OCFS2_LOCK_BUSY won't be cleared as new master won't send ast any more because it thinks already be authorized.

spin_lock(&res->spinlock); lock->convert_pending = 0;
if (status != DLM_NORMAL)
dlm_revert_pending_convert(res, lock);
spin_unlock(&res->spinlock);
}

In this case, just leave it in convert list and new master will take care of it after recovery. And if convert request returns other than DLM_NORMAL, convert thread will do the revert itself. So remove the revert logic in dlm_move_lockres_to_recovery_list.

Quoted 10 lines
Quoted 8 lines
Quoted 4 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

changelog since v1:
Clean up convert_pending since it is now useless.

.

.

.