Skip to contentSkip to conversation

ocfs2/dlm: fix race between convert and recovery

PATCH New Joseph Qi 2 participants 6 messages Sep 24, 2015 – Sep 25, 2015 OCFS2

lore
Share & tools
lore

Joseph Qi

author
#0 lore raw

PATCH v3 1/2 ocfs2/dlm: fix race between convert and recovery

There is a race window between dlmconvert_remote and dlm_move_lockres_to_recovery_list, which will cause a lock with OCFS2_LOCK_BUSY in grant list, thus system hangs.

dlmconvert_remote

{
        spin_lock(&res->spinlock);
        list_move_tail(&lock->list, &res->converting);
        lock->convert_pending = 1;
        spin_unlock(&res->spinlock);

        status = dlm_send_remote_convert_request();
        >>>>>> race window, master has queued ast and return DLM_NORMAL,
               and then down before sending ast.
               this node detects master down and calls
               dlm_move_lockres_to_recovery_list, which will revert the
               lock to grant list.
               Then OCFS2_LOCK_BUSY won't be cleared as new master won't
               send ast any more because it thinks already be authorized.

        spin_lock(&res->spinlock);
        lock->convert_pending = 0;
        if (status != DLM_NORMAL)
                dlm_revert_pending_convert(res, lock);
        spin_unlock(&res->spinlock);
}

In this case, check if res->state has DLM_LOCK_RES_RECOVERING bit set (res is still in recovering) or res master changed (new master has finished recovery), reset the status to DLM_RECOVERING, then it will retry convert.

---
 fs/ocfs2/dlm/dlmconvert.c | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)
fs/ocfs2/dlm/dlmconvert.c +10 -1
index e36d63f..9e6116e 100644
@@ -262,6 +262,7 @@ enum dlm_status dlmconvert_remote(struct dlm_ctxt *dlm,
 				  struct dlm_lock *lock, int flags, int type)
 {
 	enum dlm_status status;
+	u8 old_owner = res->owner;
 
 	mlog(0, "type=%d, convert_type=%d, busy=%d\n", lock->ml.type,
 	     lock->ml.convert_type, res->state & DLM_LOCK_RES_IN_PROGRESS);
@@ -316,11 +317,19 @@ enum dlm_status dlmconvert_remote(struct dlm_ctxt *dlm,
 	spin_lock(&res->spinlock);
 	res->state &= ~DLM_LOCK_RES_IN_PROGRESS;
 	lock->convert_pending = 0;
-	/* if it failed, move it back to granted queue */
+	/* if it failed, move it back to granted queue.
+	 * if master returns DLM_NORMAL and then down before sending ast,
+	 * it may have already been moved to granted queue, reset to
+	 * DLM_RECOVERING and retry convert */
 	if (status != DLM_NORMAL) {
 		if (status != DLM_NOTQUEUED)
 			dlm_error(status);
 		dlm_revert_pending_convert(res, lock);
+	} else if ((res->state & DLM_LOCK_RES_RECOVERING) ||
+			(old_owner != res->owner)) {
+		mlog(0, "res %.*s is in recovering or has been recovered.\n",
+				res->lockname.len, res->lockname.name);
+		status = DLM_RECOVERING;
 	}
 bail:
 	spin_unlock(&res->spinlock);
Signature · 1 line
1.8.4.3
#1 lore raw

PATCH v3 1/2 ocfs2/dlm: fix race between convert and recovery

Quoted 11 lines · Joseph Qi

There is a race window between dlmconvert_remote and dlm_move_lockres_to_recovery_list, which will cause a lock with OCFS2_LOCK_BUSY in grant list, thus system hangs.

dlmconvert_remote
{
spin_lock(&res->spinlock); list_move_tail(&lock->list, &res->converting); lock->convert_pending = 1; spin_unlock(&res->spinlock);

Quoted 8 lines

status = dlm_send_remote_convert_request();
>>>>>> race window, master has queued ast and return DLM_NORMAL,
and then down before sending ast.
this node detects master down and calls dlm_move_lockres_to_recovery_list, which will revert the lock to grant list.
Then OCFS2_LOCK_BUSY won't be cleared as new master won't send ast any more because it thinks already be authorized.

How this race windowed fixed?
the process have sent convert request to master node successfully(return value DLM_NORMAL) then wait on LOCK_BUSY, then when master node panic before send out ast, dlm_move_lockres_to_recovery_list() move the lock to grant list. Ast never come.

Thanks,
Junxiao.

Quoted 54 lines
        spin_lock(&res->spinlock);
        lock->convert_pending = 0;
        if (status != DLM_NORMAL)
                dlm_revert_pending_convert(res, lock);
        spin_unlock(&res->spinlock);
}

In this case, check if res->state has DLM_LOCK_RES_RECOVERING bit set (res is still in recovering) or res master changed (new master has finished recovery), reset the status to DLM_RECOVERING, then it will retry convert.

---
 fs/ocfs2/dlm/dlmconvert.c | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)
fs/ocfs2/dlm/dlmconvert.c +10 -1
index e36d63f..9e6116e 100644
@@ -262,6 +262,7 @@ enum dlm_status dlmconvert_remote(struct dlm_ctxt *dlm,
 				  struct dlm_lock *lock, int flags, int type)
 {
 	enum dlm_status status;
+	u8 old_owner = res->owner;
 
 	mlog(0, "type=%d, convert_type=%d, busy=%d\n", lock->ml.type,
 	     lock->ml.convert_type, res->state & DLM_LOCK_RES_IN_PROGRESS);
@@ -316,11 +317,19 @@ enum dlm_status dlmconvert_remote(struct dlm_ctxt *dlm,
 	spin_lock(&res->spinlock);
 	res->state &= ~DLM_LOCK_RES_IN_PROGRESS;
 	lock->convert_pending = 0;
-	/* if it failed, move it back to granted queue */
+	/* if it failed, move it back to granted queue.
+	 * if master returns DLM_NORMAL and then down before sending ast,
+	 * it may have already been moved to granted queue, reset to
+	 * DLM_RECOVERING and retry convert */
 	if (status != DLM_NORMAL) {
 		if (status != DLM_NOTQUEUED)
 			dlm_error(status);
 		dlm_revert_pending_convert(res, lock);
+	} else if ((res->state & DLM_LOCK_RES_RECOVERING) ||
+			(old_owner != res->owner)) {
+		mlog(0, "res %.*s is in recovering or has been recovered.\n",
+				res->lockname.len, res->lockname.name);
+		status = DLM_RECOVERING;
 	}
 bail:
 	spin_unlock(&res->spinlock);
author
#2 lore raw

PATCH v3 1/2 ocfs2/dlm: fix race between convert and recovery

Hi Junxiao,

Quoted 20 lines · Junxiao Bi
Quoted 11 lines · Joseph Qi

There is a race window between dlmconvert_remote and dlm_move_lockres_to_recovery_list, which will cause a lock with OCFS2_LOCK_BUSY in grant list, thus system hangs.

dlmconvert_remote
{
spin_lock(&res->spinlock); list_move_tail(&lock->list, &res->converting); lock->convert_pending = 1; spin_unlock(&res->spinlock);

Quoted 8 lines

status = dlm_send_remote_convert_request();
>>>>>> race window, master has queued ast and return DLM_NORMAL,
and then down before sending ast.
this node detects master down and calls dlm_move_lockres_to_recovery_list, which will revert the lock to grant list.
Then OCFS2_LOCK_BUSY won't be cleared as new master won't send ast any more because it thinks already be authorized.

Quoted 5 lines

How this race windowed fixed?
the process have sent convert request to master node successfully(return value DLM_NORMAL) then wait on LOCK_BUSY, then when master node panic before send out ast, dlm_move_lockres_to_recovery_list() move the lock to grant list. Ast never come.

res->state is now having DLM_LOCK_RES_RECOVERING set. This patch will reset the status to DLM_RECOVERING and then retry convert request. Then new master will handle it and send ast.

Quoted 62 lines

Thanks,
Junxiao.

Quoted 44 lines
        spin_lock(&res->spinlock);
        lock->convert_pending = 0;
        if (status != DLM_NORMAL)
                dlm_revert_pending_convert(res, lock);
        spin_unlock(&res->spinlock);
}

In this case, check if res->state has DLM_LOCK_RES_RECOVERING bit set (res is still in recovering) or res master changed (new master has finished recovery), reset the status to DLM_RECOVERING, then it will retry convert.

---
 fs/ocfs2/dlm/dlmconvert.c | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)
fs/ocfs2/dlm/dlmconvert.c +5 -1
index e36d63f..9e6116e 100644
@@ -262,6 +262,7 @@ enum dlm_status dlmconvert_remote(struct dlm_ctxt *dlm,
 				  struct dlm_lock *lock, int flags, int type)
 {
 	enum dlm_status status;
+	u8 old_owner = res->owner;
 
 	mlog(0, "type=%d, convert_type=%d, busy=%d\n", lock->ml.type,
 	     lock->ml.convert_type, res->state & DLM_LOCK_RES_IN_PROGRESS);
@@ -316,11 +317,19 @@ enum dlm_status dlmconvert_remote(struct dlm_ctxt *dlm,
 	spin_lock(&res->spinlock);
 	res->state &= ~DLM_LOCK_RES_IN_PROGRESS;
 	lock->convert_pending = 0;
-	/* if it failed, move it back to granted queue */
+	/* if it failed, move it back to granted queue.
+	 * if master returns DLM_NORMAL and then down before sending ast,
+	 * it may have already been moved to granted queue, reset to
+	 * DLM_RECOVERING and retry convert */
 	if (status != DLM_NORMAL) {
 		if (status != DLM_NOTQUEUED)

fs/ocfs2/dlm/dlmconvert.c enum dlm_status dlmconvert_remote(struct dlm_ctxt *dlm, · line 335

Quoted 10 lines
fs/ocfs2/dlm/dlmconvert.c +5 -0
 			dlm_error(status);
 		dlm_revert_pending_convert(res, lock);
+	} else if ((res->state & DLM_LOCK_RES_RECOVERING) ||
+			(old_owner != res->owner)) {
+		mlog(0, "res %.*s is in recovering or has been recovered.\n",
+				res->lockname.len, res->lockname.name);
+		status = DLM_RECOVERING;
 	}
 bail:
 	spin_unlock(&res->spinlock);

.

#3 lore raw

PATCH v3 1/2 ocfs2/dlm: fix race between convert and recovery

Quoted 23 lines · Joseph Qi

Hi Junxiao,

Quoted 20 lines · Junxiao Bi
Quoted 19 lines · Joseph Qi

There is a race window between dlmconvert_remote and dlm_move_lockres_to_recovery_list, which will cause a lock with OCFS2_LOCK_BUSY in grant list, thus system hangs.

dlmconvert_remote
{
spin_lock(&res->spinlock); list_move_tail(&lock->list, &res->converting); lock->convert_pending = 1; spin_unlock(&res->spinlock);

status = dlm_send_remote_convert_request();
>>>>>> race window, master has queued ast and return DLM_NORMAL,
and then down before sending ast.
this node detects master down and calls dlm_move_lockres_to_recovery_list, which will revert the lock to grant list.
Then OCFS2_LOCK_BUSY won't be cleared as new master won't send ast any more because it thinks already be authorized.

Quoted 6 lines
Quoted 5 lines

How this race windowed fixed?
the process have sent convert request to master node successfully(return value DLM_NORMAL) then wait on LOCK_BUSY, then when master node panic before send out ast, dlm_move_lockres_to_recovery_list() move the lock to grant list. Ast never come.

res->state is now having DLM_LOCK_RES_RECOVERING set.

But what happened if master node panic after dlm_send_remote_convert_request() return DLM_NORMAL and wait on LOCK_BUSY?

Thanks,
Junxiao.

This patch will

Quoted 67 lines

reset the status to DLM_RECOVERING and then retry convert request. Then new master will handle it and send ast.

Quoted 62 lines

Thanks,
Junxiao.

Quoted 44 lines
        spin_lock(&res->spinlock);
        lock->convert_pending = 0;
        if (status != DLM_NORMAL)
                dlm_revert_pending_convert(res, lock);
        spin_unlock(&res->spinlock);
}

In this case, check if res->state has DLM_LOCK_RES_RECOVERING bit set (res is still in recovering) or res master changed (new master has finished recovery), reset the status to DLM_RECOVERING, then it will retry convert.

---
 fs/ocfs2/dlm/dlmconvert.c | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)
fs/ocfs2/dlm/dlmconvert.c +5 -1
index e36d63f..9e6116e 100644
@@ -262,6 +262,7 @@ enum dlm_status dlmconvert_remote(struct dlm_ctxt *dlm,
 				  struct dlm_lock *lock, int flags, int type)
 {
 	enum dlm_status status;
+	u8 old_owner = res->owner;
 
 	mlog(0, "type=%d, convert_type=%d, busy=%d\n", lock->ml.type,
 	     lock->ml.convert_type, res->state & DLM_LOCK_RES_IN_PROGRESS);
@@ -316,11 +317,19 @@ enum dlm_status dlmconvert_remote(struct dlm_ctxt *dlm,
 	spin_lock(&res->spinlock);
 	res->state &= ~DLM_LOCK_RES_IN_PROGRESS;
 	lock->convert_pending = 0;
-	/* if it failed, move it back to granted queue */
+	/* if it failed, move it back to granted queue.
+	 * if master returns DLM_NORMAL and then down before sending ast,
+	 * it may have already been moved to granted queue, reset to
+	 * DLM_RECOVERING and retry convert */
 	if (status != DLM_NORMAL) {
 		if (status != DLM_NOTQUEUED)

fs/ocfs2/dlm/dlmconvert.c enum dlm_status dlmconvert_remote(struct dlm_ctxt *dlm, · line 335

Quoted 10 lines
fs/ocfs2/dlm/dlmconvert.c +5 -0
 			dlm_error(status);
 		dlm_revert_pending_convert(res, lock);
+	} else if ((res->state & DLM_LOCK_RES_RECOVERING) ||
+			(old_owner != res->owner)) {
+		mlog(0, "res %.*s is in recovering or has been recovered.\n",
+				res->lockname.len, res->lockname.name);
+		status = DLM_RECOVERING;
 	}
 bail:
 	spin_unlock(&res->spinlock);

.

author
#4 lore raw

PATCH v3 1/2 ocfs2/dlm: fix race between convert and recovery

Quoted 24 lines · Junxiao Bi
Quoted 23 lines · Joseph Qi

Hi Junxiao,

Quoted 20 lines · Junxiao Bi
Quoted 19 lines · Joseph Qi

There is a race window between dlmconvert_remote and dlm_move_lockres_to_recovery_list, which will cause a lock with OCFS2_LOCK_BUSY in grant list, thus system hangs.

dlmconvert_remote
{
spin_lock(&res->spinlock); list_move_tail(&lock->list, &res->converting); lock->convert_pending = 1; spin_unlock(&res->spinlock);

status = dlm_send_remote_convert_request();
>>>>>> race window, master has queued ast and return DLM_NORMAL,
and then down before sending ast.
this node detects master down and calls dlm_move_lockres_to_recovery_list, which will revert the lock to grant list.
Then OCFS2_LOCK_BUSY won't be cleared as new master won't send ast any more because it thinks already be authorized.

Quoted 9 lines
Quoted 6 lines
Quoted 5 lines

How this race windowed fixed?
the process have sent convert request to master node successfully(return value DLM_NORMAL) then wait on LOCK_BUSY, then when master node panic before send out ast, dlm_move_lockres_to_recovery_list() move the lock to grant list. Ast never come.

res->state is now having DLM_LOCK_RES_RECOVERING set.

But what happened if master node panic after dlm_send_remote_convert_request() return DLM_NORMAL and wait on LOCK_BUSY?

Only two cases that it will call dlm_revert_pending_convert:
1) request returns other than DLM_NORMAL, current code will retry;
2) request returns DLM_NORMAL and recovery does it, this is the case we discuss. Consider the lockres spinlock has to be taken when doing this, only dlm_move_lockres_to_recovery_list runs first before request return code handling can we have the race window. If convert thread runs first, it's still in convert list, and recovery won't move it to grant list because convert_pending is cleared. And the new master will send ast after recovery.

Thanks,
Joseph

Quoted 75 lines

Thanks,
Junxiao.

This patch will

Quoted 67 lines

reset the status to DLM_RECOVERING and then retry convert request. Then new master will handle it and send ast.

Quoted 62 lines

Thanks,
Junxiao.

Quoted 44 lines
        spin_lock(&res->spinlock);
        lock->convert_pending = 0;
        if (status != DLM_NORMAL)
                dlm_revert_pending_convert(res, lock);
        spin_unlock(&res->spinlock);
}

In this case, check if res->state has DLM_LOCK_RES_RECOVERING bit set (res is still in recovering) or res master changed (new master has finished recovery), reset the status to DLM_RECOVERING, then it will retry convert.

---
 fs/ocfs2/dlm/dlmconvert.c | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)
fs/ocfs2/dlm/dlmconvert.c +5 -1
index e36d63f..9e6116e 100644
@@ -262,6 +262,7 @@ enum dlm_status dlmconvert_remote(struct dlm_ctxt *dlm,
 				  struct dlm_lock *lock, int flags, int type)
 {
 	enum dlm_status status;
+	u8 old_owner = res->owner;
 
 	mlog(0, "type=%d, convert_type=%d, busy=%d\n", lock->ml.type,
 	     lock->ml.convert_type, res->state & DLM_LOCK_RES_IN_PROGRESS);
@@ -316,11 +317,19 @@ enum dlm_status dlmconvert_remote(struct dlm_ctxt *dlm,
 	spin_lock(&res->spinlock);
 	res->state &= ~DLM_LOCK_RES_IN_PROGRESS;
 	lock->convert_pending = 0;
-	/* if it failed, move it back to granted queue */
+	/* if it failed, move it back to granted queue.
+	 * if master returns DLM_NORMAL and then down before sending ast,
+	 * it may have already been moved to granted queue, reset to
+	 * DLM_RECOVERING and retry convert */
 	if (status != DLM_NORMAL) {
 		if (status != DLM_NOTQUEUED)

fs/ocfs2/dlm/dlmconvert.c enum dlm_status dlmconvert_remote(struct dlm_ctxt *dlm, · line 335

Quoted 10 lines
fs/ocfs2/dlm/dlmconvert.c +5 -0
 			dlm_error(status);
 		dlm_revert_pending_convert(res, lock);
+	} else if ((res->state & DLM_LOCK_RES_RECOVERING) ||
+			(old_owner != res->owner)) {
+		mlog(0, "res %.*s is in recovering or has been recovered.\n",
+				res->lockname.len, res->lockname.name);
+		status = DLM_RECOVERING;
 	}
 bail:
 	spin_unlock(&res->spinlock);

.

.

author
#5 lore raw

PATCH v3 1/2 ocfs2/dlm: fix race between convert and recovery

Hi Junxiao,
Do you have any other questions about this solution?

Quoted 131 lines · Joseph Qi
Quoted 24 lines · Junxiao Bi
Quoted 23 lines · Joseph Qi

Hi Junxiao,

Quoted 20 lines · Junxiao Bi
Quoted 19 lines · Joseph Qi

There is a race window between dlmconvert_remote and dlm_move_lockres_to_recovery_list, which will cause a lock with OCFS2_LOCK_BUSY in grant list, thus system hangs.

dlmconvert_remote
{
spin_lock(&res->spinlock); list_move_tail(&lock->list, &res->converting); lock->convert_pending = 1; spin_unlock(&res->spinlock);

status = dlm_send_remote_convert_request();
>>>>>> race window, master has queued ast and return DLM_NORMAL,
and then down before sending ast.
this node detects master down and calls dlm_move_lockres_to_recovery_list, which will revert the lock to grant list.
Then OCFS2_LOCK_BUSY won't be cleared as new master won't send ast any more because it thinks already be authorized.

Quoted 9 lines
Quoted 6 lines
Quoted 5 lines

How this race windowed fixed?
the process have sent convert request to master node successfully(return value DLM_NORMAL) then wait on LOCK_BUSY, then when master node panic before send out ast, dlm_move_lockres_to_recovery_list() move the lock to grant list. Ast never come.

res->state is now having DLM_LOCK_RES_RECOVERING set.

But what happened if master node panic after dlm_send_remote_convert_request() return DLM_NORMAL and wait on LOCK_BUSY?

Only two cases that it will call dlm_revert_pending_convert:
1) request returns other than DLM_NORMAL, current code will retry;
2) request returns DLM_NORMAL and recovery does it, this is the case we discuss. Consider the lockres spinlock has to be taken when doing this, only dlm_move_lockres_to_recovery_list runs first before request return code handling can we have the race window. If convert thread runs first, it's still in convert list, and recovery won't move it to grant list because convert_pending is cleared. And the new master will send ast after recovery.

Thanks,
Joseph

Quoted 73 lines

Thanks,
Junxiao.

This patch will

Quoted 67 lines

reset the status to DLM_RECOVERING and then retry convert request. Then new master will handle it and send ast.

Quoted 62 lines

Thanks,
Junxiao.

Quoted 44 lines
        spin_lock(&res->spinlock);
        lock->convert_pending = 0;
        if (status != DLM_NORMAL)
                dlm_revert_pending_convert(res, lock);
        spin_unlock(&res->spinlock);
}

In this case, check if res->state has DLM_LOCK_RES_RECOVERING bit set (res is still in recovering) or res master changed (new master has finished recovery), reset the status to DLM_RECOVERING, then it will retry convert.

---
 fs/ocfs2/dlm/dlmconvert.c | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)
fs/ocfs2/dlm/dlmconvert.c +5 -1
index e36d63f..9e6116e 100644
@@ -262,6 +262,7 @@ enum dlm_status dlmconvert_remote(struct dlm_ctxt *dlm,
 				  struct dlm_lock *lock, int flags, int type)
 {
 	enum dlm_status status;
+	u8 old_owner = res->owner;
 
 	mlog(0, "type=%d, convert_type=%d, busy=%d\n", lock->ml.type,
 	     lock->ml.convert_type, res->state & DLM_LOCK_RES_IN_PROGRESS);
@@ -316,11 +317,19 @@ enum dlm_status dlmconvert_remote(struct dlm_ctxt *dlm,
 	spin_lock(&res->spinlock);
 	res->state &= ~DLM_LOCK_RES_IN_PROGRESS;
 	lock->convert_pending = 0;
-	/* if it failed, move it back to granted queue */
+	/* if it failed, move it back to granted queue.
+	 * if master returns DLM_NORMAL and then down before sending ast,
+	 * it may have already been moved to granted queue, reset to
+	 * DLM_RECOVERING and retry convert */
 	if (status != DLM_NORMAL) {
 		if (status != DLM_NOTQUEUED)

fs/ocfs2/dlm/dlmconvert.c enum dlm_status dlmconvert_remote(struct dlm_ctxt *dlm, · line 335

Quoted 10 lines
fs/ocfs2/dlm/dlmconvert.c +5 -0
 			dlm_error(status);
 		dlm_revert_pending_convert(res, lock);
+	} else if ((res->state & DLM_LOCK_RES_RECOVERING) ||
+			(old_owner != res->owner)) {
+		mlog(0, "res %.*s is in recovering or has been recovered.\n",
+				res->lockname.len, res->lockname.name);
+		status = DLM_RECOVERING;
 	}
 bail:
 	spin_unlock(&res->spinlock);

.

Quoted 2 lines

.

_______________________________________________
Ocfs2-devel mailing list
Ocfs2-devel at oss.oracle.com
https://oss.oracle.com/mailman/listinfo/ocfs2-devel