Skip to contentSkip to conversation

ocfs2: wait for recovering done after direct unlock request

PATCH New Changwei Ge 3 participants 11 messages Feb 14, 2019 – Sep 17, 2019 OCFS2

lore
Share & tools
lore
#0 lore raw

PATCH 1/1 ocfs2: wait for recovering done after direct unlock request

There is scenario causing ocfs2 umount hang when multiple hosts are rebooting at the same time.

NODE1 NODE2 NODE3
send unlock requset to NODE2
dies
become recovery master recover NODE2
find NODE2 dead
mark resource RECOVERING
directly remove lock from grant list
calculate usage but RECOVERING marked
**miss the window of purging
clear RECOVERING

To reproduce this iusse, crash a host and then umount ocfs2 from another node.

To sovle this, just let unlock progress wait for recovery done.

---
 fs/ocfs2/dlm/dlmunlock.c | 23 +++++++++++++++++++----
 1 file changed, 19 insertions(+), 4 deletions(-)
fs/ocfs2/dlm/dlmunlock.c +19 -4
index 63d701c..c8e9b70 100644
@@ -105,7 +105,8 @@ static enum dlm_status dlmunlock_common(struct dlm_ctxt *dlm,
 	enum dlm_status status;
 	int actions = 0;
 	int in_use;
-        u8 owner;
+	u8 owner;
+	int recovery_wait = 0;
 
 	mlog(0, "master_node = %d, valblk = %d\n", master_node,
 	     flags & LKM_VALBLK);
@@ -208,9 +209,12 @@ static enum dlm_status dlmunlock_common(struct dlm_ctxt *dlm,
 		}
 		if (flags & LKM_CANCEL)
 			lock->cancel_pending = 0;
-		else
-			lock->unlock_pending = 0;
-
+		else {
+			if (!lock->unlock_pending)
+				recovery_wait = 1;
+			else
+				lock->unlock_pending = 0;
+		}
 	}
 
 	/* get an extra ref on lock.  if we are just switching
@@ -244,6 +248,17 @@ static enum dlm_status dlmunlock_common(struct dlm_ctxt *dlm,
 	spin_unlock(&res->spinlock);
 	wake_up(&res->wq);
 
+	if (recovery_wait) {
+		spin_lock(&res->spinlock);
+		/* Unlock request will directly succeed after owner dies,
+		 * and the lock is already removed from grant list. We have to
+		 * wait for RECOVERING done or we miss the chance to purge it
+		 * since the removement is much faster than RECOVERING proc.
+		 */
+		__dlm_wait_on_lockres_flags(res, DLM_LOCK_RES_RECOVERING);
+		spin_unlock(&res->spinlock);
+	}
+
 	/* let the caller's final dlm_lock_put handle the actual kfree */
 	if (actions & DLM_UNLOCK_FREE_LOCK) {
 		/* this should always be coupled with list removal */
Signature · 1 line
2.7.4
#1 lore raw

PATCH 1/1 ocfs2: wait for recovering done after direct unlock request

Hi Changwei,

The DLM process is a little bit complex, so I suggest pasting the code path. And I wonder if my code is right?

Thanks,
Jun

Quoted 11 lines · Changwei Ge

There is scenario causing ocfs2 umount hang when multiple hosts are rebooting at the same time.

NODE1 NODE2 NODE3
send unlock requset to NODE2
dies
become recovery master recover NODE2
find NODE2 dead
mark resource RECOVERING
directly remove lock from grant list

dlm_do_local_recovery_cleanup
dlm_move_lockres_to_recovery_list

    res->state |= DLM_LOCK_RES_RECOVERING;
    list_add_tail(&res->recovering, &dlm->reco.resources);
Quoted 2 lines

calculate usage but RECOVERING marked
**miss the window of purging

dlmunlock
dlmunlock_remote
dlmunlock_common // unlock successfully directly

dlm_lockres_calc_usage

    __dlm_lockres_calc_usage
      __dlm_lockres_unused
        if (res->state & (DLM_LOCK_RES_RECOVERING| // won't purge lock as DLM_LOCK_RES_RECOVERING is set
Quoted 1 line

clear RECOVERING

dlm_finish_local_lockres_recovery
res->state &= ~DLM_LOCK_RES_RECOVERING;

Could you help explaining where getting stuck?

Quoted 60 lines

To reproduce this iusse, crash a host and then umount ocfs2 from another node.

To sovle this, just let unlock progress wait for recovery done.

---
 fs/ocfs2/dlm/dlmunlock.c | 23 +++++++++++++++++++----
 1 file changed, 19 insertions(+), 4 deletions(-)
fs/ocfs2/dlm/dlmunlock.c +19 -4
index 63d701c..c8e9b70 100644
@@ -105,7 +105,8 @@ static enum dlm_status dlmunlock_common(struct dlm_ctxt *dlm,
 	enum dlm_status status;
 	int actions = 0;
 	int in_use;
-        u8 owner;
+	u8 owner;
+	int recovery_wait = 0;
 
 	mlog(0, "master_node = %d, valblk = %d\n", master_node,
 	     flags & LKM_VALBLK);
@@ -208,9 +209,12 @@ static enum dlm_status dlmunlock_common(struct dlm_ctxt *dlm,
 		}
 		if (flags & LKM_CANCEL)
 			lock->cancel_pending = 0;
-		else
-			lock->unlock_pending = 0;
-
+		else {
+			if (!lock->unlock_pending)
+				recovery_wait = 1;
+			else
+				lock->unlock_pending = 0;
+		}
 	}
 
 	/* get an extra ref on lock.  if we are just switching
@@ -244,6 +248,17 @@ static enum dlm_status dlmunlock_common(struct dlm_ctxt *dlm,
 	spin_unlock(&res->spinlock);
 	wake_up(&res->wq);
 
+	if (recovery_wait) {
+		spin_lock(&res->spinlock);
+		/* Unlock request will directly succeed after owner dies,
+		 * and the lock is already removed from grant list. We have to
+		 * wait for RECOVERING done or we miss the chance to purge it
+		 * since the removement is much faster than RECOVERING proc.
+		 */
+		__dlm_wait_on_lockres_flags(res, DLM_LOCK_RES_RECOVERING);
+		spin_unlock(&res->spinlock);
+	}
+
 	/* let the caller's final dlm_lock_put handle the actual kfree */
 	if (actions & DLM_UNLOCK_FREE_LOCK) {
 		/* this should always be coupled with list removal */
#2 lore raw

PATCH 1/1 ocfs2: wait for recovering done after direct unlock request

Hi Jun,

Quoted 27 lines · piaojun

Hi Changwei,

The DLM process is a little bit complex, so I suggest pasting the code path. And I wonder if my code is right?

Thanks,
Jun

Quoted 11 lines · Changwei Ge

There is scenario causing ocfs2 umount hang when multiple hosts are rebooting at the same time.

NODE1 NODE2 NODE3
send unlock requset to NODE2
dies
become recovery master recover NODE2
find NODE2 dead
mark resource RECOVERING
directly remove lock from grant list

dlm_do_local_recovery_cleanup
dlm_move_lockres_to_recovery_list

     res->state |= DLM_LOCK_RES_RECOVERING;
     list_add_tail(&res->recovering, &dlm->reco.resources);
Quoted 2 lines

calculate usage but RECOVERING marked
**miss the window of purging

Quoted 8 lines

dlmunlock
dlmunlock_remote
dlmunlock_common // unlock successfully directly

   dlm_lockres_calc_usage
     __dlm_lockres_calc_usage
       __dlm_lockres_unused
         if (res->state & (DLM_LOCK_RES_RECOVERING| // won't purge lock as DLM_LOCK_RES_RECOVERING is set

True.

Quoted 6 lines
Quoted 2 lines

clear RECOVERING

dlm_finish_local_lockres_recovery
res->state &= ~DLM_LOCK_RES_RECOVERING;

Could you help explaining where getting stuck?

Sure,
As dlm missed the window to purge lock resource, it can't be unhashed.

During umount:
dlm_unregister_domain
dlm_migrate_all_locks -> there is always a lock resource hashed, so can't return from dlm_migrate_all_locks() thus hang during umount.

Thanks,
Changwei

Quoted 62 lines
Quoted 61 lines

To reproduce this iusse, crash a host and then umount ocfs2 from another node.

To sovle this, just let unlock progress wait for recovery done.

---
  fs/ocfs2/dlm/dlmunlock.c | 23 +++++++++++++++++++----
  1 file changed, 19 insertions(+), 4 deletions(-)
fs/ocfs2/dlm/dlmunlock.c +19 -4
index 63d701c..c8e9b70 100644
@@ -105,7 +105,8 @@ static enum dlm_status dlmunlock_common(struct dlm_ctxt *dlm,
  	enum dlm_status status;
  	int actions = 0;
  	int in_use;
-        u8 owner;
+	u8 owner;
+	int recovery_wait = 0;
 
  	mlog(0, "master_node = %d, valblk = %d\n", master_node,
  	     flags & LKM_VALBLK);
@@ -208,9 +209,12 @@ static enum dlm_status dlmunlock_common(struct dlm_ctxt *dlm,
  		}
  		if (flags & LKM_CANCEL)
  			lock->cancel_pending = 0;
-		else
-			lock->unlock_pending = 0;
-
+		else {
+			if (!lock->unlock_pending)
+				recovery_wait = 1;
+			else
+				lock->unlock_pending = 0;
+		}
  	}
 
  	/* get an extra ref on lock.  if we are just switching
@@ -244,6 +248,17 @@ static enum dlm_status dlmunlock_common(struct dlm_ctxt *dlm,
  	spin_unlock(&res->spinlock);
  	wake_up(&res->wq);
 
+	if (recovery_wait) {
+		spin_lock(&res->spinlock);
+		/* Unlock request will directly succeed after owner dies,
+		 * and the lock is already removed from grant list. We have to
+		 * wait for RECOVERING done or we miss the chance to purge it
+		 * since the removement is much faster than RECOVERING proc.
+		 */
+		__dlm_wait_on_lockres_flags(res, DLM_LOCK_RES_RECOVERING);
+		spin_unlock(&res->spinlock);
+	}
+
  	/* let the caller's final dlm_lock_put handle the actual kfree */
  	if (actions & DLM_UNLOCK_FREE_LOCK) {
  		/* this should always be coupled with list removal */
#3 lore raw

PATCH 1/1 ocfs2: wait for recovering done after direct unlock request

Hi Changwei,

Thanks for your explaination, and I still have a question below.

Quoted 46 lines · Changwei Ge

Hi Jun,

Quoted 27 lines · piaojun

Hi Changwei,

The DLM process is a little bit complex, so I suggest pasting the code path. And I wonder if my code is right?

Thanks,
Jun

Quoted 11 lines · Changwei Ge

There is scenario causing ocfs2 umount hang when multiple hosts are rebooting at the same time.

NODE1 NODE2 NODE3
send unlock requset to NODE2
dies
become recovery master recover NODE2
find NODE2 dead
mark resource RECOVERING
directly remove lock from grant list

dlm_do_local_recovery_cleanup
dlm_move_lockres_to_recovery_list

     res->state |= DLM_LOCK_RES_RECOVERING;
     list_add_tail(&res->recovering, &dlm->reco.resources);
Quoted 2 lines

calculate usage but RECOVERING marked
**miss the window of purging

Quoted 8 lines

dlmunlock
dlmunlock_remote
dlmunlock_common // unlock successfully directly

   dlm_lockres_calc_usage
     __dlm_lockres_calc_usage
       __dlm_lockres_unused
         if (res->state & (DLM_LOCK_RES_RECOVERING| // won't purge lock as DLM_LOCK_RES_RECOVERING is set

True.

Quoted 5 lines
Quoted 2 lines

clear RECOVERING

dlm_finish_local_lockres_recovery
res->state &= ~DLM_LOCK_RES_RECOVERING;

Quoted 8 lines
Quoted 1 line

Could you help explaining where getting stuck?

Sure,
As dlm missed the window to purge lock resource, it can't be unhashed.

During umount:
dlm_unregister_domain
dlm_migrate_all_locks -> there is always a lock resource hashed, so can't return from dlm_migrate_all_locks() thus hang during umount.

In dlm_migrate_all_locks, lockres will be move to purge_list and purged again:
dlm_migrate_all_locks
__dlm_lockres_calc_usage

    list_add_tail(&res->purge, &dlm->purge_list);

Do you mean this process does not work?

Quoted 69 lines

Thanks,
Changwei

Quoted 46 lines
Quoted 46 lines

To reproduce this iusse, crash a host and then umount ocfs2 from another node.

To sovle this, just let unlock progress wait for recovery done.

---
  fs/ocfs2/dlm/dlmunlock.c | 23 +++++++++++++++++++----
  1 file changed, 19 insertions(+), 4 deletions(-)
fs/ocfs2/dlm/dlmunlock.c +8 -4
index 63d701c..c8e9b70 100644
@@ -105,7 +105,8 @@ static enum dlm_status dlmunlock_common(struct dlm_ctxt *dlm,
  	enum dlm_status status;
  	int actions = 0;
  	int in_use;
-        u8 owner;
+	u8 owner;
+	int recovery_wait = 0;
 
  	mlog(0, "master_node = %d, valblk = %d\n", master_node,
  	     flags & LKM_VALBLK);
@@ -208,9 +209,12 @@ static enum dlm_status dlmunlock_common(struct dlm_ctxt *dlm,
  		}
  		if (flags & LKM_CANCEL)
  			lock->cancel_pending = 0;
-		else
-			lock->unlock_pending = 0;
-
+		else {
+			if (!lock->unlock_pending)
+				recovery_wait = 1;
+			else
+				lock->unlock_pending = 0;
+		}
  	}
 
  	/* get an extra ref on lock.  if we are just switching
@@ -244,6 +248,17 @@ static enum dlm_status dlmunlock_common(struct dlm_ctxt *dlm,
  	spin_unlock(&res->spinlock);
  	wake_up(&res->wq);
 
Quoted 16 lines
Quoted 5 lines
fs/ocfs2/dlm/dlmunlock.c +4 -0
+	if (recovery_wait) {
+		spin_lock(&res->spinlock);
+		/* Unlock request will directly succeed after owner dies,
+		 * and the lock is already removed from grant list. We have to

fs/ocfs2/dlm/dlmunlock.c static enum dlm_status dlmunlock_common(struct dlm_ctxt *dlm, · line 264

Quoted 10 lines
fs/ocfs2/dlm/dlmunlock.c +7 -0
+		 * wait for RECOVERING done or we miss the chance to purge it
+		 * since the removement is much faster than RECOVERING proc.
+		 */
+		__dlm_wait_on_lockres_flags(res, DLM_LOCK_RES_RECOVERING);
+		spin_unlock(&res->spinlock);
+	}
+
  	/* let the caller's final dlm_lock_put handle the actual kfree */
  	if (actions & DLM_UNLOCK_FREE_LOCK) {
  		/* this should always be coupled with list removal */

.

#4 lore raw

PATCH 1/1 ocfs2: wait for recovering done after direct unlock request

Hi Jun,

Quoted 3 lines · piaojun

Hi Changwei,

Thanks for your explaination, and I still have a question below.

Thank you for looking into this.

Quoted 57 lines · Changwei Ge
Quoted 46 lines

Hi Jun,

Quoted 27 lines · piaojun

Hi Changwei,

The DLM process is a little bit complex, so I suggest pasting the code path. And I wonder if my code is right?

Thanks,
Jun

Quoted 11 lines · Changwei Ge

There is scenario causing ocfs2 umount hang when multiple hosts are rebooting at the same time.

NODE1 NODE2 NODE3
send unlock requset to NODE2
dies
become recovery master recover NODE2
find NODE2 dead
mark resource RECOVERING
directly remove lock from grant list

dlm_do_local_recovery_cleanup

    dlm_move_lockres_to_recovery_list
      res->state |= DLM_LOCK_RES_RECOVERING;
      list_add_tail(&res->recovering, &dlm->reco.resources);
Quoted 2 lines

calculate usage but RECOVERING marked
**miss the window of purging

Quoted 8 lines

dlmunlock
dlmunlock_remote
dlmunlock_common // unlock successfully directly

dlm_lockres_calc_usage
__dlm_lockres_calc_usage
__dlm_lockres_unused
if (res->state & (DLM_LOCK_RES_RECOVERING| // won't purge lock as DLM_LOCK_RES_RECOVERING is set

True.

Quoted 5 lines
Quoted 2 lines

clear RECOVERING

dlm_finish_local_lockres_recovery

    res->state &= ~DLM_LOCK_RES_RECOVERING;
Quoted 8 lines
Quoted 1 line

Could you help explaining where getting stuck?

Sure,
As dlm missed the window to purge lock resource, it can't be unhashed.

During umount:
dlm_unregister_domain

    dlm_migrate_all_locks -> there is always a lock resource hashed, so can't return from dlm_migrate_all_locks() thus hang during umount.
Quoted 6 lines

In dlm_migrate_all_locks, lockres will be move to purge_list and purged again:
dlm_migrate_all_locks
__dlm_lockres_calc_usage

     list_add_tail(&res->purge, &dlm->purge_list);

Do you mean this process does not work?

Yes, only for Migrating lock resources, it has a chance to call __dlm_lockres_calc_usage()
But in our situation, the problematic lock resource is obviously no master. So no chance for it to set stack variable *dropped* in dlm_migrate_all_locks()

Thanks,
Changwei

Quoted 71 lines
Quoted 70 lines

Thanks,
Changwei

Quoted 46 lines
Quoted 46 lines

To reproduce this iusse, crash a host and then umount ocfs2 from another node.

To sovle this, just let unlock progress wait for recovery done.

---
   fs/ocfs2/dlm/dlmunlock.c | 23 +++++++++++++++++++----
   1 file changed, 19 insertions(+), 4 deletions(-)
fs/ocfs2/dlm/dlmunlock.c +8 -4
index 63d701c..c8e9b70 100644
@@ -105,7 +105,8 @@ static enum dlm_status dlmunlock_common(struct dlm_ctxt *dlm,
   	enum dlm_status status;
   	int actions = 0;
   	int in_use;
-        u8 owner;
+	u8 owner;
+	int recovery_wait = 0;
 
   	mlog(0, "master_node = %d, valblk = %d\n", master_node,
   	     flags & LKM_VALBLK);
@@ -208,9 +209,12 @@ static enum dlm_status dlmunlock_common(struct dlm_ctxt *dlm,
   		}
   		if (flags & LKM_CANCEL)
   			lock->cancel_pending = 0;
-		else
-			lock->unlock_pending = 0;
-
+		else {
+			if (!lock->unlock_pending)
+				recovery_wait = 1;
+			else
+				lock->unlock_pending = 0;
+		}
   	}
 
   	/* get an extra ref on lock.  if we are just switching
@@ -244,6 +248,17 @@ static enum dlm_status dlmunlock_common(struct dlm_ctxt *dlm,
   	spin_unlock(&res->spinlock);
   	wake_up(&res->wq);
 
Quoted 16 lines
Quoted 5 lines
fs/ocfs2/dlm/dlmunlock.c +4 -0
+	if (recovery_wait) {
+		spin_lock(&res->spinlock);
+		/* Unlock request will directly succeed after owner dies,
+		 * and the lock is already removed from grant list. We have to

fs/ocfs2/dlm/dlmunlock.c static enum dlm_status dlmunlock_common(struct dlm_ctxt *dlm, · line 264

Quoted 10 lines
fs/ocfs2/dlm/dlmunlock.c +7 -0
+		 * wait for RECOVERING done or we miss the chance to purge it
+		 * since the removement is much faster than RECOVERING proc.
+		 */
+		__dlm_wait_on_lockres_flags(res, DLM_LOCK_RES_RECOVERING);
+		spin_unlock(&res->spinlock);
+	}
+
   	/* let the caller's final dlm_lock_put handle the actual kfree */
   	if (actions & DLM_UNLOCK_FREE_LOCK) {
   		/* this should always be coupled with list removal */

.

#5 lore raw

PATCH 1/1 ocfs2: wait for recovering done after direct unlock request

Hi Jun,

Ping...

Do you have any further question?
If any question uncleared, please let me know.
Otherwise, could you please give a feedback?

Thanks,
Changwei

Quoted 156 lines · Changwei Ge

Hi Jun,

Quoted 3 lines · piaojun

Hi Changwei,

Thanks for your explaination, and I still have a question below.

Thank you for looking into this.

Quoted 57 lines · Changwei Ge
Quoted 46 lines

Hi Jun,

Quoted 27 lines · piaojun

Hi Changwei,

The DLM process is a little bit complex, so I suggest pasting the code path. And I wonder if my code is right?

Thanks,
Jun

Quoted 11 lines · Changwei Ge

There is scenario causing ocfs2 umount hang when multiple hosts are rebooting at the same time.

NODE1 NODE2 NODE3
send unlock requset to NODE2
dies
become recovery master recover NODE2
find NODE2 dead
mark resource RECOVERING
directly remove lock from grant list

dlm_do_local_recovery_cleanup

     dlm_move_lockres_to_recovery_list
       res->state |= DLM_LOCK_RES_RECOVERING;
       list_add_tail(&res->recovering, &dlm->reco.resources);
Quoted 2 lines

calculate usage but RECOVERING marked
**miss the window of purging

Quoted 8 lines

dlmunlock
dlmunlock_remote
dlmunlock_common // unlock successfully directly

dlm_lockres_calc_usage
__dlm_lockres_calc_usage
__dlm_lockres_unused
if (res->state & (DLM_LOCK_RES_RECOVERING| // won't purge lock as DLM_LOCK_RES_RECOVERING is set

True.

Quoted 5 lines
Quoted 2 lines

clear RECOVERING

dlm_finish_local_lockres_recovery

     res->state &= ~DLM_LOCK_RES_RECOVERING;
Quoted 8 lines
Quoted 1 line

Could you help explaining where getting stuck?

Sure,
As dlm missed the window to purge lock resource, it can't be unhashed.

During umount:
dlm_unregister_domain

     dlm_migrate_all_locks -> there is always a lock resource hashed, so can't return from dlm_migrate_all_locks() thus hang during umount.
Quoted 6 lines

In dlm_migrate_all_locks, lockres will be move to purge_list and purged again:
dlm_migrate_all_locks

    __dlm_lockres_calc_usage
      list_add_tail(&res->purge, &dlm->purge_list);

Do you mean this process does not work?

Yes, only for Migrating lock resources, it has a chance to call __dlm_lockres_calc_usage()
But in our situation, the problematic lock resource is obviously no master. So no chance for it to set stack variable *dropped* in dlm_migrate_all_locks()

Thanks,
Changwei

Quoted 68 lines
Quoted 68 lines

Thanks,
Changwei

Quoted 62 lines
Quoted 61 lines

To reproduce this iusse, crash a host and then umount ocfs2 from another node.

To sovle this, just let unlock progress wait for recovery done.

---
    fs/ocfs2/dlm/dlmunlock.c | 23 +++++++++++++++++++----
    1 file changed, 19 insertions(+), 4 deletions(-)
fs/ocfs2/dlm/dlmunlock.c +19 -4
index 63d701c..c8e9b70 100644
@@ -105,7 +105,8 @@ static enum dlm_status dlmunlock_common(struct dlm_ctxt *dlm,
    	enum dlm_status status;
    	int actions = 0;
    	int in_use;
-        u8 owner;
+	u8 owner;
+	int recovery_wait = 0;
 
    	mlog(0, "master_node = %d, valblk = %d\n", master_node,
    	     flags & LKM_VALBLK);
@@ -208,9 +209,12 @@ static enum dlm_status dlmunlock_common(struct dlm_ctxt *dlm,
    		}
    		if (flags & LKM_CANCEL)
    			lock->cancel_pending = 0;
-		else
-			lock->unlock_pending = 0;
-
+		else {
+			if (!lock->unlock_pending)
+				recovery_wait = 1;
+			else
+				lock->unlock_pending = 0;
+		}
    	}
 
    	/* get an extra ref on lock.  if we are just switching
@@ -244,6 +248,17 @@ static enum dlm_status dlmunlock_common(struct dlm_ctxt *dlm,
    	spin_unlock(&res->spinlock);
    	wake_up(&res->wq);
 
+	if (recovery_wait) {
+		spin_lock(&res->spinlock);
+		/* Unlock request will directly succeed after owner dies,
+		 * and the lock is already removed from grant list. We have to
+		 * wait for RECOVERING done or we miss the chance to purge it
+		 * since the removement is much faster than RECOVERING proc.
+		 */
+		__dlm_wait_on_lockres_flags(res, DLM_LOCK_RES_RECOVERING);
+		spin_unlock(&res->spinlock);
+	}
+
    	/* let the caller's final dlm_lock_put handle the actual kfree */
    	if (actions & DLM_UNLOCK_FREE_LOCK) {
    		/* this should always be coupled with list removal */
Quoted 3 lines
Quoted 2 lines

.

_______________________________________________
Ocfs2-devel mailing list
Ocfs2-devel at oss.oracle.com
https://oss.oracle.com/mailman/listinfo/ocfs2-devel

#6 lore raw

PATCH 1/1 ocfs2: wait for recovering done after direct unlock request

Hi Changwei,

Thanks for your patient explaination, and I get your point. But I have some suggestion about your patch below.

Quoted 121 lines · Changwei Ge

Hi Jun,

Ping...

Do you have any further question?
If any question uncleared, please let me know.
Otherwise, could you please give a feedback?

Thanks,
Changwei

Quoted 109 lines · Changwei Ge

Hi Jun,

Quoted 3 lines · piaojun

Hi Changwei,

Thanks for your explaination, and I still have a question below.

Thank you for looking into this.

Quoted 57 lines · Changwei Ge
Quoted 46 lines

Hi Jun,

Quoted 27 lines · piaojun

Hi Changwei,

The DLM process is a little bit complex, so I suggest pasting the code path. And I wonder if my code is right?

Thanks,
Jun

Quoted 11 lines · Changwei Ge

There is scenario causing ocfs2 umount hang when multiple hosts are rebooting at the same time.

NODE1 NODE2 NODE3
send unlock requset to NODE2
dies
become recovery master recover NODE2
find NODE2 dead
mark resource RECOVERING
directly remove lock from grant list

dlm_do_local_recovery_cleanup

     dlm_move_lockres_to_recovery_list
       res->state |= DLM_LOCK_RES_RECOVERING;
       list_add_tail(&res->recovering, &dlm->reco.resources);
Quoted 2 lines

calculate usage but RECOVERING marked
**miss the window of purging

Quoted 8 lines

dlmunlock
dlmunlock_remote
dlmunlock_common // unlock successfully directly

dlm_lockres_calc_usage
__dlm_lockres_calc_usage
__dlm_lockres_unused
if (res->state & (DLM_LOCK_RES_RECOVERING| // won't purge lock as DLM_LOCK_RES_RECOVERING is set

True.

Quoted 5 lines
Quoted 2 lines

clear RECOVERING

dlm_finish_local_lockres_recovery

     res->state &= ~DLM_LOCK_RES_RECOVERING;
Quoted 8 lines
Quoted 1 line

Could you help explaining where getting stuck?

Sure,
As dlm missed the window to purge lock resource, it can't be unhashed.

During umount:
dlm_unregister_domain

     dlm_migrate_all_locks -> there is always a lock resource hashed, so can't return from dlm_migrate_all_locks() thus hang during umount.
Quoted 6 lines

In dlm_migrate_all_locks, lockres will be move to purge_list and purged again:
dlm_migrate_all_locks

    __dlm_lockres_calc_usage
      list_add_tail(&res->purge, &dlm->purge_list);

Do you mean this process does not work?

Yes, only for Migrating lock resources, it has a chance to call __dlm_lockres_calc_usage()
But in our situation, the problematic lock resource is obviously no master. So no chance for it to set stack variable *dropped* in dlm_migrate_all_locks()

Thanks,
Changwei

Quoted 30 lines
Quoted 30 lines

Thanks,
Changwei

Quoted 24 lines
Quoted 24 lines

To reproduce this iusse, crash a host and then umount ocfs2 from another node.

To sovle this, just let unlock progress wait for recovery done.

---
    fs/ocfs2/dlm/dlmunlock.c | 23 +++++++++++++++++++----
    1 file changed, 19 insertions(+), 4 deletions(-)
fs/ocfs2/dlm/dlmunlock.c +2 -1
index 63d701c..c8e9b70 100644
@@ -105,7 +105,8 @@ static enum dlm_status dlmunlock_common(struct dlm_ctxt *dlm,
    	enum dlm_status status;
    	int actions = 0;
    	int in_use;
-        u8 owner;
+	u8 owner;
+	int recovery_wait = 0;
 
Quoted 12 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
fs/ocfs2/dlm/dlmunlock.c +0 -0
    	mlog(0, "master_node = %d, valblk = %d\n", master_node,
    	     flags & LKM_VALBLK);
@@ -208,9 +209,12 @@ static enum dlm_status dlmunlock_common(struct dlm_ctxt *dlm,
    		}
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines

fs/ocfs2/dlm/dlmunlock.c static enum dlm_status dlmunlock_common(struct dlm_ctxt *dlm, · line 214

Quoted 8 lines
fs/ocfs2/dlm/dlmunlock.c +3 -3
    		if (flags & LKM_CANCEL)
    			lock->cancel_pending = 0;
-		else
-			lock->unlock_pending = 0;
-
+		else {
+			if (!lock->unlock_pending)
+				recovery_wait = 1;

Could we just check if res->state is in DLM_LOCK_RES_RECOVERING or status is DLM_RECOVERING? And there is no need to define an extra variable.

Quoted 37 lines
Quoted 26 lines
Quoted 26 lines
Quoted 26 lines
Quoted 26 lines
Quoted 25 lines
fs/ocfs2/dlm/dlmunlock.c +14 -0
+			else
+				lock->unlock_pending = 0;
+		}
    	}
 
    	/* get an extra ref on lock.  if we are just switching
@@ -244,6 +248,17 @@ static enum dlm_status dlmunlock_common(struct dlm_ctxt *dlm,
    	spin_unlock(&res->spinlock);
    	wake_up(&res->wq);
 
+	if (recovery_wait) {
+		spin_lock(&res->spinlock);
+		/* Unlock request will directly succeed after owner dies,
+		 * and the lock is already removed from grant list. We have to
+		 * wait for RECOVERING done or we miss the chance to purge it
+		 * since the removement is much faster than RECOVERING proc.
+		 */
+		__dlm_wait_on_lockres_flags(res, DLM_LOCK_RES_RECOVERING);
+		spin_unlock(&res->spinlock);
+	}
+
    	/* let the caller's final dlm_lock_put handle the actual kfree */
    	if (actions & DLM_UNLOCK_FREE_LOCK) {
    		/* this should always be coupled with list removal */
Quoted 9 lines
Quoted 3 lines
Quoted 2 lines

.

_______________________________________________
Ocfs2-devel mailing list
Ocfs2-devel at oss.oracle.com
https://oss.oracle.com/mailman/listinfo/ocfs2-devel

.

#7 lore raw

PATCH 1/1 ocfs2: wait for recovering done after direct unlock request

Hi Jun,

Quoted 4 lines · piaojun

Hi Changwei,

Thanks for your patient explaination, and I get your point. But I have some suggestion about your patch below.

Thanks your advise to this patch.

Quoted 123 lines · Changwei Ge
Quoted 121 lines

Hi Jun,

Ping...

Do you have any further question?
If any question uncleared, please let me know.
Otherwise, could you please give a feedback?

Thanks,
Changwei

Quoted 109 lines · Changwei Ge

Hi Jun,

Quoted 3 lines · piaojun

Hi Changwei,

Thanks for your explaination, and I still have a question below.

Thank you for looking into this.

Quoted 57 lines · Changwei Ge
Quoted 46 lines

Hi Jun,

Quoted 27 lines · piaojun

Hi Changwei,

The DLM process is a little bit complex, so I suggest pasting the code path. And I wonder if my code is right?

Thanks,
Jun

Quoted 11 lines · Changwei Ge

There is scenario causing ocfs2 umount hang when multiple hosts are rebooting at the same time.

NODE1 NODE2 NODE3
send unlock requset to NODE2
dies
become recovery master recover NODE2
find NODE2 dead
mark resource RECOVERING
directly remove lock from grant list

dlm_do_local_recovery_cleanup

      dlm_move_lockres_to_recovery_list
        res->state |= DLM_LOCK_RES_RECOVERING;
        list_add_tail(&res->recovering, &dlm->reco.resources);
Quoted 2 lines

calculate usage but RECOVERING marked
**miss the window of purging

Quoted 8 lines

dlmunlock
dlmunlock_remote
dlmunlock_common // unlock successfully directly

dlm_lockres_calc_usage
__dlm_lockres_calc_usage
__dlm_lockres_unused
if (res->state & (DLM_LOCK_RES_RECOVERING| // won't purge lock as DLM_LOCK_RES_RECOVERING is set

True.

Quoted 5 lines
Quoted 2 lines

clear RECOVERING

dlm_finish_local_lockres_recovery

      res->state &= ~DLM_LOCK_RES_RECOVERING;
Quoted 8 lines
Quoted 1 line

Could you help explaining where getting stuck?

Sure,
As dlm missed the window to purge lock resource, it can't be unhashed.

During umount:
dlm_unregister_domain

      dlm_migrate_all_locks -> there is always a lock resource hashed, so can't return from dlm_migrate_all_locks() thus hang during umount.
Quoted 6 lines

In dlm_migrate_all_locks, lockres will be move to purge_list and purged again:
dlm_migrate_all_locks

     __dlm_lockres_calc_usage
       list_add_tail(&res->purge, &dlm->purge_list);

Do you mean this process does not work?

Yes, only for Migrating lock resources, it has a chance to call __dlm_lockres_calc_usage()
But in our situation, the problematic lock resource is obviously no master. So no chance for it to set stack variable *dropped* in dlm_migrate_all_locks()

Thanks,
Changwei

Quoted 30 lines
Quoted 30 lines

Thanks,
Changwei

Quoted 24 lines
Quoted 24 lines

To reproduce this iusse, crash a host and then umount ocfs2 from another node.

To sovle this, just let unlock progress wait for recovery done.

---
     fs/ocfs2/dlm/dlmunlock.c | 23 +++++++++++++++++++----
     1 file changed, 19 insertions(+), 4 deletions(-)
fs/ocfs2/dlm/dlmunlock.c +2 -1
index 63d701c..c8e9b70 100644
@@ -105,7 +105,8 @@ static enum dlm_status dlmunlock_common(struct dlm_ctxt *dlm,
     	enum dlm_status status;
     	int actions = 0;
     	int in_use;
-        u8 owner;
+	u8 owner;
+	int recovery_wait = 0;
 
Quoted 16 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
fs/ocfs2/dlm/dlmunlock.c +0 -0
     	mlog(0, "master_node = %d, valblk = %d\n", master_node,
     	     flags & LKM_VALBLK);
@@ -208,9 +209,12 @@ static enum dlm_status dlmunlock_common(struct dlm_ctxt *dlm,
     		}
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines

fs/ocfs2/dlm/dlmunlock.c static enum dlm_status dlmunlock_common(struct dlm_ctxt *dlm, · line 214

Quoted 8 lines
fs/ocfs2/dlm/dlmunlock.c +3 -3
     		if (flags & LKM_CANCEL)
     			lock->cancel_pending = 0;
-		else
-			lock->unlock_pending = 0;
-
+		else {
+			if (!lock->unlock_pending)
+				recovery_wait = 1;

Could we just check if res->state is in DLM_LOCK_RES_RECOVERING or status is DLM_RECOVERING? And there is no need to define an extra variable.

As the lock resource master had died, DLM_RECOVERING can't be returned.

I prefer adding a stack variable like *recovery_wait* to tell if involved lock resource has a chance to be recovered. In this way, we won't make code subtle comparing with normal code path. As you know, the case we discuss here is not that possible to happen when each node in cluster works well. And we can also save some CPU cycles this way.

Thanks,
Changwei

Quoted 39 lines
Quoted 38 lines
Quoted 27 lines
Quoted 27 lines
Quoted 27 lines
Quoted 27 lines
Quoted 26 lines
fs/ocfs2/dlm/dlmunlock.c +14 -0
+			else
+				lock->unlock_pending = 0;
+		}
     	}
 
     	/* get an extra ref on lock.  if we are just switching
@@ -244,6 +248,17 @@ static enum dlm_status dlmunlock_common(struct dlm_ctxt *dlm,
     	spin_unlock(&res->spinlock);
     	wake_up(&res->wq);
 
+	if (recovery_wait) {
+		spin_lock(&res->spinlock);
+		/* Unlock request will directly succeed after owner dies,
+		 * and the lock is already removed from grant list. We have to
+		 * wait for RECOVERING done or we miss the chance to purge it
+		 * since the removement is much faster than RECOVERING proc.
+		 */
+		__dlm_wait_on_lockres_flags(res, DLM_LOCK_RES_RECOVERING);
+		spin_unlock(&res->spinlock);
+	}
+
     	/* let the caller's final dlm_lock_put handle the actual kfree */
     	if (actions & DLM_UNLOCK_FREE_LOCK) {
     		/* this should always be coupled with list removal */
Quoted 9 lines
Quoted 3 lines
Quoted 2 lines

.

_______________________________________________
Ocfs2-devel mailing list
Ocfs2-devel at oss.oracle.com
https://oss.oracle.com/mailman/listinfo/ocfs2-devel

.

#8 lore raw

PATCH 1/1 ocfs2: wait for recovering done after direct unlock request

Hi Changwei,

Quoted 150 lines · Changwei Ge

Hi Jun,

Quoted 4 lines · piaojun

Hi Changwei,

Thanks for your patient explaination, and I get your point. But I have some suggestion about your patch below.

Thanks your advise to this patch.

Quoted 123 lines · Changwei Ge
Quoted 121 lines

Hi Jun,

Ping...

Do you have any further question?
If any question uncleared, please let me know.
Otherwise, could you please give a feedback?

Thanks,
Changwei

Quoted 109 lines · Changwei Ge

Hi Jun,

Quoted 3 lines · piaojun

Hi Changwei,

Thanks for your explaination, and I still have a question below.

Thank you for looking into this.

Quoted 57 lines · Changwei Ge
Quoted 46 lines

Hi Jun,

Quoted 27 lines · piaojun

Hi Changwei,

The DLM process is a little bit complex, so I suggest pasting the code path. And I wonder if my code is right?

Thanks,
Jun

Quoted 11 lines · Changwei Ge

There is scenario causing ocfs2 umount hang when multiple hosts are rebooting at the same time.

NODE1 NODE2 NODE3
send unlock requset to NODE2
dies
become recovery master recover NODE2
find NODE2 dead
mark resource RECOVERING
directly remove lock from grant list

dlm_do_local_recovery_cleanup

      dlm_move_lockres_to_recovery_list
        res->state |= DLM_LOCK_RES_RECOVERING;
        list_add_tail(&res->recovering, &dlm->reco.resources);
Quoted 2 lines

calculate usage but RECOVERING marked
**miss the window of purging

Quoted 8 lines

dlmunlock
dlmunlock_remote
dlmunlock_common // unlock successfully directly

dlm_lockres_calc_usage
__dlm_lockres_calc_usage
__dlm_lockres_unused
if (res->state & (DLM_LOCK_RES_RECOVERING| // won't purge lock as DLM_LOCK_RES_RECOVERING is set

True.

Quoted 5 lines
Quoted 2 lines

clear RECOVERING

dlm_finish_local_lockres_recovery

      res->state &= ~DLM_LOCK_RES_RECOVERING;
Quoted 8 lines
Quoted 1 line

Could you help explaining where getting stuck?

Sure,
As dlm missed the window to purge lock resource, it can't be unhashed.

During umount:
dlm_unregister_domain

      dlm_migrate_all_locks -> there is always a lock resource hashed, so can't return from dlm_migrate_all_locks() thus hang during umount.
Quoted 6 lines

In dlm_migrate_all_locks, lockres will be move to purge_list and purged again:
dlm_migrate_all_locks

     __dlm_lockres_calc_usage
       list_add_tail(&res->purge, &dlm->purge_list);

Do you mean this process does not work?

Yes, only for Migrating lock resources, it has a chance to call __dlm_lockres_calc_usage()
But in our situation, the problematic lock resource is obviously no master. So no chance for it to set stack variable *dropped* in dlm_migrate_all_locks()

Thanks,
Changwei

Quoted 30 lines
Quoted 30 lines

Thanks,
Changwei

Quoted 24 lines
Quoted 24 lines

To reproduce this iusse, crash a host and then umount ocfs2 from another node.

To sovle this, just let unlock progress wait for recovery done.

---
     fs/ocfs2/dlm/dlmunlock.c | 23 +++++++++++++++++++----
     1 file changed, 19 insertions(+), 4 deletions(-)
fs/ocfs2/dlm/dlmunlock.c +2 -1
index 63d701c..c8e9b70 100644
@@ -105,7 +105,8 @@ static enum dlm_status dlmunlock_common(struct dlm_ctxt *dlm,
     	enum dlm_status status;
     	int actions = 0;
     	int in_use;
-        u8 owner;
+	u8 owner;
+	int recovery_wait = 0;
 
Quoted 16 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
fs/ocfs2/dlm/dlmunlock.c +0 -0
     	mlog(0, "master_node = %d, valblk = %d\n", master_node,
     	     flags & LKM_VALBLK);
@@ -208,9 +209,12 @@ static enum dlm_status dlmunlock_common(struct dlm_ctxt *dlm,
     		}
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines

fs/ocfs2/dlm/dlmunlock.c static enum dlm_status dlmunlock_common(struct dlm_ctxt *dlm, · line 214

Quoted 8 lines
fs/ocfs2/dlm/dlmunlock.c +3 -3
     		if (flags & LKM_CANCEL)
     			lock->cancel_pending = 0;
-		else
-			lock->unlock_pending = 0;
-
+		else {
+			if (!lock->unlock_pending)
+				recovery_wait = 1;

Could we just check if res->state is in DLM_LOCK_RES_RECOVERING or status is DLM_RECOVERING? And there is no need to define an extra variable.

Quoted 5 lines

As the lock resource master had died, DLM_RECOVERING can't be returned.

I prefer adding a stack variable like *recovery_wait* to tell if involved lock resource has a chance to be recovered. In this way, we won't make code subtle comparing with normal code path. As you know, the case we discuss here is not that possible to happen when each node in cluster works well. And we can also save some CPU cycles this way.

I mean we could check if the res->state is in DLM_LOCK_RES_RECOVERING. As when lock->unlock_pending is cleared in dlm_move_lockres_to_recovery_list, res->state must be set DLM_LOCK_RES_RECOVERING.

And I think of another solution which seems a little easier and save some CPU work. We could call __dlm_lockres_calc_usage to put unused lockres into purge list in dlm_finish_local_lockres_recovery. And this will not stuck the unlocking process.

Quoted 45 lines

Thanks,
Changwei

Quoted 31 lines
Quoted 31 lines
Quoted 31 lines
Quoted 30 lines
Quoted 29 lines
Quoted 11 lines
Quoted 11 lines
fs/ocfs2/dlm/dlmunlock.c +3 -0
+			else
+				lock->unlock_pending = 0;
+		}
     	}
 
     	/* get an extra ref on lock.  if we are just switching
@@ -244,6 +248,17 @@ static enum dlm_status dlmunlock_common(struct dlm_ctxt *dlm,
     	spin_unlock(&res->spinlock);
     	wake_up(&res->wq);
 
Quoted 16 lines
Quoted 5 lines
fs/ocfs2/dlm/dlmunlock.c +4 -0
+	if (recovery_wait) {
+		spin_lock(&res->spinlock);
+		/* Unlock request will directly succeed after owner dies,
+		 * and the lock is already removed from grant list. We have to

fs/ocfs2/dlm/dlmunlock.c static enum dlm_status dlmunlock_common(struct dlm_ctxt *dlm, · line 264

Quoted 10 lines
fs/ocfs2/dlm/dlmunlock.c +7 -0
+		 * wait for RECOVERING done or we miss the chance to purge it
+		 * since the removement is much faster than RECOVERING proc.
+		 */
+		__dlm_wait_on_lockres_flags(res, DLM_LOCK_RES_RECOVERING);
+		spin_unlock(&res->spinlock);
+	}
+
     	/* let the caller's final dlm_lock_put handle the actual kfree */
     	if (actions & DLM_UNLOCK_FREE_LOCK) {
     		/* this should always be coupled with list removal */

.

Quoted 8 lines
Quoted 7 lines
Quoted 5 lines

_______________________________________________
Ocfs2-devel mailing list
Ocfs2-devel at oss.oracle.com
https://oss.oracle.com/mailman/listinfo/ocfs2-devel

.

.

#9 lore raw

PATCH 1/1 ocfs2: wait for recovering done after direct unlock request

Hi Jun,

On 2019/2/19 16:36, piaojun wrote:

		recovery_wait = 1;
Quoted 16 lines
Quoted 10 lines
Quoted 4 lines

Could we just check if res->state is in DLM_LOCK_RES_RECOVERING or status is DLM_RECOVERING? And there is no need to define an extra variable.

As the lock resource master had died, DLM_RECOVERING can't be returned.

I prefer adding a stack variable like *recovery_wait* to tell if involved lock resource has a chance to be recovered. In this way, we won't make code subtle comparing with normal code path. As you know, the case we discuss here is not that possible to happen when each node in cluster works well. And we can also save some CPU cycles this way.

I mean we could check if the res->state is in DLM_LOCK_RES_RECOVERING. As when lock->unlock_pending is cleared in dlm_move_lockres_to_recovery_list, res->state must be set DLM_LOCK_RES_RECOVERING.

Quoted 4 lines

And I think of another solution which seems a little easier and save some CPU work. We could call __dlm_lockres_calc_usage to put unused lockres into purge list in dlm_finish_local_lockres_recovery. And this will not stuck the unlocking process.

Sounds a good idea, I will make some test and if feasible, I will send V2

Thanks,
Changwei

Quoted 47 lines
Quoted 46 lines

Thanks,
Changwei

Quoted 31 lines
Quoted 31 lines
Quoted 31 lines
Quoted 30 lines
Quoted 29 lines
Quoted 11 lines
Quoted 11 lines
(quoted diff) +3 -0
+			else
+				lock->unlock_pending = 0;
+		}
      	}
 
      	/* get an extra ref on lock.  if we are just switching
@@ -244,6 +248,17 @@ static enum dlm_status dlmunlock_common(struct dlm_ctxt *dlm,
      	spin_unlock(&res->spinlock);
      	wake_up(&res->wq);
 
Quoted 16 lines
Quoted 5 lines
(quoted diff) +4 -0
+	if (recovery_wait) {
+		spin_lock(&res->spinlock);
+		/* Unlock request will directly succeed after owner dies,
+		 * and the lock is already removed from grant list. We have to

Review static enum dlm_status dlmunlock_common(struct dlm_ctxt *dlm, · line 264

Quoted 10 lines
(quoted diff) +7 -0
+		 * wait for RECOVERING done or we miss the chance to purge it
+		 * since the removement is much faster than RECOVERING proc.
+		 */
+		__dlm_wait_on_lockres_flags(res, DLM_LOCK_RES_RECOVERING);
+		spin_unlock(&res->spinlock);
+	}
+
      	/* let the caller's final dlm_lock_put handle the actual kfree */
      	if (actions & DLM_UNLOCK_FREE_LOCK) {
      		/* this should always be coupled with list removal */

.

Quoted 8 lines
Quoted 7 lines
Quoted 5 lines

_______________________________________________
Ocfs2-devel mailing list
Ocfs2-devel at oss.oracle.com
https://oss.oracle.com/mailman/listinfo/ocfs2-devel

.

.

#10 lore raw

PATCH 1/1 ocfs2: wait for recovering done after direct unlock request

In reply to Changwei Ge
Quoted 15 lines · Changwei Ge

There is scenario causing ocfs2 umount hang when multiple hosts are rebooting at the same time.

NODE1 NODE2 NODE3
send unlock requset to NODE2
dies
become recovery master recover NODE2
find NODE2 dead
mark resource RECOVERING
directly remove lock from grant list
calculate usage but RECOVERING marked
**miss the window of purging
clear RECOVERING

Quoted 6 lines

To reproduce this iusse, crash a host and then umount ocfs2 from another node.

To sovle this, just let unlock progress wait for recovery done.

Quoted 53 lines
---
 fs/ocfs2/dlm/dlmunlock.c | 23 +++++++++++++++++++----
 1 file changed, 19 insertions(+), 4 deletions(-)
fs/ocfs2/dlm/dlmunlock.c +19 -4
index 63d701c..c8e9b70 100644
@@ -105,7 +105,8 @@ static enum dlm_status dlmunlock_common(struct dlm_ctxt *dlm,
 	enum dlm_status status;
 	int actions = 0;
 	int in_use;
-        u8 owner;
+	u8 owner;
+	int recovery_wait = 0;
 
 	mlog(0, "master_node = %d, valblk = %d\n", master_node,
 	     flags & LKM_VALBLK);
@@ -208,9 +209,12 @@ static enum dlm_status dlmunlock_common(struct dlm_ctxt *dlm,
 		}
 		if (flags & LKM_CANCEL)
 			lock->cancel_pending = 0;
-		else
-			lock->unlock_pending = 0;
-
+		else {
+			if (!lock->unlock_pending)
+				recovery_wait = 1;
+			else
+				lock->unlock_pending = 0;
+		}
 	}
 
 	/* get an extra ref on lock.  if we are just switching
@@ -244,6 +248,17 @@ static enum dlm_status dlmunlock_common(struct dlm_ctxt *dlm,
 	spin_unlock(&res->spinlock);
 	wake_up(&res->wq);
 
+	if (recovery_wait) {
+		spin_lock(&res->spinlock);
+		/* Unlock request will directly succeed after owner dies,
+		 * and the lock is already removed from grant list. We have to
+		 * wait for RECOVERING done or we miss the chance to purge it
+		 * since the removement is much faster than RECOVERING proc.
+		 */
+		__dlm_wait_on_lockres_flags(res, DLM_LOCK_RES_RECOVERING);
+		spin_unlock(&res->spinlock);
+	}
+
 	/* let the caller's final dlm_lock_put handle the actual kfree */
 	if (actions & DLM_UNLOCK_FREE_LOCK) {
 		/* this should always be coupled with list removal */