ocfs2/dlm: don't handle migrate lockres if already in shutdown
PATCH v2 1/1 ocfs2/dlm: don't handle migrate lockres if already in shutdown
We should not handle migrate lockres if we are already in 'DLM_CTXT_IN_SHUTDOWN', as that will cause lockres remains after leaving dlm domain. At last other nodes will get stuck into infinite loop when requsting lock from us.
The problem is caused by concurrency umount between nodes. Before receiveing N1's DLM_BEGIN_EXIT_DOMAIN_MSG, N2 has picked up N1 as the migrate target. So N2 will continue sending lockres to N1 even though N1 has left domain.
N1 N2 (owner)
touch file
access the file,
and get pr lock
begin leave domain and pick up N1 as new owner
begin leave domain and
migrate all lockres done
begin migrate lockres to N1
end leave domain, but
the lockres left
unexpectedly, because
migrate task has passed
--- fs/ocfs2/dlm/dlmdomain.c | 14 ++++++++++++++ fs/ocfs2/dlm/dlmdomain.h | 1 + fs/ocfs2/dlm/dlmrecovery.c | 9 +++++++++ 3 files changed, 24 insertions(+)
fs/ocfs2/dlm/dlmdomain.c +14 -0
@@ -675,6 +675,20 @@ static void dlm_leave_domain(struct dlm_ctxt *dlm)spin_unlock (&dlm->spinlock); } +int dlm_joined (struct dlm_ctxt *dlm) +{ +int ret =0 ; + +spin_lock (&dlm_domain_lock); + +if (dlm->dlm_state ==DLM_CTXT_JOINED ) + ret =1 ; + +spin_unlock (&dlm_domain_lock); + +return ret; +} +int dlm_shutting_down (struct dlm_ctxt *dlm) {int ret =0 ;
fs/ocfs2/dlm/dlmdomain.h +1 -0
@@ -28,6 +28,7 @@extern spinlock_t dlm_domain_lock;extern struct list_head dlm_domains; +int dlm_joined (struct dlm_ctxt *dlm);int dlm_shutting_down (struct dlm_ctxt *dlm);void dlm_fire_domain_eviction_callbacks (struct dlm_ctxt *dlm,int node_num);
fs/ocfs2/dlm/dlmrecovery.c +9 -0
@@ -1378,6 +1378,15 @@ int dlm_mig_lockres_handler(struct o2net_msg *msg, u32 len, void *data,if (!dlm_grab (dlm))return -EINVAL ; +if (!dlm_joined (dlm)) { +mlog (ML_ERROR ,"Domain %s not joined! " +"lockres %.*s, master %u\n" , + dlm->name, mres->lockname_len, + mres->lockname, mres->master); +dlm_put (dlm); +return -EINVAL ; + } +BUG_ON (!(mres->flags & (DLM_MRES_RECOVERY |DLM_MRES_MIGRATION ))); real_master = mres->master;
PATCH v2 1/1 ocfs2/dlm: don't handle migrate lockres if already in shutdown
Quoted 41 lines · piaojun
We should not handle migrate lockres if we are already in '
DLM_CTXT_IN_SHUTDOWN', as that will cause lockres remains after leaving dlm domain. At last other nodes will get stuck into infinite loop when requsting lock from us.The problem is caused by concurrency umount between nodes. Before receiveing N1's
DLM_BEGIN_EXIT_DOMAIN_MSG, N2 has picked up N1 as the migrate target. So N2 will continue sending lockres to N1 even though N1 has left domain.N1 N2 (owner)
touch fileaccess the file,
and get pr lockbegin leave domain and pick up N1 as new owner
begin leave domain and
migrate all lockres donebegin migrate lockres to N1
end leave domain, but
the lockres left
unexpectedly, because
migrate task has passedSigned-off-by Jun Piao <[email protected]>Reviewed-by Yiwen Jiang <[email protected]> ---fs/ocfs2/dlm/dlmdomain.c | 14 ++++++++++++++ fs/ocfs2/dlm/dlmdomain.h | 1 + fs/ocfs2/dlm/dlmrecovery.c | 9 +++++++++ 3 files changed, 24 insertions(+)fs/ocfs2/dlm/dlmdomain.c +0
-0
fs/ocfs2/dlm/dlmdomain.c static void dlm_leave_domain(struct dlm_ctxt *dlm) · line 678
Quoted 4 lines
fs/ocfs2/dlm/dlmdomain.c +1
-0@@ -675,6 +675,20 @@ static void dlm_leave_domain(struct dlm_ctxt *dlm)spin_unlock (&dlm->spinlock); } +int dlm_joined (struct dlm_ctxt *dlm)
This helper can be static inline and placed into header.
fs/ocfs2/dlm/dlmdomain.c static void dlm_leave_domain(struct dlm_ctxt *dlm) · line 683
Quoted 5 lines
fs/ocfs2/dlm/dlmdomain.c +5
-0+{ +int ret =0 ; + +spin_lock (&dlm_domain_lock); +
Delete blank line here.
fs/ocfs2/dlm/dlmdomain.c static void dlm_leave_domain(struct dlm_ctxt *dlm) · line 686
Quoted 3 lines
fs/ocfs2/dlm/dlmdomain.c +3
-0+if (dlm->dlm_state ==DLM_CTXT_JOINED ) + ret =1 ; +
Also here.
Except the above concern, it looks good to me.
With they are fixed, feel free to add:
Quoted 41 lines
fs/ocfs2/dlm/dlmdomain.c +5
-0+spin_unlock (&dlm_domain_lock); + +return ret; +} +int dlm_shutting_down (struct dlm_ctxt *dlm) {int ret =0 ;fs/ocfs2/dlm/dlmdomain.h +1
-0@@ -28,6 +28,7 @@extern spinlock_t dlm_domain_lock;extern struct list_head dlm_domains; +int dlm_joined (struct dlm_ctxt *dlm);int dlm_shutting_down (struct dlm_ctxt *dlm);void dlm_fire_domain_eviction_callbacks (struct dlm_ctxt *dlm,int node_num);fs/ocfs2/dlm/dlmrecovery.c +9
-0@@ -1378,6 +1378,15 @@ int dlm_mig_lockres_handler(struct o2net_msg *msg, u32 len, void *data,if (!dlm_grab (dlm))return -EINVAL ; +if (!dlm_joined (dlm)) { +mlog (ML_ERROR ,"Domain %s not joined! " +"lockres %.*s, master %u\n" , + dlm->name, mres->lockname_len, + mres->lockname, mres->master); +dlm_put (dlm); +return -EINVAL ; + } +BUG_ON (!(mres->flags & (DLM_MRES_RECOVERY |DLM_MRES_MIGRATION ))); real_master = mres->master;
PATCH v2 1/1 ocfs2/dlm: don't handle migrate lockres if already in shutdown
Hi Joseph,
Quoted 47 lines · Joseph Qi
Quoted 44 lines · piaojun
We should not handle migrate lockres if we are already in '
DLM_CTXT_IN_SHUTDOWN', as that will cause lockres remains after leaving dlm domain. At last other nodes will get stuck into infinite loop when requsting lock from us.The problem is caused by concurrency umount between nodes. Before receiveing N1's
DLM_BEGIN_EXIT_DOMAIN_MSG, N2 has picked up N1 as the migrate target. So N2 will continue sending lockres to N1 even though N1 has left domain.N1 N2 (owner)
touch fileaccess the file,
and get pr lockbegin leave domain and pick up N1 as new owner
begin leave domain and
migrate all lockres donebegin migrate lockres to N1
end leave domain, but
the lockres left
unexpectedly, because
migrate task has passedSigned-off-by Jun Piao <[email protected]>Reviewed-by Yiwen Jiang <[email protected]> ---fs/ocfs2/dlm/dlmdomain.c | 14 ++++++++++++++ fs/ocfs2/dlm/dlmdomain.h | 1 + fs/ocfs2/dlm/dlmrecovery.c | 9 +++++++++ 3 files changed, 24 insertions(+)fs/ocfs2/dlm/dlmdomain.c +0
-0@@ -675,6 +675,20 @@ static void dlm_leave_domain(struct dlm_ctxt *dlm)spin_unlock (&dlm->spinlock); }
Quoted 2 lines
fs/ocfs2/dlm/dlmdomain.c static void dlm_leave_domain(struct dlm_ctxt *dlm) · line 678
Quoted 1 line
fs/ocfs2/dlm/dlmdomain.c +1
-0+int dlm_joined (struct dlm_ctxt *dlm)This helper can be static inline and placed into header.
Agree, and I decide move dlm_shutting_down() into dlmdomain.h together.
thansk,
Jun
Quoted 61 lines
fs/ocfs2/dlm/dlmdomain.c static void dlm_leave_domain(struct dlm_ctxt *dlm) · line 683
Quoted 6 lines
fs/ocfs2/dlm/dlmdomain.c +5
-0+{ +int ret =0 ; + +spin_lock (&dlm_domain_lock); +Delete blank line here.
fs/ocfs2/dlm/dlmdomain.c static void dlm_leave_domain(struct dlm_ctxt *dlm) · line 686
Quoted 3 lines
fs/ocfs2/dlm/dlmdomain.c +3
-0+if (dlm->dlm_state ==DLM_CTXT_JOINED ) + ret =1 ; +Also here.
Except the above concern, it looks good to me.
With they are fixed, feel free to add:Reviewed-by Joseph Qi <[email protected]>Quoted 31 lines
fs/ocfs2/dlm/dlmdomain.c +5
-0+spin_unlock (&dlm_domain_lock); + +return ret; +} +int dlm_shutting_down (struct dlm_ctxt *dlm) {int ret =0 ;fs/ocfs2/dlm/dlmdomain.h +1
-0@@ -28,6 +28,7 @@extern spinlock_t dlm_domain_lock;extern struct list_head dlm_domains; +int dlm_joined (struct dlm_ctxt *dlm);int dlm_shutting_down (struct dlm_ctxt *dlm);void dlm_fire_domain_eviction_callbacks (struct dlm_ctxt *dlm,int node_num);fs/ocfs2/dlm/dlmrecovery.c +2
-0@@ -1378,6 +1378,15 @@ int dlm_mig_lockres_handler(struct o2net_msg *msg, u32 len, void *data,if (!dlm_grab (dlm))return -EINVAL ; +if (!dlm_joined (dlm)) { + mlog(ML_ERROR, "Domain %s not joined! "fs/ocfs2/dlm/dlmrecovery.c int dlm_mig_lockres_handler(struct o2net_msg *msg, u32 len, void *data, · line 1392
Quoted 10 lines
fs/ocfs2/dlm/dlmrecovery.c +7
-0+"lockres %.*s, master %u\n" , + dlm->name, mres->lockname_len, + mres->lockname, mres->master); +dlm_put (dlm); +return -EINVAL ; + } +BUG_ON (!(mres->flags & (DLM_MRES_RECOVERY |DLM_MRES_MIGRATION ))); real_master = mres->master;.