drbd_nl: dont allow detating to be inttrupted in waiting D_DETACHING to DISKLESS
PATCH 1/11 drbd_nl: dont allow detating to be inttrupted in waiting D_DETACHING to DISKLESS
From: Dongsheng Yang <[email protected]>
In our network failure and drbd down testing, we found warning in dmesg and drbd down process into D state:
"kernel: drbd /unregistered/ramtest3/0 drbd103: ASSERTION device->disk_state[NOW] == D_FAILED || device->disk_state[NOW] == D_DETACHING FAILED in go_diskless"
the problem is the wait_event is inttruptable, it could be intrupted by signal and call drbd_cleanup_device before go_diskless()
--- drbd/drbd_nl.c | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-)
drbd/drbd_nl.c +3 -6
@@ -3676,7 +3676,7 @@ static int adm_detach(struct drbd_device *device, bool force, bool intentional_d const char *tag, struct sk_buff *reply_skb) { const char *err_str = NULL; -int ret, retcode; +int retcode; device->device_conf.intentional_diskless = intentional_diskless;if (force) {
@@ -3692,19 +3692,16 @@ static int adm_detach(struct drbd_device *device, bool force, bool intentional_d CS_VERBOSE | CS_WAIT_COMPLETE | CS_SERIALIZE, tag, &err_str));/* D_DETACHING will transition to DISKLESS. */ drbd_resume_io (device); - ret =wait_event_interruptible (device->misc_wait, -get_disk_state (device) !=D_DETACHING ); +wait_event (device->misc_wait,get_disk_state (device) !=D_DETACHING );if (retcode >=SS_SUCCESS ) {/* wait for completion of drbd_ldev_destroy() */ -wait_event_interruptible (device->misc_wait, !test_bit (GOING_DISKLESS , &device->flags)); +wait_event (device->misc_wait, !test_bit (GOING_DISKLESS , &device->flags));drbd_cleanup_device (device); }else device->device_conf.intentional_diskless = false;if (retcode ==SS_IS_DISKLESS ) retcode =SS_NOTHING_TO_DO ; -if (ret) - retcode =ERR_INTR ; out:if (err_str) {drbd_msg_put_info (reply_skb, err_str);
Signature · 1 line
2.27.0
PATCH 2/11 drbd_receiver: get_ldev before use device->ldev for drbd_reconsider_queue_parameters()
From: Dongsheng Yang <[email protected]>
check ldev is not NULL before use it in drbd_reconsider_queue_parameters()
--- drbd/drbd_receiver.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-)
drbd/drbd_receiver.c +6 -1
@@ -9845,7 +9845,12 @@ static void conn_disconnect(struct drbd_connection *connection)rcu_read_unlock ();peer_device_disconnected (peer_device); -drbd_reconsider_queue_parameters (device, device->ldev); +if (get_ldev (device)) { +drbd_reconsider_queue_parameters (device, device->ldev); +put_ldev (device); + }else { +drbd_reconsider_queue_parameters (device,NULL ); + }kref_put (&device->kref, drbd_destroy_device);rcu_read_lock ();
Signature · 1 line
2.27.0
Philipp Reisner
maintainerRe: [PATCH 02/11] drbd_receiver: get_ldev before use device->ldev for drbd_reconsider_queue_parameters()
Thanks.
Quoted 31 lines · zhengbing.huang
From: Dongsheng Yang <[email protected]>
check ldev is not NULL before use it in
drbd_reconsider_queue_parameters()Signed-off-by Dongsheng Yang <[email protected]>--- drbd/drbd_receiver.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-)drbd/drbd_receiver.c +6
-2@@ -9845,7 +9845,12 @@ static void conn_disconnect(struct drbd_connection *connection)rcu_read_unlock ();peer_device_disconnected (peer_device); -drbd_reconsider_queue_parameters (device, device->ldev); +if (get_ldev (device)) { +drbd_reconsider_queue_parameters (device, device->ldev); +put_ldev (device); + }else { +drbd_reconsider_queue_parameters (device,NULL ); + }kref_put (&device->kref, drbd_destroy_device);rcu_read_lock (); --2.27.0
PATCH 3/11 drbd_transport_rdma: put kref for cm in dtr_path_established in error path
In reply to zhengbing.huangFrom: Dongsheng Yang <[email protected]>
--- drbd/drbd_transport_rdma.c | 1 + 1 file changed, 1 insertion(+)
drbd/drbd_transport_rdma.c +1 -0
@@ -922,6 +922,7 @@ static void dtr_path_established(struct dtr_cm *cm)atomic_set (&cs->active_state,PCS_INACTIVE );wake_up (&cs->wq); } +kref_put (&cm->kref, dtr_destroy_cm);return ; }
Signature · 1 line
2.27.0
Philipp Reisner
maintainerRe: [PATCH 03/11] drbd_transport_rdma: put kref for cm in dtr_path_established in error path
Hello Dongsheng,
Please add more information why you think this change fixes a bug. Have you experienced a leak of cm structs?
We got a RDMA_CM_EVENT_ESTABLISHED event. Even if DRBD does not do anything with this cm, we sill expect a RDMA_CM_EVENT_DISCONNECTED in the future. Is a problem in the handling of the disconnect?
best regards,
Philipp
Quoted 23 lines · zhengbing.huang
From: Dongsheng Yang <[email protected]>
Signed-off-by Dongsheng Yang <[email protected]>--- drbd/drbd_transport_rdma.c | 1 + 1 file changed, 1 insertion(+)drbd/drbd_transport_rdma.c +1
-1@@ -922,6 +922,7 @@ static void dtr_path_established(struct dtr_cm *cm)atomic_set (&cs->active_state,PCS_INACTIVE );wake_up (&cs->wq); } +kref_put (&cm->kref, dtr_destroy_cm);return ; } --2.27.0
Re: [PATCH 03/11] drbd_transport_rdma: put kref for cm in dtr_path_established in error path
Quoted 7 lines · Philipp Reisner
Hello Dongsheng,
Please add more information why you think this change fixes a bug. Have you experienced a leak of cm structs?
We got aRDMA_CM_EVENT_ESTABLISHEDevent. Even if DRBD does not do anything with this cm, we sill expect aRDMA_CM_EVENT_DISCONNECTEDin the future. Is a problem in the handling of the disconnect?
If dtr_path_established() go into this branch, it will not schedule_work(&cm->establish_work);
That means path->cm->state = DSM_CONNECTED; will not be done in dtr_path_established_work_fn(), so __dtr_disconnect_path() will not call rdma_disconnect(). That means this reference will never be put.
Quoted 29 lines
best regards,
PhilippQuoted 23 lines · zhengbing.huang
From: Dongsheng Yang <[email protected]>
Signed-off-by Dongsheng Yang <[email protected]>--- drbd/drbd_transport_rdma.c | 1 + 1 file changed, 1 insertion(+)drbd/drbd_transport_rdma.c +1
-1@@ -922,6 +922,7 @@ static void dtr_path_established(struct dtr_cm *cm)atomic_set (&cs->active_state,PCS_INACTIVE );wake_up (&cs->wq); } +kref_put (&cm->kref, dtr_destroy_cm);return ; } --2.27.0
Re: [PATCH 03/11] drbd_transport_rdma: put kref for cm in dtr_path_established in error path
Quoted 11 lines · Dongsheng Yang
Quoted 7 lines · Philipp Reisner
Hello Dongsheng,
Please add more information why you think this change fixes a bug. Have you experienced a leak of cm structs?
We got aRDMA_CM_EVENT_ESTABLISHEDevent. Even if DRBD does not do anything with this cm, we sill expect aRDMA_CM_EVENT_DISCONNECTEDin the future. Is a problem in the handling of the disconnect?
Quoted 3 lines
If
dtr_path_established()go into this branch, it will notschedule_work(&cm->establish_work);That means
path->cm->state=DSM_CONNECTED; will not be done indtr_path_established_work_fn(), so__dtr_disconnect_path()will not callrdma_disconnect(). That means this reference will never be put.
let me consider this example:
a) rdma_connect() called and RDMA_CM_EVENT_ESTABLISHED received.
b) network failure and dtr_path_established() go into error path.
c) establish_work will not be scheduled.
d) drbdadm down test will hang because cm ref is not put.
Quoted 29 lines
Quoted 29 lines
best regards,
PhilippQuoted 23 lines · zhengbing.huang
From: Dongsheng Yang <[email protected]>
Signed-off-by Dongsheng Yang <[email protected]>---
drbd/drbd_transport_rdma.c| 1 +1 file changed, 1 insertion(+)drbd/drbd_transport_rdma.c +1
-0@@ -922,6 +922,7 @@ static void dtr_path_established(struct dtr_cm *cm) atomic_set(&cs->active_state, PCS_INACTIVE); wake_up(&cs->wq); } + kref_put(&cm->kref, dtr_destroy_cm); return; }Signature · 1 line
2.27.0
Philipp Reisner
maintainerRe: [PATCH 03/11] drbd_transport_rdma: put kref for cm in dtr_path_established in error path
Hello easystack team,
I probably fixed the (or some of the) problems you intended to fix with this patch series.
Please see these recent commits on the master branch:
3f82aed6e drbd_transport: fix a use after free in drbd_get_listener() 1f2f11c47 rdma: remove dtr_path_established()e38120f3d rdma: Fix leaking a cm_id objecte0f5e307e rdma: fix superficial kref_put() in error code path in dtr_cma_accept()35a6b002c rdma: fix concurrency of activate_path() and __dtr_disconnect_path() 6ccd39432 rdma: fix an access after free() in dtr_destroy_cm() 5a711b347 rdma: fix free() of scheduled delayed_work847aab659 rdma: remove misguided kref_get()/kref_put()
I will go over your patch series and your comments. I'm sorry I dropped the ball on this. I was out on vacation in July, and when I came back, 1000 things needed my attention.
Best regards,
Philipp
Quoted 59 lines · Dongsheng Yang
Quoted 11 lines · Dongsheng Yang
Quoted 7 lines · Philipp Reisner
Hello Dongsheng,
Please add more information why you think this change fixes a bug. Have you experienced a leak of cm structs?
We got aRDMA_CM_EVENT_ESTABLISHEDevent. Even if DRBD does not do anything with this cm, we sill expect aRDMA_CM_EVENT_DISCONNECTEDin the future. Is a problem in the handling of the disconnect?Quoted 6 lines
If
dtr_path_established()go into this branch, it will notschedule_work(&cm->establish_work);That means
path->cm->state=DSM_CONNECTED; will not be done indtr_path_established_work_fn(), so__dtr_disconnect_path()will not callrdma_disconnect(). That means this reference will never be put.let me consider this example:
a)rdma_connect()called andRDMA_CM_EVENT_ESTABLISHEDreceived.b) network failure and
dtr_path_established()go into error path.c)
establish_workwill not be scheduled.d) drbdadm down test will hang because cm ref is not put.
Quoted 29 lines
Quoted 29 lines
best regards,
PhilippQuoted 23 lines · zhengbing.huang
From: Dongsheng Yang <[email protected]>
Signed-off-by Dongsheng Yang <[email protected]>--- drbd/drbd_transport_rdma.c | 1 + 1 file changed, 1 insertion(+)drbd/drbd_transport_rdma.c +1
-1@@ -922,6 +922,7 @@ static void dtr_path_established(struct dtr_cm *cm)atomic_set (&cs->active_state,PCS_INACTIVE );wake_up (&cs->wq); } +kref_put (&cm->kref, dtr_destroy_cm);return ; } --2.27.0
Re:Re: [PATCH 03/11] drbd_transport_rdma: put kref for cm in dtr_path_established in error path
Hi Philipp,
Thank you for your reply.
We'll get to know those patches.
Best regards,
zhengbing
From: Philipp Reisner <[email protected]>
Date: 2024-10-17 00:44:40
To: Dongsheng Yang <[email protected]>
Cc: "zhengbing.huang" <[email protected]>,[email protected],[email protected]
Subject: Re: [PATCH 03/11] drbd_transport_rdma: put kref for cm in dtr_path_established in error path>Hello easystack team,
Quoted 84 lines
I probably fixed the (or some of the) problems you intended to fix with this patch series.
Please see these recent commits on the master branch:
3f82aed6edrbd_transport: fix a use after free indrbd_get_listener()1f2f11c47rdma: removedtr_path_established()e38120f3drdma: Fix leaking acm_idobjecte0f5e307erdma: fix superficialkref_put()in error code path indtr_cma_accept()35a6b002crdma: fix concurrency ofactivate_path()and__dtr_disconnect_path()6ccd39432rdma: fix an access afterfree()indtr_destroy_cm()5a711b347rdma: fixfree()of scheduleddelayed_work847aab659rdma: remove misguided kref_get()/kref_put()I will go over your patch series and your comments. I'm sorry I dropped the ball on this. I was out on vacation in July, and when I came back, 1000 things needed my attention.
Best regards,
PhilippQuoted 59 lines · Dongsheng Yang
Quoted 11 lines · Dongsheng Yang
Quoted 7 lines · Philipp Reisner
Hello Dongsheng,
Please add more information why you think this change fixes a bug. Have you experienced a leak of cm structs?
We got aRDMA_CM_EVENT_ESTABLISHEDevent. Even if DRBD does not do anything with this cm, we sill expect aRDMA_CM_EVENT_DISCONNECTEDin the future. Is a problem in the handling of the disconnect?Quoted 6 lines
If
dtr_path_established()go into this branch, it will notschedule_work(&cm->establish_work);That means
path->cm->state=DSM_CONNECTED; will not be done indtr_path_established_work_fn(), so__dtr_disconnect_path()will not callrdma_disconnect(). That means this reference will never be put.let me consider this example:
a)rdma_connect()called andRDMA_CM_EVENT_ESTABLISHEDreceived.b) network failure and
dtr_path_established()go into error path.c)
establish_workwill not be scheduled.d) drbdadm down test will hang because cm ref is not put.
Quoted 29 lines
Quoted 29 lines
best regards,
PhilippQuoted 23 lines · zhengbing.huang
From: Dongsheng Yang <[email protected]>
Signed-off-by Dongsheng Yang <[email protected]>--- drbd/drbd_transport_rdma.c | 1 + 1 file changed, 1 insertion(+)drbd/drbd_transport_rdma.c +1
-1@@ -922,6 +922,7 @@ static void dtr_path_established(struct dtr_cm *cm)atomic_set (&cs->active_state,PCS_INACTIVE );wake_up (&cs->wq); } +kref_put (&cm->kref, dtr_destroy_cm);return ; } --2.27.0
PATCH 4/11 drbd_transport_rdma: dont schedule retry_connect_work in active is false
In reply to zhengbing.huangFrom: Dongsheng Yang <[email protected]>
--- drbd/drbd_transport_rdma.c | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-)
drbd/drbd_transport_rdma.c +10 -2
@@ -1089,9 +1089,13 @@ static void dtr_cma_retry_connect_work_fn(struct work_struct *work)if (err) { struct dtr_path *path = container_of(cs, struct dtr_path, cs);struct drbd_transport *transport = path->path.transport; +struct dtr_transport *rdma_transport = + container_of(transport, struct dtr_transport, transport);tr_err (transport,"dtr_start_try_connect failed %d\n" , err); -schedule_delayed_work (&cs->retry_connect_work,HZ ); +if (rdma_transport->active) { +schedule_delayed_work (&cs->retry_connect_work,HZ ); + } } }
@@ -1116,6 +1120,8 @@ static void dtr_remove_cm_from_path(struct dtr_path *path, struct dtr_cm *failedstatic void dtr_cma_retry_connect (struct dtr_path *path,struct dtr_cm *failed_cm) {struct drbd_transport *transport = path->path.transport; +struct dtr_transport *rdma_transport = + container_of(transport, struct dtr_transport, transport);struct dtr_connect_state *cs = &path->cs;long connect_int =10 *HZ ;struct net_conf *nc;
@@ -1128,7 +1134,9 @@ static void dtr_cma_retry_connect(struct dtr_path *path, struct dtr_cm *failed_c connect_int = nc->connect_int *HZ ;rcu_read_unlock (); -schedule_delayed_work (&cs->retry_connect_work, connect_int); +if (rdma_transport->active) { +schedule_delayed_work (&cs->retry_connect_work, connect_int); + } }static void dtr_cma_connect_work_fn (struct work_struct *work)
Signature · 1 line
2.27.0
Philipp Reisner
maintainerRe: [PATCH 04/11] drbd_transport_rdma: dont schedule retry_connect_work in active is false
Hello Dongsheng,
Please explain what problem you are fixing with this change. Do you have a log that shows a problem in this area? Please describe why your proposed change improves DRBD's behavior.
best regards,
Philipp
Quoted 50 lines · zhengbing.huang
From: Dongsheng Yang <[email protected]>
Signed-off-by Dongsheng Yang <[email protected]>--- drbd/drbd_transport_rdma.c | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-)drbd/drbd_transport_rdma.c +10
-3@@ -1089,9 +1089,13 @@ static void dtr_cma_retry_connect_work_fn(struct work_struct *work)if (err) { struct dtr_path *path = container_of(cs, struct dtr_path, cs);struct drbd_transport *transport = path->path.transport; +struct dtr_transport *rdma_transport = + container_of(transport, struct dtr_transport, transport);tr_err (transport,"dtr_start_try_connect failed %d\n" , err); -schedule_delayed_work (&cs->retry_connect_work,HZ ); +if (rdma_transport->active) { +schedule_delayed_work (&cs->retry_connect_work,HZ ); + } } }@@ -1116,6 +1120,8 @@ static void dtr_remove_cm_from_path(struct dtr_path *path, struct dtr_cm *failedstatic void dtr_cma_retry_connect (struct dtr_path *path,struct dtr_cm *failed_cm) {struct drbd_transport *transport = path->path.transport; +struct dtr_transport *rdma_transport = + container_of(transport, struct dtr_transport, transport);struct dtr_connect_state *cs = &path->cs;long connect_int =10 *HZ ;struct net_conf *nc;@@ -1128,7 +1134,9 @@ static void dtr_cma_retry_connect(struct dtr_path *path, struct dtr_cm *failed_c connect_int = nc->connect_int *HZ ;rcu_read_unlock (); -schedule_delayed_work (&cs->retry_connect_work, connect_int); +if (rdma_transport->active) { +schedule_delayed_work (&cs->retry_connect_work, connect_int); + } }static void dtr_cma_connect_work_fn (struct work_struct *work) --2.27.0
Re: [PATCH 04/11] drbd_transport_rdma: dont schedule retry_connect_work in active is false
Quoted 5 lines · Philipp Reisner
Hello Dongsheng,
Please explain what problem you are fixing with this change. Do you have a log that shows a problem in this area? Please describe why your proposed change improves DRBD's behavior.
retry_connect_work can be flushed in dtr_free, that's correct. but if we schedule new work after that, there is a NULL pointer dereference in our testing. So dont schedule new retry_connect_work when rdma_transport->active is false. it is set to false in dtr_free before flushing retry_connect_work.
Quoted 58 lines
best regards,
PhilippQuoted 48 lines · zhengbing.huang
From: Dongsheng Yang <[email protected]>
Signed-off-by Dongsheng Yang <[email protected]>--- drbd/drbd_transport_rdma.c | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-)drbd/drbd_transport_rdma.c +10
-3@@ -1089,9 +1089,13 @@ static void dtr_cma_retry_connect_work_fn(struct work_struct *work)if (err) { struct dtr_path *path = container_of(cs, struct dtr_path, cs);struct drbd_transport *transport = path->path.transport; +struct dtr_transport *rdma_transport = + container_of(transport, struct dtr_transport, transport);tr_err (transport,"dtr_start_try_connect failed %d\n" , err); -schedule_delayed_work (&cs->retry_connect_work,HZ ); +if (rdma_transport->active) { +schedule_delayed_work (&cs->retry_connect_work,HZ ); + } } }@@ -1116,6 +1120,8 @@ static void dtr_remove_cm_from_path(struct dtr_path *path, struct dtr_cm *failedstatic void dtr_cma_retry_connect (struct dtr_path *path,struct dtr_cm *failed_cm) {struct drbd_transport *transport = path->path.transport; +struct dtr_transport *rdma_transport = + container_of(transport, struct dtr_transport, transport);struct dtr_connect_state *cs = &path->cs;long connect_int =10 *HZ ;struct net_conf *nc;@@ -1128,7 +1134,9 @@ static void dtr_cma_retry_connect(struct dtr_path *path, struct dtr_cm *failed_c connect_int = nc->connect_int *HZ ;rcu_read_unlock (); -schedule_delayed_work (&cs->retry_connect_work, connect_int); +if (rdma_transport->active) { +schedule_delayed_work (&cs->retry_connect_work, connect_int); + } }static void dtr_cma_connect_work_fn (struct work_struct *work) --Quoted 2 lines
2.27.0
.
PATCH 5/11 drbd_transport_rdma: dont break in dtr_tx_cq_event_handler if (cm->state != DSM_CONNECTED)
In reply to zhengbing.huangFrom: Dongsheng Yang <[email protected]>
We need to drain all tx in disconnect to put all kref for cm
--- drbd/drbd_transport_rdma.c | 3 --- 1 file changed, 3 deletions(-)
drbd/drbd_transport_rdma.c +0 -3
@@ -1956,9 +1956,6 @@ static void dtr_tx_cq_event_handler(struct ib_cq *cq, void *ctx) err =dtr_handle_tx_cq_event (cq, cm); }while (!err); -if (cm->state !=DSM_CONNECTED ) -break ; - rc =ib_req_notify_cq (cq,IB_CQ_NEXT_COMP |IB_CQ_REPORT_MISSED_EVENTS );if (unlikely (rc <0 )) {struct drbd_transport *transport = cm->path->path.transport;
Signature · 1 line
2.27.0
Philipp Reisner
maintainerRe: [PATCH 05/11] drbd_transport_rdma: dont break in dtr_tx_cq_event_handler if (cm->state != DSM_CONNECTED)
Hello Dongsheng,
It appears that you are trying to fix a leak of cm structures. Is that correct? Do you the reference on cm that is held because of the timer? Please describe what the problem is, and how you are improving the situation.
In case this approach is the right solution, the patch should also change the dtr_handle_tx_cq_event() function to type void.
best regards,
Philipp
Quoted 27 lines · zhengbing.huang
From: Dongsheng Yang <[email protected]>
We need to drain all tx in disconnect to put all kref for cm
Signed-off-by Dongsheng Yang <[email protected]>--- drbd/drbd_transport_rdma.c | 3 --- 1 file changed, 3 deletions(-)drbd/drbd_transport_rdma.c +0
-4@@ -1956,9 +1956,6 @@ static void dtr_tx_cq_event_handler(struct ib_cq *cq, void *ctx) err =dtr_handle_tx_cq_event (cq, cm); }while (!err); -if (cm->state !=DSM_CONNECTED ) -break ; - rc =ib_req_notify_cq (cq,IB_CQ_NEXT_COMP |IB_CQ_REPORT_MISSED_EVENTS );if (unlikely (rc <0 )) {struct drbd_transport *transport = cm->path->path.transport; --2.27.0
Re: [PATCH 05/11] drbd_transport_rdma: dont break in dtr_tx_cq_event_handler if (cm->state != DSM_CONNECTED)
Quoted 3 lines · Philipp Reisner
Hello Dongsheng,
It appears that you are trying to fix a leak of cm structures. Is that correct?
Yes, in our network faulure testing, we found drbdadm down command hang at dtr_free() to wait_event(rdma_transport->cm_count_wait,!atomic_read(&rdma_transport->cm_count));,
we can find out the leak cm in memory and found the tx_descs_posted is not 0. then we did more hacking and found this problem in [05/11]
let's say this case:
a) post two tx_desc and tx_desc_posted to 2.
b) first tx_desc complete and call dtr_tx_cq_event_handler and into dtr_handle_tx_cq_event().
c) network failure and dtr_tx_timeout_work_fn() clear CONNECTED.
d) dtr_handle_tx_cq_event() returns, at this time , the second tx_desc is already complete, we expect rc = ib_req_notify_cq(cq, IB_CQ_NEXT_COMP | IB_CQ_REPORT_MISSED_EVENTS); to return 1 in rc and continue to call dtr_handle_tx_cq_event() in next while loop.
d) but it check cm->state is not CONNECTED, and break the outer while loop, so the second tx_desc will never be handled.
Quoted 40 lines
Do you the reference on cm that is held because of the timer? Please describe what the problem is, and how you are improving the situation.
In case this approach is the right solution, the patch should also change the
dtr_handle_tx_cq_event()function to type void.best regards,
PhilippQuoted 25 lines · zhengbing.huang
From: Dongsheng Yang <[email protected]>
We need to drain all tx in disconnect to put all kref for cm
Signed-off-by Dongsheng Yang <[email protected]>--- drbd/drbd_transport_rdma.c | 3 --- 1 file changed, 3 deletions(-)drbd/drbd_transport_rdma.c +0
-4@@ -1956,9 +1956,6 @@ static void dtr_tx_cq_event_handler(struct ib_cq *cq, void *ctx) err =dtr_handle_tx_cq_event (cq, cm); }while (!err); -if (cm->state !=DSM_CONNECTED ) -break ; - rc =ib_req_notify_cq (cq,IB_CQ_NEXT_COMP |IB_CQ_REPORT_MISSED_EVENTS );if (unlikely (rc <0 )) {struct drbd_transport *transport = cm->path->path.transport; --Quoted 2 lines
2.27.0
.
PATCH 6/11 drbd_transport_rdma: put kref in error path
In reply to zhengbing.huangFrom: Dongsheng Yang <[email protected]>
--- drbd/drbd_transport_rdma.c | 1 + 1 file changed, 1 insertion(+)
drbd/drbd_transport_rdma.c +1 -0
@@ -1157,6 +1157,7 @@ static void dtr_cma_connect_work_fn(struct work_struct *work)kref_get (&cm->kref);/* for the path->cm pointer */ err =dtr_path_prepare (path, cm, true);if (err) { +kref_put (&cm->kref, dtr_destroy_cm);tr_err (transport,"dtr_path_prepare() = %d\n" , err); goto out; }
Signature · 1 line
2.27.0
Philipp Reisner
maintainerRe: [PATCH 06/11] drbd_transport_rdma: put kref in error path
Hello Dongsheng,
It looks plausible, but still, please provide a description.
Quoted 23 lines · zhengbing.huang
From: Dongsheng Yang <[email protected]>
Signed-off-by Dongsheng Yang <[email protected]>--- drbd/drbd_transport_rdma.c | 1 + 1 file changed, 1 insertion(+)drbd/drbd_transport_rdma.c +1
-1@@ -1157,6 +1157,7 @@ static void dtr_cma_connect_work_fn(struct work_struct *work)kref_get (&cm->kref);/* for the path->cm pointer */ err =dtr_path_prepare (path, cm, true);if (err) { +kref_put (&cm->kref, dtr_destroy_cm);tr_err (transport,"dtr_path_prepare() = %d\n" , err); goto out; } --2.27.0
PATCH 7/11 drbd_transport_rdma: put kref in dtr_remap_tx_desc error
In reply to zhengbing.huangFrom: Dongsheng Yang <[email protected]>
--- drbd/drbd_transport_rdma.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-)
drbd/drbd_transport_rdma.c +4 -1
@@ -2355,8 +2355,11 @@ static int dtr_repost_tx_desc(struct dtr_cm *old_cm, struct dtr_tx_desc *tx_descreturn -ECONNRESET ; err =dtr_remap_tx_desc (old_cm, cm, tx_desc); -if (err) +if (err) { +pr_err ("dtr_remap_tx_desc failed cm : %px\n" , cm); +kref_put (&cm->kref, dtr_destroy_cm);continue ; + } err =__dtr_post_tx_desc (cm, tx_desc);if (!err) {
Signature · 1 line
2.27.0
Philipp Reisner
maintainerRe: [PATCH 07/11] drbd_transport_rdma: put kref in dtr_remap_tx_desc error
Hello Dongsheng,
This looks wrong. In this loop, we are trying to find a path on which to repost this tx_desc. When the remapping fails, there is no reason to drop the drop a ref on the cm.
So, please provide a description what you are intending here with this change?
best regards,
Philipp
Quoted 28 lines · zhengbing.huang
From: Dongsheng Yang <[email protected]>
Signed-off-by Dongsheng Yang <[email protected]>--- drbd/drbd_transport_rdma.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-)drbd/drbd_transport_rdma.c +4
-2@@ -2355,8 +2355,11 @@ static int dtr_repost_tx_desc(struct dtr_cm *old_cm, struct dtr_tx_desc *tx_descreturn -ECONNRESET ; err =dtr_remap_tx_desc (old_cm, cm, tx_desc); -if (err) +if (err) { +pr_err ("dtr_remap_tx_desc failed cm : %px\n" , cm); +kref_put (&cm->kref, dtr_destroy_cm);continue ; + } err =__dtr_post_tx_desc (cm, tx_desc);if (!err) { --2.27.0
Re: [PATCH 07/11] drbd_transport_rdma: put kref in dtr_remap_tx_desc error
Quoted 5 lines · Philipp Reisner
Hello Dongsheng,
This looks wrong. In this loop, we are trying to find a path on which to repost this
tx_desc. When the remapping fails, there is no reason to drop the drop a ref on the cm.
But dtr_select_and_get_cm_for_tx() get a ref, if we dont put it before continue, who will put this ref?
Quoted 38 lines
So, please provide a description what you are intending here with this change?
best regards,
PhilippQuoted 26 lines · zhengbing.huang
From: Dongsheng Yang <[email protected]>
Signed-off-by Dongsheng Yang <[email protected]>--- drbd/drbd_transport_rdma.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-)drbd/drbd_transport_rdma.c +4
-2@@ -2355,8 +2355,11 @@ static int dtr_repost_tx_desc(struct dtr_cm *old_cm, struct dtr_tx_desc *tx_descreturn -ECONNRESET ; err =dtr_remap_tx_desc (old_cm, cm, tx_desc); -if (err) +if (err) { +pr_err ("dtr_remap_tx_desc failed cm : %px\n" , cm); +kref_put (&cm->kref, dtr_destroy_cm);continue ; + } err =__dtr_post_tx_desc (cm, tx_desc);if (!err) { --Quoted 2 lines
2.27.0
.
PATCH 8/11 drbd_transport_rdma: fix a race between dtr_connect and drbd_thread_stop
In reply to zhengbing.huangFrom: Dongsheng Yang <[email protected]>
If the send_sig() in drbd_thread_stop before wait_for_completion_interruptible() in dtr_connect(), it can't return from dtr_connect in network failure.
So replace wait_for_completion_interruptible with wait_for_completion_interruptible_timeout, and check status by dtr_connect() itself.
This behavior is similar with tcp transport
--- drbd/drbd_transport_rdma.c | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-)
drbd/drbd_transport_rdma.c +12 -3
@@ -2996,12 +2996,21 @@ static int dtr_connect(struct drbd_transport *transport) {struct dtr_transport *rdma_transport = container_of(transport, struct dtr_transport, transport); -int i, err = -ENOMEM ; +int i, err; - err =wait_for_completion_interruptible (&rdma_transport->connected); -if (err) { +again: +if (drbd_should_abort_listening (transport)) { + err = -EAGAIN ; + goto abort; + } + + err =wait_for_completion_interruptible_timeout (&rdma_transport->connected,HZ ); +if (err <0 ) {flush_signals (current); goto abort; + }else if (err ==0 ) { +/* timed out */ + goto again; } err =atomic_read (&rdma_transport->first_path_connect_err);
Signature · 1 line
2.27.0
Philipp Reisner
maintainerRe: [PATCH 08/11] drbd_transport_rdma: fix a race between dtr_connect and drbd_thread_stop
Hello Dongsheng,
I am repeating your description in my own words so that you can verify I got it right:
CPU 0 executes dtr_connect() and is still before the wait_for_completion_interruptible().
CPU 1 executes send_sig() in drbd_thread_stop().
Then you conclude that wait_for_completion_interruptible() will not abort, because the signal
was raised before CPU 0 reached wait_for_completion_interruptible().
If that is your description, then it is wrong. This is not how signals and the wait_event() macros work.
best regards,
Philipp
Quoted 48 lines · zhengbing.huang
From: Dongsheng Yang <[email protected]>
If the
send_sig()indrbd_thread_stopbeforewait_for_completion_interruptible()indtr_connect(), it can't return fromdtr_connectin network failure.So replace
wait_for_completion_interruptiblewithwait_for_completion_interruptible_timeout, and check status bydtr_connect()itself.This behavior is similar with tcp transport
Signed-off-by Dongsheng Yang <[email protected]>--- drbd/drbd_transport_rdma.c | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-)drbd/drbd_transport_rdma.c +12
-4@@ -2996,12 +2996,21 @@ static int dtr_connect(struct drbd_transport *transport) {struct dtr_transport *rdma_transport = container_of(transport, struct dtr_transport, transport); -int i, err = -ENOMEM ; +int i, err; - err =wait_for_completion_interruptible (&rdma_transport->connected); -if (err) { +again: +if (drbd_should_abort_listening (transport)) { + err = -EAGAIN ; + goto abort; + } + + err =wait_for_completion_interruptible_timeout (&rdma_transport->connected,HZ ); +if (err <0 ) {flush_signals (current); goto abort; + }else if (err ==0 ) { +/* timed out */ + goto again; } err =atomic_read (&rdma_transport->first_path_connect_err); --2.27.0
Re: [PATCH 08/11] drbd_transport_rdma: fix a race between dtr_connect and drbd_thread_stop
Quoted 5 lines · Philipp Reisner
Hello Dongsheng,
I am repeating your description in my own words so that you can verify I got it right:
Quoted 7 lines
CPU 0 executes
dtr_connect()and is still before thewait_for_completion_interruptible().
CPU 1 executessend_sig()indrbd_thread_stop().Then you conclude that
wait_for_completion_interruptible()will not abort, because the signal
was raised before CPU 0 reachedwait_for_completion_interruptible().
The problem is dtr_prepare_connect() calles flush_signals(), so the signal from drbd_thread_stop() can be flushed by dtr_prepare_connect().
Quoted 57 lines
If that is your description, then it is wrong. This is not how signals and the
wait_event()macros work.best regards,
PhilippQuoted 48 lines · zhengbing.huang
From: Dongsheng Yang <[email protected]>
If the
send_sig()indrbd_thread_stopbeforewait_for_completion_interruptible()indtr_connect(), it can't return fromdtr_connectin network failure.So replace
wait_for_completion_interruptiblewithwait_for_completion_interruptible_timeout, and check status bydtr_connect()itself.This behavior is similar with tcp transport
Signed-off-by Dongsheng Yang <[email protected]>--- drbd/drbd_transport_rdma.c | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-)drbd/drbd_transport_rdma.c +12
-4@@ -2996,12 +2996,21 @@ static int dtr_connect(struct drbd_transport *transport) {struct dtr_transport *rdma_transport = container_of(transport, struct dtr_transport, transport); -int i, err = -ENOMEM ; +int i, err; - err =wait_for_completion_interruptible (&rdma_transport->connected); -if (err) { +again: +if (drbd_should_abort_listening (transport)) { + err = -EAGAIN ; + goto abort; + } + + err =wait_for_completion_interruptible_timeout (&rdma_transport->connected,HZ ); +if (err <0 ) {flush_signals (current); goto abort; + }else if (err ==0 ) { +/* timed out */ + goto again; } err =atomic_read (&rdma_transport->first_path_connect_err); --2.27.0
PATCH 9/11 drbd_transport_rdma: introduce timeout for rdma_disocnnect
In reply to zhengbing.huangFrom: Dongsheng Yang <[email protected]>
The rdma driver timeout for dreq is too long in network failure, we can introduce a timeout for rdma_disconnect().
If timeout we will put kref, and finaly it will go to rdma_destory_id(), which will cancel all dreq in rdma driver, so dont worry about use-after-free problem in dtr_cma_event_handler.
--- drbd/drbd_transport_rdma.c | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-)
drbd/drbd_transport_rdma.c +9 -3
@@ -2760,9 +2760,15 @@ static void __dtr_disconnect_path(struct dtr_path *path) }/* There might be a signal pending here. Not incorruptible! */ -wait_event_timeout (cm->state_wq, - !test_bit (DSB_CONNECTED , &cm->state), -HZ ); + err =wait_event_timeout (cm->state_wq, + !test_bit (DSB_CONNECTED , &cm->state),20 *HZ ); + +if (err ==0 &&test_and_clear_bit (DSB_CONNECTED , &cm->state)) { +dtr_remove_cm_from_path (path, cm); + +kref_put (&cm->kref, dtr_destroy_cm); +clear_bit (TR_ESTABLISHED , &path->path.flags); + }if (test_bit (DSB_CONNECTED , &cm->state)) tr_warn(transport, "WARN: not properly disconnected, state = %lu\n",
Signature · 1 line
2.27.0
PATCH 10/11 drbd_transport_rdma: introduce timeout for rdma_connect
In reply to zhengbing.huangFrom: Dongsheng Yang <[email protected]>
--- drbd/drbd_transport_rdma.c | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+)
drbd/drbd_transport_rdma.c +18 -0
@@ -140,11 +140,13 @@ union dtr_immediate {enum dtr_state_bits {DSB_CONNECT_REQ , +DSB_CONNECTING ,DSB_CONNECTED ,DSB_ERROR , };#define DSM_CONNECT_REQ (1 << DSB_CONNECT_REQ) +#define DSM_CONNECTING (1 << DSB_CONNECTING)#define DSM_CONNECTED (1 << DSB_CONNECTED)#define DSM_ERROR (1 << DSB_ERROR)
@@ -1033,6 +1035,7 @@ static int dtr_cma_accept(struct dtr_listener *listener, struct rdma_cm_id *new_return -EAGAIN ; } +set_bit (DSB_CONNECTING , &cm->state); err =rdma_accept (new_cm_id, &dtr_conn_param);if (err)kref_put (&cm->kref, dtr_destroy_cm);
@@ -1163,6 +1166,7 @@ static void dtr_cma_connect_work_fn(struct work_struct *work) }kref_get (&cm->kref);/* Expecting RDMA_CM_EVENT_ESTABLISHED */ +set_bit (DSB_CONNECTING , &cm->state); err =rdma_connect (cm->id, &dtr_conn_param);if (err) {kref_put (&cm->kref, dtr_destroy_cm);/* no RDMA_CM_EVENT_ESTABLISHED */
@@ -1170,6 +1174,15 @@ static void dtr_cma_connect_work_fn(struct work_struct *work) goto out; } + err =wait_event_timeout (cm->state_wq, + !test_bit (DSB_CONNECTING , &cm->state),20 *HZ ); + +if (err ==0 &&test_and_clear_bit (DSB_CONNECTING , &cm->state)) { +kref_put (&cm->kref, dtr_destroy_cm); +tr_err (transport,"rdma_connect timeout\n" ); + goto out; + } +kref_put (&cm->kref, dtr_destroy_cm);/* for work */ return ; out:
@@ -1293,6 +1306,9 @@ static int dtr_cma_event_handler(struct rdma_cm_id *cm_id, struct rdma_cm_eventcase RDMA_CM_EVENT_ESTABLISHED :// pr_info("%s: RDMA_CM_EVENT_ESTABLISHED\n", cm->name); +if (!test_and_clear_bit (DSB_CONNECTING , &cm->state)) +return 0 ; +wake_up (&cm->state_wq);/* cm->state = DSM_CONNECTED; is set later in the work item */ /* This is called for active and passive connections */
@@ -1313,6 +1329,8 @@ static int dtr_cma_event_handler(struct rdma_cm_id *cm_id, struct rdma_cm_event// pr_info("%s: RDMA_CM_EVENT_REJECTED\n", cm->name); // pr_info("event = %d, status = %d\n", event->event, event->status); set_bit (DSB_ERROR , &cm->state); +if (!test_and_clear_bit (DSB_CONNECTING , &cm->state)) +return 0 ;dtr_cma_retry_connect (cm->path, cm);break ;
Signature · 1 line
2.27.0
PATCH 11/11 drbd_transport_rdma: wake up state_wq after clear DSB_CONNECTED in dtr_tx_timeout_work_fn
In reply to zhengbing.huangFrom: Dongsheng Yang <[email protected]>
--- drbd/drbd_transport_rdma.c | 1 + 1 file changed, 1 insertion(+)
drbd/drbd_transport_rdma.c +1 -0
@@ -1572,6 +1572,7 @@ static void dtr_tx_timeout_work_fn(struct work_struct *work)if (!test_and_clear_bit (DSB_CONNECTED , &cm->state) || !path) goto out; +wake_up (&cm->state_wq); transport = path->path.transport; tr_warn(transport, "%pI4 - %pI4: tx timeout\n", &((struct sockaddr_in *)&path->path.my_addr)->sin_addr,
Signature · 1 line
2.27.0
Philipp Reisner
maintainerHello Dongsheng,
First of all, thanks for contributing patches to us. Please find my reply on the patch below the quote:
Quoted 9 lines · zhengbing.huang
From: Dongsheng Yang <[email protected]>
In our network failure and drbd down testing, we found warning in dmesg and drbd down process into D state:
"kernel: drbd /unregistered/ramtest3/0 drbd103: ASSERTION device->disk_state[NOW] ==
D_FAILED|| device->disk_state[NOW] ==D_DETACHINGFAILED ingo_diskless"the problem is the
wait_eventis inttruptable, it could be intrupted by signal and calldrbd_cleanup_devicebeforego_diskless()
In this case, I suggest improving the expression in the assertion. Improving an assertion can also mean removing that assertion.
The wait_event_interruptible() is there for a reason. Think of a backing disk that behaves like a tar pit—a backing device that no longer finishes IO requests. You want a way to interrupt the drbdsetup waiting in detach.
PS: A bit more elaborative commit messages are welcome.
best regards,
Philipp
Quoted 12 lines · Philipp Reisner
Hello Dongsheng,
First of all, thanks for contributing patches to us. Please find my reply on the patch below the quote:
Quoted 5 lines · zhengbing.huang
From: Dongsheng Yang <[email protected]>
In our network failure and drbd down testing, we found warning in dmesg and drbd down process into D state:
Quoted 7 lines
Quoted 4 lines
"kernel: drbd /unregistered/ramtest3/0 drbd103: ASSERTION device->disk_state[NOW] ==
D_FAILED|| device->disk_state[NOW] ==D_DETACHINGFAILED ingo_diskless"the problem is the
wait_eventis inttruptable, it could be intrupted by signal and calldrbd_cleanup_devicebeforego_diskless()In this case, I suggest improving the expression in the assertion. Improving an assertion can also mean removing that assertion.
Hi Philipp,
This patchset is fixing the problems found by a network failure test
script[1].
The [1/11] is not about just a WARNING, it will result a process with D
state in wait_event(device->misc_wait, !test_bit(GOING_DISKLESS, &device->flags)); in adm_del_minor().
let's think about this sequence:
a) drbd_adm_down -> adm_detach -> change_disk_state(device, D_DETACHING...
b) it will call put_ldev(), set GOING_DISKLESS and post a work for GO_DISKLESS
c) adm_detach() start wait_event_interruptible(device->misc_wait,
get_disk_state(device) != D_DETACHING);
but it can be intrrupted, then call drbd_cleanup_device() to set device->disk_state[NOW] = D_DISKLESS;
after that, it will go to adm_del_minor() and wait_event(device->misc_wait, !test_bit(GOING_DISKLESS, &device->flags)); which expects drbd_ldev_destroy to clear GOING_DISKLESS.
d) on the other hand, go_diskless work start and warn on the message in commit message. it will do change_disk_state(device, D_DISKLESS, CS_HARD, "go-diskless", NULL); But the disk_state[NOW] is already D_DISKLESS. So it will not schedule &device->ldev_destroy_work.
As a result, the wait_event in c) will never return.
[1]:check_drbd_process() {
ps aux | grep " D"|grep drbd
}
check_node_2_drbd_process() {
ssh node-2 'ps aux' | grep " D"|grep drbd
}
wait_for_no_drbd_d_state() {
count=0
while true; do
if check_drbd_process; then
echo "Found drbd process in D state, sleeping for ${count}
second..."
sleep 1
count=$((count + 1))
else
echo "No drbd process in D state."
break
fi
done
while true; do
if check_node_2_drbd_process; then
echo "Found drbd process in D state, sleeping for ${count}
second..."
sleep 1
count=$((count + 1))
else
echo "No drbd process in D state."
break
fi
done
}
random_sleep=$((RANDOM % 100))
ssh node-2 "ifup Bond2-roce.1469"
ifup Bond2-roce.1469
sleep 5
for i in seq 0 9; do
drbdadm up ramtest${i}
ssh node-2 "drbdadm up ramtest${i}"done
sleep ${random_sleep}
ssh node-2 "ifdown Bond2-roce.1469"
random_sleep=$((RANDOM % 10))
for i in seq 0 9; do
drbdsetup fail-io ramtest${i} &
drbdadm down ramtest${i} &done
sleep 10
wait_for_no_drbd_d_state
Quoted 11 lines
The
wait_event_interruptible()is there for a reason. Think of a backing disk that behaves like a tar pit—a backing device that no longer finishes IO requests. You want a way to interrupt the drbdsetup waiting in detach.PS: A bit more elaborative commit messages are welcome.
best regards,
Philipp
Philipp Reisner
maintainerHi Dongsheng,
Thanks for all this information! That is already nearly a perfect commit message.
Still, I am looking for a better approach to solving this problem. Instead of making the detach uninterruptible, I suggest finding a way to still schedule the ldev_destroy_work in this corner case.
PS: I prefer changes with 100 lines of commit message that touches 3 lines of code over 3 lines of commit message for 100 lines of code changes.
best regards,
Philipp
Quoted 127 lines · Dongsheng Yang
Quoted 12 lines · Philipp Reisner
Hello Dongsheng,
First of all, thanks for contributing patches to us. Please find my reply on the patch below the quote:
Quoted 5 lines · zhengbing.huang
From: Dongsheng Yang <[email protected]>
In our network failure and drbd down testing, we found warning in dmesg and drbd down process into D state:
Quoted 7 lines
Quoted 4 lines
"kernel: drbd /unregistered/ramtest3/0 drbd103: ASSERTION device->disk_state[NOW] ==
D_FAILED|| device->disk_state[NOW] ==D_DETACHINGFAILED ingo_diskless"the problem is the
wait_eventis inttruptable, it could be intrupted by signal and calldrbd_cleanup_devicebeforego_diskless()In this case, I suggest improving the expression in the assertion. Improving an assertion can also mean removing that assertion.
Hi Philipp,
This patchset is fixing the problems found by a network failure test
script[1].
The [1/11] is not about just a WARNING, it will result a process with D
state inwait_event(device->misc_wait, !test_bit(GOING_DISKLESS, &device->flags)); inadm_del_minor().let's think about this sequence:
a)
drbd_adm_down->adm_detach-> change_disk_state(device,D_DETACHING...b) it will call
put_ldev(), setGOING_DISKLESSand post a work forGO_DISKLESSc)
adm_detach()start wait_event_interruptible(device->misc_wait,get_disk_state(device) != D_DETACHING);but it can be intrrupted, then call
drbd_cleanup_device()to set device->disk_state[NOW] =D_DISKLESS;after that, it will go to
adm_del_minor()andwait_event(device->misc_wait, !test_bit(GOING_DISKLESS, &device->flags)); which expectsdrbd_ldev_destroyto clearGOING_DISKLESS.d) on the other hand,
go_disklesswork start and warn on the message in commit message. it will dochange_disk_state(device, D_DISKLESS, CS_HARD, "go-diskless", NULL); But the disk_state[NOW] is alreadyD_DISKLESS. So it will not schedule&device->ldev_destroy_work.As a result, the
wait_eventin c) will never return.[1]:
check_drbd_process(){
ps aux | grep " D"|grep drbd
}
check_node_2_drbd_process(){
ssh node-2 'ps aux' | grep " D"|grep drbd
}
wait_for_no_drbd_d_state(){
count=0
while true; do
ifcheck_drbd_process; then
echo "Found drbd process in D state, sleeping for ${count}
second..."
sleep 1
count=$((count + 1))
else
echo "No drbd process in D state."
break
fi
done
while true; do
ifcheck_node_2_drbd_process; then
echo "Found drbd process in D state, sleeping for ${count}
second..."
sleep 1
count=$((count + 1))
else
echo "No drbd process in D state."
break
fi
done
}random_sleep=$((RANDOM % 100))
ssh node-2 "ifup Bond2-roce.1469"
ifup Bond2-roce.1469sleep 5
for i in
seq 0 9; dodrbdadm up ramtest${i} ssh node-2 "drbdadm up ramtest${i}"done
sleep ${random_sleep}
ssh node-2 "ifdown Bond2-roce.1469"
random_sleep=$((RANDOM % 10))
for i in
seq 0 9; dodrbdsetup fail-io ramtest${i} & drbdadm down ramtest${i} &done
sleep 10
wait_for_no_drbd_d_stateQuoted 11 lines
The
wait_event_interruptible()is there for a reason. Think of a backing disk that behaves like a tar pit—a backing device that no longer finishes IO requests. You want a way to interrupt the drbdsetup waiting in detach.PS: A bit more elaborative commit messages are welcome.
best regards,
Philipp
Quoted 7 lines · Philipp Reisner
Hi Dongsheng,
Thanks for all this information! That is already nearly a perfect commit message.
Still, I am looking for a better approach to solving this problem. Instead of making the detach uninterruptible, I suggest finding a way to still schedule theldev_destroy_workin this corner case.
Sounds good, we are willing to do review and test for your patch when it's ready.
Quoted 4 lines
PS: I prefer changes with 100 lines of commit message that touches 3 lines of code over 3 lines of commit message for 100 lines of code changes.
Totally understood, that's important for discussion and code maintaining.
Thanx
Quoted 133 lines
best regards,
PhilippQuoted 127 lines · Dongsheng Yang
Quoted 12 lines · Philipp Reisner
Hello Dongsheng,
First of all, thanks for contributing patches to us. Please find my reply on the patch below the quote:
Quoted 5 lines · zhengbing.huang
From: Dongsheng Yang <[email protected]>
In our network failure and drbd down testing, we found warning in dmesg and drbd down process into D state:
Quoted 7 lines
Quoted 4 lines
"kernel: drbd /unregistered/ramtest3/0 drbd103: ASSERTION device->disk_state[NOW] ==
D_FAILED|| device->disk_state[NOW] ==D_DETACHINGFAILED ingo_diskless"the problem is the
wait_eventis inttruptable, it could be intrupted by signal and calldrbd_cleanup_devicebeforego_diskless()In this case, I suggest improving the expression in the assertion. Improving an assertion can also mean removing that assertion.
Hi Philipp,
This patchset is fixing the problems found by a network failure test
script[1].
The [1/11] is not about just a WARNING, it will result a process with D
state inwait_event(device->misc_wait, !test_bit(GOING_DISKLESS, &device->flags)); inadm_del_minor().let's think about this sequence:
a)
drbd_adm_down->adm_detach-> change_disk_state(device,D_DETACHING...b) it will call
put_ldev(), setGOING_DISKLESSand post a work forGO_DISKLESSc)
adm_detach()start wait_event_interruptible(device->misc_wait,get_disk_state(device) != D_DETACHING);but it can be intrrupted, then call
drbd_cleanup_device()to set device->disk_state[NOW] =D_DISKLESS;after that, it will go to
adm_del_minor()andwait_event(device->misc_wait, !test_bit(GOING_DISKLESS, &device->flags)); which expectsdrbd_ldev_destroyto clearGOING_DISKLESS.d) on the other hand,
go_disklesswork start and warn on the message in commit message. it will dochange_disk_state(device, D_DISKLESS, CS_HARD, "go-diskless", NULL); But the disk_state[NOW] is alreadyD_DISKLESS. So it will not schedule&device->ldev_destroy_work.As a result, the
wait_eventin c) will never return.[1]:
check_drbd_process(){
ps aux | grep " D"|grep drbd
}
check_node_2_drbd_process(){
ssh node-2 'ps aux' | grep " D"|grep drbd
}
wait_for_no_drbd_d_state(){
count=0
while true; do
ifcheck_drbd_process; then
echo "Found drbd process in D state, sleeping for ${count}
second..."
sleep 1
count=$((count + 1))
else
echo "No drbd process in D state."
break
fi
done
while true; do
ifcheck_node_2_drbd_process; then
echo "Found drbd process in D state, sleeping for ${count}
second..."
sleep 1
count=$((count + 1))
else
echo "No drbd process in D state."
break
fi
done
}random_sleep=$((RANDOM % 100))
ssh node-2 "ifup Bond2-roce.1469"
ifup Bond2-roce.1469sleep 5
for i in
seq 0 9; dodrbdadm up ramtest${i} ssh node-2 "drbdadm up ramtest${i}"done
sleep ${random_sleep}
ssh node-2 "ifdown Bond2-roce.1469"
random_sleep=$((RANDOM % 10))
for i in
seq 0 9; dodrbdsetup fail-io ramtest${i} & drbdadm down ramtest${i} &done
sleep 10
wait_for_no_drbd_d_stateQuoted 11 lines
The
wait_event_interruptible()is there for a reason. Think of a backing disk that behaves like a tar pit—a backing device that no longer finishes IO requests. You want a way to interrupt the drbdsetup waiting in detach.PS: A bit more elaborative commit messages are welcome.
best regards,
Philipp
Philipp Reisner
maintainerPATCH 1/1 drbd: make drbd_adm_detach() interruptible
If a backing device suddenly ceases delivering I/O completions, and in reaction, the user issues a drbdsetup detach, the operation will hang when it tries to write internal meta-data.
The user should have used drbdsetup --force detach, but it is too late. There was no way to interrupt the hanging drbdsetup detach.
Improve the situation by making detach operations interruptible.
--- drbd/drbd_actlog.c | 5 ++++- drbd/drbd_int.h | 1 + drbd/drbd_state.c | 29 +++++++++++++++++++++++++++-- 3 files changed, 32 insertions(+), 3 deletions(-)
drbd/drbd_actlog.c +4 -1
@@ -74,7 +74,10 @@ void wait_until_done_or_force_detached(struct drbd_device *device, struct drbd_b dt =MAX_SCHEDULE_TIMEOUT ; dt =wait_event_timeout (device->misc_wait, - *done ||test_bit (FORCE_DETACH , &device->flags), dt); + *done || +test_bit (FORCE_DETACH , &device->flags) || +test_bit (INTERRUPT_DETACH , &device->flags), + dt);if (dt ==0 ) {drbd_err (device,"meta-data IO operation timed out\n" );drbd_handle_io_error (device,DRBD_FORCE_DETACH );
drbd/drbd_int.h +1 -0
@@ -521,6 +521,7 @@ enum device_flag {MD_NO_FUA ,/* meta data device does not support barriers, so don't even try */ FORCE_DETACH ,/* Force-detach from local disk, aborting any pending local IO */ +INTERRUPT_DETACH ,/* Interrupt an ongoing detach operation */ NEW_CUR_UUID ,/* Create new current UUID when thawing IO or issuing local IO */ __NEW_CUR_UUID,/* Set NEW_CUR_UUID as soon as state change visible */ WRITING_NEW_CUR_UUID, /* Set while the new current ID gets generated. */
drbd/drbd_state.c +27 -2
@@ -924,14 +924,39 @@ void state_change_lock(struct drbd_resource *resource, unsigned long *irq_flags, resource->state_change_flags = flags; } +/* Interrupt writing meta-data */ +static void interrupt_detach (struct drbd_resource *resource,struct completion *done) +{ +struct drbd_device *device; +int vnr; + +idr_for_each_entry (&resource->devices, device, vnr) { +if (device->disk_state[NOW ] ==D_DETACHING ) { +set_bit (INTERRUPT_DETACH , &device->flags); +wake_up_all (&device->misc_wait); + } + } + +wait_for_completion (done); + +idr_for_each_entry (&resource->devices, device, vnr) { +if (test_bit (INTERRUPT_DETACH , &device->flags)) +clear_bit (INTERRUPT_DETACH , &device->flags); + } +} +static void __state_change_unlock (struct drbd_resource *resource,unsigned long *irq_flags,struct completion *done) {enum chg_state_flags flags = resource->state_change_flags; resource->state_change_flags =0 ;write_unlock_irqrestore (&resource->state_rwlock, *irq_flags); -if (done &&expect (resource, current != resource->worker.task)) -wait_for_completion (done); +if (done &&expect (resource, current != resource->worker.task)) { +int err =wait_for_completion_interruptible (done); + +if (err == -ERESTARTSYS ) +interrupt_detach (resource, done); + }if ((flags &CS_SERIALIZE ) && !(flags & (CS_ALREADY_SERIALIZED |CS_PREPARE )))up (&resource->state_sem); }
Signature · 1 line
2.45.2
Re:[PATCH] drbd: make drbd_adm_detach() interruptible
Hi Philipp,
IIUC, this patch is going to fix the following problem:
(1) backing disk error and no completion returned from hardware.
(2) drbdsetup detach command will queue_after_state_change_work(resource, done, work); this work will be handled in drbd_worker, but it con't finish in drbd_md_sync()->wait_until_done_or_force_detached()
(3) drbdsetup detach process will continue to __state_change_unlock and hang as the related after_state_change work not complete.
So this patch allow user to use kill command to send signal to drbdsetup detach command, then it go into interrupt_detach(), and interrupt_detach() can make wait_until_done_or_force_detached() continue.
After that, w_after_state_change() can continue to complete(worker->done), which makes drbdsetup detach process continue.
If this is what you want, I think it fix a different problem case based on the [1/11] in our patchset, So we need [1/11] and this patch both, right?
best regards,
zhengbing
From: Philipp Reisner <[email protected]>Date: 2024-07-03 22:31:35
To: Dongsheng Yang <[email protected]>
Cc: "zhengbing . huang" <[email protected]>,[email protected],Philipp Reisner <[email protected]>
Subject: [PATCH] drbd: make drbd_adm_detach() interruptible>If a backing device suddenly ceases delivering I/O completions, and in
Quoted 90 lines
reaction, the user issues a
drbdsetup detach, the operation will hang when it tries to write internal meta-data.The user should have used
drbdsetup --force detach, but it is too late. There was no way to interrupt the hanging drbdsetup detach.Improve the situation by making detach operations interruptible.
---drbd/drbd_actlog.c | 5 ++++- drbd/drbd_int.h | 1 + drbd/drbd_state.c | 29 +++++++++++++++++++++++++++-- 3 files changed, 32 insertions(+), 3 deletions(-)drbd/drbd_actlog.c +4
-1@@ -74,7 +74,10 @@ void wait_until_done_or_force_detached(struct drbd_device *device, struct drbd_b dt =MAX_SCHEDULE_TIMEOUT ; dt =wait_event_timeout (device->misc_wait, - *done ||test_bit (FORCE_DETACH , &device->flags), dt); + *done || +test_bit (FORCE_DETACH , &device->flags) || +test_bit (INTERRUPT_DETACH , &device->flags), + dt);if (dt ==0 ) {drbd_err (device,"meta-data IO operation timed out\n" );drbd_handle_io_error (device,DRBD_FORCE_DETACH );drbd/drbd_int.h +1
-0@@ -521,6 +521,7 @@ enum device_flag {MD_NO_FUA ,/* meta data device does not support barriers, so don't even try */ FORCE_DETACH ,/* Force-detach from local disk, aborting any pending local IO */ +INTERRUPT_DETACH ,/* Interrupt an ongoing detach operation */ NEW_CUR_UUID ,/* Create new current UUID when thawing IO or issuing local IO */ __NEW_CUR_UUID,/* Set NEW_CUR_UUID as soon as state change visible */ WRITING_NEW_CUR_UUID, /* Set while the new current ID gets generated. */drbd/drbd_state.c +27
-2@@ -924,14 +924,39 @@ void state_change_lock(struct drbd_resource *resource, unsigned long *irq_flags, resource->state_change_flags = flags; } +/* Interrupt writing meta-data */ +static void interrupt_detach (struct drbd_resource *resource,struct completion *done) +{ +struct drbd_device *device; +int vnr; + +idr_for_each_entry (&resource->devices, device, vnr) { +if (device->disk_state[NOW ] ==D_DETACHING ) { +set_bit (INTERRUPT_DETACH , &device->flags); +wake_up_all (&device->misc_wait); + } + } + +wait_for_completion (done); + +idr_for_each_entry (&resource->devices, device, vnr) { +if (test_bit (INTERRUPT_DETACH , &device->flags)) +clear_bit (INTERRUPT_DETACH , &device->flags); + } +} +static void __state_change_unlock (struct drbd_resource *resource,unsigned long *irq_flags,struct completion *done) {enum chg_state_flags flags = resource->state_change_flags; resource->state_change_flags =0 ;write_unlock_irqrestore (&resource->state_rwlock, *irq_flags); -if (done &&expect (resource, current != resource->worker.task)) -wait_for_completion (done); +if (done &&expect (resource, current != resource->worker.task)) { +int err =wait_for_completion_interruptible (done); + +if (err == -ERESTARTSYS ) +interrupt_detach (resource, done); + }if ((flags &CS_SERIALIZE ) && !(flags & (CS_ALREADY_SERIALIZED |CS_PREPARE )))up (&resource->state_sem); }Signature · 1 line
2.45.2