libceph: fix potential hang in ceph_osdc_notify()
Ilya Dryomov
maintainerPATCH 1/1 libceph: fix potential hang in ceph_osdc_notify()
If the cluster becomes unavailable, ceph_osdc_notify() may hang even with osd_request_timeout option set because linger_notify_finish_wait() waits for MWatchNotify NOTIFY_COMPLETE message with no associated OSD request in flight -- it's completely asynchronous.
Introduce an additional timeout, derived from the specified notify timeout. While at it, switch both waits to killable which is more correct.
--- net/ceph/osd_client.c | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-)
net/ceph/osd_client.c +14 -6
@@ -3334,17 +3334,24 @@ static int linger_reg_commit_wait(struct ceph_osd_linger_request *lreq)int ret;dout ("%s lreq %p linger_id %llu\n" , __func__, lreq, lreq->linger_id); - ret =wait_for_completion_interruptible (&lreq->reg_commit_wait); + ret =wait_for_completion_killable (&lreq->reg_commit_wait);return ret ?: lreq->reg_commit_error; } -static int linger_notify_finish_wait (struct ceph_osd_linger_request *lreq) +static int linger_notify_finish_wait (struct ceph_osd_linger_request *lreq, +unsigned long timeout) { -int ret; +long left;dout ("%s lreq %p linger_id %llu\n" , __func__, lreq, lreq->linger_id); - ret =wait_for_completion_interruptible (&lreq->notify_finish_wait); -return ret ?: lreq->notify_finish_error; + left =wait_for_completion_killable_timeout (&lreq->notify_finish_wait, +ceph_timeout_jiffies (timeout)); +if (left <=0 ) + left = left ?: -ETIMEDOUT ; +else + left = lreq->notify_finish_error;/* completed */ + +return left; }/*
@@ -4896,7 +4903,8 @@ int ceph_osdc_notify(struct ceph_osd_client *osdc,linger_submit (lreq); ret =linger_reg_commit_wait (lreq);if (!ret) - ret =linger_notify_finish_wait (lreq); + ret =linger_notify_finish_wait (lreq, +msecs_to_jiffies (2 * timeout *MSEC_PER_SEC ));else dout ("lreq %p failed to initiate notify %d\n" , lreq, ret);
Signature · 1 line
2.41.0
LGTM.
Quoted 59 lines · Ilya Dryomov
If the cluster becomes unavailable,
ceph_osdc_notify()may hang even withosd_request_timeoutoption set becauselinger_notify_finish_wait()waits for MWatchNotifyNOTIFY_COMPLETEmessage with no associated OSD request in flight -- it's completely asynchronous.Introduce an additional timeout, derived from the specified notify timeout. While at it, switch both waits to killable which is more correct.
Signed-off-by Ilya Dryomov <[email protected]>--- net/ceph/osd_client.c | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-)net/ceph/osd_client.c +14
-6@@ -3334,17 +3334,24 @@ static int linger_reg_commit_wait(struct ceph_osd_linger_request *lreq)int ret;dout ("%s lreq %p linger_id %llu\n" , __func__, lreq, lreq->linger_id); - ret =wait_for_completion_interruptible (&lreq->reg_commit_wait); + ret =wait_for_completion_killable (&lreq->reg_commit_wait);return ret ?: lreq->reg_commit_error; } -static int linger_notify_finish_wait (struct ceph_osd_linger_request *lreq) +static int linger_notify_finish_wait (struct ceph_osd_linger_request *lreq, +unsigned long timeout) { -int ret; +long left;dout ("%s lreq %p linger_id %llu\n" , __func__, lreq, lreq->linger_id); - ret =wait_for_completion_interruptible (&lreq->notify_finish_wait); -return ret ?: lreq->notify_finish_error; + left =wait_for_completion_killable_timeout (&lreq->notify_finish_wait, +ceph_timeout_jiffies (timeout)); +if (left <=0 ) + left = left ?: -ETIMEDOUT ; +else + left = lreq->notify_finish_error;/* completed */ + +return left; }/* @@ -4896,7 +4903,8 @@ int ceph_osdc_notify(struct ceph_osd_client *osdc,linger_submit (lreq); ret =linger_reg_commit_wait (lreq);if (!ret) - ret =linger_notify_finish_wait (lreq); + ret =linger_notify_finish_wait (lreq, +msecs_to_jiffies (2 * timeout *MSEC_PER_SEC ));else dout ("lreq %p failed to initiate notify %d\n" , lreq, ret);
Quoted 11 lines · Ilya Dryomov
If the cluster becomes unavailable,
ceph_osdc_notify()may hang even withosd_request_timeoutoption set becauselinger_notify_finish_wait()waits for MWatchNotifyNOTIFY_COMPLETEmessage with no associated OSD request in flight -- it's completely asynchronous.Introduce an additional timeout, derived from the specified notify timeout. While at it, switch both waits to killable which is more correct.
Signed-off-by Ilya Dryomov <[email protected]>
Quoted 49 lines
--- net/ceph/osd_client.c | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-)net/ceph/osd_client.c +14
-6@@ -3334,17 +3334,24 @@ static int linger_reg_commit_wait(struct ceph_osd_linger_request *lreq)int ret;dout ("%s lreq %p linger_id %llu\n" , __func__, lreq, lreq->linger_id); - ret =wait_for_completion_interruptible (&lreq->reg_commit_wait); + ret =wait_for_completion_killable (&lreq->reg_commit_wait);return ret ?: lreq->reg_commit_error; } -static int linger_notify_finish_wait (struct ceph_osd_linger_request *lreq) +static int linger_notify_finish_wait (struct ceph_osd_linger_request *lreq, +unsigned long timeout) { -int ret; +long left;dout ("%s lreq %p linger_id %llu\n" , __func__, lreq, lreq->linger_id); - ret =wait_for_completion_interruptible (&lreq->notify_finish_wait); -return ret ?: lreq->notify_finish_error; + left =wait_for_completion_killable_timeout (&lreq->notify_finish_wait, +ceph_timeout_jiffies (timeout)); +if (left <=0 ) + left = left ?: -ETIMEDOUT ; +else + left = lreq->notify_finish_error;/* completed */ + +return left; }/* @@ -4896,7 +4903,8 @@ int ceph_osdc_notify(struct ceph_osd_client *osdc,linger_submit (lreq); ret =linger_reg_commit_wait (lreq);if (!ret) - ret =linger_notify_finish_wait (lreq); + ret =linger_notify_finish_wait (lreq, +msecs_to_jiffies (2 * timeout *MSEC_PER_SEC ));else dout ("lreq %p failed to initiate notify %d\n" , lreq, ret);