ceph: stop forwarding the request when exceeding 256 times
PATCH v3 1/1 ceph: stop forwarding the request when exceeding 256 times
From: Xiubo Li <[email protected]>
The type of 'num_fwd' in ceph 'MClientRequestForward' is 'int32_t', while in 'ceph_mds_request_head' the type is '__u8'. So in case the request bounces between MDSes exceeding 256 times, the client will get stuck.
In this case it's ususally a bug in MDS and continue bouncing the request makes no sense.
---
V3:
- avoid usig the hardcode of 256
V2:
- s/EIO/EMULTIHOP/
- Fixed dereferencing NULL seq bug
- Removed the out lable
fs/ceph/mds_client.c | 39 ++++++++++++++++++++++++++++++++++----- 1 file changed, 34 insertions(+), 5 deletions(-)
fs/ceph/mds_client.c +34 -5
@@ -3293,6 +3293,7 @@ static void handle_forward(struct ceph_mds_client *mdsc,int err = -EINVAL ;void *p = msg->front.iov_base;void *end = p + msg->front.iov_len; +bool aborted = false;ceph_decode_need (&p, end,2 *sizeof (u32), bad); next_mds =ceph_decode_32 (&p);
@@ -3301,16 +3302,41 @@ static void handle_forward(struct ceph_mds_client *mdsc,mutex_lock (&mdsc->mutex); req =lookup_get_request (mdsc, tid);if (!req) { +mutex_unlock (&mdsc->mutex);dout ("forward tid %llu to mds%d - req dne\n" , tid, next_mds); - goto out;/* dup reply? */ +return ;/* dup reply? */ }if (test_bit (CEPH_MDS_R_ABORTED , &req->r_req_flags)) {dout ("forward tid %llu aborted, unregistering\n" , tid);__unregister_request (mdsc, req); }else if (fwd_seq <= req->r_num_fwd) { -dout ("forward tid %llu to mds%d - old seq %d <= %d\n" , - tid, next_mds, req->r_num_fwd, fwd_seq); +/* +* The type of 'num_fwd' in ceph 'MClientRequestForward' +* is 'int32_t', while in 'ceph_mds_request_head' the +* type is '__u8'. So in case the request bounces between +* MDSes exceeding 256 times, the client will get stuck. +* +* In this case it's ususally a bug in MDS and continue +* bouncing the request makes no sense. +* +* In future this could be fixed in ceph code, so avoid +* using the hardcode here. +*/ + int max = sizeof_field(struct ceph_mds_request_head, num_fwd); + max =1 << (max *BITS_PER_BYTE ); +if (req->r_num_fwd >= max) { +mutex_lock (&req->r_fill_mutex); + req->r_err = -EMULTIHOP ; +set_bit (CEPH_MDS_R_ABORTED , &req->r_req_flags); +mutex_unlock (&req->r_fill_mutex); + aborted = true; +pr_warn_ratelimited ("forward tid %llu seq overflow\n" , + tid); + }else { +dout ("forward tid %llu to mds%d - old seq %d <= %d\n" , + tid, next_mds, req->r_num_fwd, fwd_seq); + } }else {/* resend. forward race not possible; mds would drop */ dout ("forward tid %llu to mds%d (we resend)\n" , tid, next_mds);
@@ -3322,9 +3348,12 @@ static void handle_forward(struct ceph_mds_client *mdsc,put_request_session (req);__do_request (mdsc, req); } -ceph_mdsc_put_request (req); -out:mutex_unlock (&mdsc->mutex); + +/* kick calling process */ +if (aborted) +complete_request (mdsc, req); +ceph_mdsc_put_request (req);return ; bad:
Signature · 1 line
2.27.0
Quoted 87 lines · On Wed, 2022-03-30 at 09:25 +0800, [email protected] wrote:
From: Xiubo Li <[email protected]>
The type of '
num_fwd' in ceph 'MClientRequestForward' is 'int32_t', while in 'ceph_mds_request_head' the type is '__u8'. So in case the request bounces between MDSes exceeding 256 times, the client will get stuck.In this case it's ususally a bug in MDS and continue bouncing the request makes no sense.
Signed-off-by Xiubo Li <[email protected]>---
V3:
- avoid usig the hardcode of 256V2:
- s/EIO/EMULTIHOP/
- Fixed dereferencing NULL seq bug
- Removed the out lablefs/ceph/mds_client.c | 39 ++++++++++++++++++++++++++++++++++----- 1 file changed, 34 insertions(+), 5 deletions(-)fs/ceph/mds_client.c +29
-4@@ -3293,6 +3293,7 @@ static void handle_forward(struct ceph_mds_client *mdsc,int err = -EINVAL ;void *p = msg->front.iov_base;void *end = p + msg->front.iov_len; +bool aborted = false;ceph_decode_need (&p, end,2 *sizeof (u32), bad); next_mds =ceph_decode_32 (&p);@@ -3301,16 +3302,41 @@ static void handle_forward(struct ceph_mds_client *mdsc,mutex_lock (&mdsc->mutex); req =lookup_get_request (mdsc, tid);if (!req) { +mutex_unlock (&mdsc->mutex);dout ("forward tid %llu to mds%d - req dne\n" , tid, next_mds); - goto out;/* dup reply? */ +return ;/* dup reply? */ }if (test_bit (CEPH_MDS_R_ABORTED , &req->r_req_flags)) {dout ("forward tid %llu aborted, unregistering\n" , tid);__unregister_request (mdsc, req); }else if (fwd_seq <= req->r_num_fwd) { -dout ("forward tid %llu to mds%d - old seq %d <= %d\n" , - tid, next_mds, req->r_num_fwd, fwd_seq); +/* +* The type of 'num_fwd' in ceph 'MClientRequestForward' +* is 'int32_t', while in 'ceph_mds_request_head' the +* type is '__u8'. So in case the request bounces between +* MDSes exceeding 256 times, the client will get stuck. +* +* In this case it's ususally a bug in MDS and continue +* bouncing the request makes no sense. +* +* In future this could be fixed in ceph code, so avoid +* using the hardcode here. +*/ + int max = sizeof_field(struct ceph_mds_request_head, num_fwd); + max =1 << (max *BITS_PER_BYTE ); +if (req->r_num_fwd >= max) { +mutex_lock (&req->r_fill_mutex); + req->r_err = -EMULTIHOP ; +set_bit (CEPH_MDS_R_ABORTED , &req->r_req_flags); +mutex_unlock (&req->r_fill_mutex); + aborted = true; +pr_warn_ratelimited ("forward tid %llu seq overflow\n" , + tid); + }else { +dout ("forward tid %llu to mds%d - old seq %d <= %d\n" , + tid, next_mds, req->r_num_fwd, fwd_seq); + } }else {/* resend. forward race not possible; mds would drop */ dout ("forward tid %llu to mds%d (we resend)\n" , tid, next_mds);@@ -3322,9 +3348,12 @@ static void handle_forward(struct ceph_mds_client *mdsc,put_request_session (req);__do_request (mdsc, req); } -ceph_mdsc_put_request (req);
fs/ceph/mds_client.c static void handle_forward(struct ceph_mds_client *mdsc, · line 3359
Quoted 10 lines
fs/ceph/mds_client.c +5
-1-out:mutex_unlock (&mdsc->mutex); + +/* kick calling process */ +if (aborted) +complete_request (mdsc, req); +ceph_mdsc_put_request (req);return ; bad:
Quoted 101 lines · [email protected] writes:
From: Xiubo Li <[email protected]>
The type of '
num_fwd' in ceph 'MClientRequestForward' is 'int32_t', while in 'ceph_mds_request_head' the type is '__u8'. So in case the request bounces between MDSes exceeding 256 times, the client will get stuck.In this case it's ususally a bug in MDS and continue bouncing the request makes no sense.
Signed-off-by Xiubo Li <[email protected]>---
V3:
- avoid usig the hardcode of 256V2:
- s/EIO/EMULTIHOP/
- Fixed dereferencing NULL seq bug
- Removed the out lablefs/ceph/mds_client.c | 39 ++++++++++++++++++++++++++++++++++----- 1 file changed, 34 insertions(+), 5 deletions(-)fs/ceph/mds_client.c +34
-5@@ -3293,6 +3293,7 @@ static void handle_forward(struct ceph_mds_client *mdsc,int err = -EINVAL ;void *p = msg->front.iov_base;void *end = p + msg->front.iov_len; +bool aborted = false;ceph_decode_need (&p, end,2 *sizeof (u32), bad); next_mds =ceph_decode_32 (&p);@@ -3301,16 +3302,41 @@ static void handle_forward(struct ceph_mds_client *mdsc,mutex_lock (&mdsc->mutex); req =lookup_get_request (mdsc, tid);if (!req) { +mutex_unlock (&mdsc->mutex);dout ("forward tid %llu to mds%d - req dne\n" , tid, next_mds); - goto out;/* dup reply? */ +return ;/* dup reply? */ }if (test_bit (CEPH_MDS_R_ABORTED , &req->r_req_flags)) {dout ("forward tid %llu aborted, unregistering\n" , tid);__unregister_request (mdsc, req); }else if (fwd_seq <= req->r_num_fwd) { -dout ("forward tid %llu to mds%d - old seq %d <= %d\n" , - tid, next_mds, req->r_num_fwd, fwd_seq); +/* +* The type of 'num_fwd' in ceph 'MClientRequestForward' +* is 'int32_t', while in 'ceph_mds_request_head' the +* type is '__u8'. So in case the request bounces between +* MDSes exceeding 256 times, the client will get stuck. +* +* In this case it's ususally a bug in MDS and continue +* bouncing the request makes no sense. +* +* In future this could be fixed in ceph code, so avoid +* using the hardcode here. +*/ + int max = sizeof_field(struct ceph_mds_request_head, num_fwd); + max =1 << (max *BITS_PER_BYTE ); +if (req->r_num_fwd >= max) { +mutex_lock (&req->r_fill_mutex); + req->r_err = -EMULTIHOP ; +set_bit (CEPH_MDS_R_ABORTED , &req->r_req_flags); +mutex_unlock (&req->r_fill_mutex); + aborted = true; +pr_warn_ratelimited ("forward tid %llu seq overflow\n" , + tid); + }else { +dout ("forward tid %llu to mds%d - old seq %d <= %d\n" , + tid, next_mds, req->r_num_fwd, fwd_seq); + } }else {/* resend. forward race not possible; mds would drop */ dout ("forward tid %llu to mds%d (we resend)\n" , tid, next_mds);@@ -3322,9 +3348,12 @@ static void handle_forward(struct ceph_mds_client *mdsc,put_request_session (req);__do_request (mdsc, req); } -ceph_mdsc_put_request (req); -out:mutex_unlock (&mdsc->mutex); + +/* kick calling process */ +if (aborted) +complete_request (mdsc, req); +ceph_mdsc_put_request (req);return ; bad:Signature · 1 line
2.27.0
Yeah, looks good to me. Thanks.
Cheers,
Signature · 1 line
Luís