ceph: stop forwarding the request when exceeding 256 times
PATCH v2 1/1 ceph: stop forwarding the request when exceeding 256 times
From: Xiubo Li <[email protected]>
The type of 'num_fwd' in ceph 'MClientRequestForward' is 'int32_t', while in 'ceph_mds_request_head' the type is '__u8'. So in case the request bounces between MDSes exceeding 256 times, the client will get stuck.
In this case it's ususally a bug in MDS and continue bouncing the request makes no sense.
---
V2:
- s/EIO/EMULTIHOP/
- Fixed dereferencing NULL seq bug
- Removed the out lable
fs/ceph/mds_client.c | 34 +++++++++++++++++++++++++++++----- 1 file changed, 29 insertions(+), 5 deletions(-)
fs/ceph/mds_client.c +29 -5
@@ -3293,6 +3293,7 @@ static void handle_forward(struct ceph_mds_client *mdsc,int err = -EINVAL ;void *p = msg->front.iov_base;void *end = p + msg->front.iov_len; +bool aborted = false;ceph_decode_need (&p, end,2 *sizeof (u32), bad); next_mds =ceph_decode_32 (&p);
@@ -3301,16 +3302,36 @@ static void handle_forward(struct ceph_mds_client *mdsc,mutex_lock (&mdsc->mutex); req =lookup_get_request (mdsc, tid);if (!req) { +mutex_unlock (&mdsc->mutex);dout ("forward tid %llu to mds%d - req dne\n" , tid, next_mds); - goto out;/* dup reply? */ +return ;/* dup reply? */ }if (test_bit (CEPH_MDS_R_ABORTED , &req->r_req_flags)) {dout ("forward tid %llu aborted, unregistering\n" , tid);__unregister_request (mdsc, req); }else if (fwd_seq <= req->r_num_fwd) { -dout ("forward tid %llu to mds%d - old seq %d <= %d\n" , - tid, next_mds, req->r_num_fwd, fwd_seq); +/* +* The type of 'num_fwd' in ceph 'MClientRequestForward' +* is 'int32_t', while in 'ceph_mds_request_head' the +* type is '__u8'. So in case the request bounces between +* MDSes exceeding 256 times, the client will get stuck. +* +* In this case it's ususally a bug in MDS and continue +* bouncing the request makes no sense. +*/ +if (req->r_num_fwd ==256 ) { +mutex_lock (&req->r_fill_mutex); + req->r_err = -EMULTIHOP ; +set_bit (CEPH_MDS_R_ABORTED , &req->r_req_flags); +mutex_unlock (&req->r_fill_mutex); + aborted = true; +pr_warn_ratelimited ("forward tid %llu seq overflow\n" , + tid); + }else { +dout ("forward tid %llu to mds%d - old seq %d <= %d\n" , + tid, next_mds, req->r_num_fwd, fwd_seq); + } }else {/* resend. forward race not possible; mds would drop */ dout ("forward tid %llu to mds%d (we resend)\n" , tid, next_mds);
@@ -3322,9 +3343,12 @@ static void handle_forward(struct ceph_mds_client *mdsc,put_request_session (req);__do_request (mdsc, req); } -ceph_mdsc_put_request (req); -out:mutex_unlock (&mdsc->mutex); + +/* kick calling process */ +if (aborted) +complete_request (mdsc, req); +ceph_mdsc_put_request (req);return ; bad:
Signature · 1 line
2.27.0
Quoted 50 lines · On Tue, 2022-03-29 at 20:20 +0800, [email protected] wrote:
From: Xiubo Li <[email protected]>
The type of '
num_fwd' in ceph 'MClientRequestForward' is 'int32_t', while in 'ceph_mds_request_head' the type is '__u8'. So in case the request bounces between MDSes exceeding 256 times, the client will get stuck.In this case it's ususally a bug in MDS and continue bouncing the request makes no sense.
Signed-off-by Xiubo Li <[email protected]>---
V2:
- s/EIO/EMULTIHOP/
- Fixed dereferencing NULL seq bug
- Removed the out lablefs/ceph/mds_client.c | 34 +++++++++++++++++++++++++++++----- 1 file changed, 29 insertions(+), 5 deletions(-)fs/ceph/mds_client.c +3
-3@@ -3293,6 +3293,7 @@ static void handle_forward(struct ceph_mds_client *mdsc,int err = -EINVAL ;void *p = msg->front.iov_base;void *end = p + msg->front.iov_len; +bool aborted = false;ceph_decode_need (&p, end,2 *sizeof (u32), bad); next_mds =ceph_decode_32 (&p);@@ -3301,16 +3302,36 @@ static void handle_forward(struct ceph_mds_client *mdsc,mutex_lock (&mdsc->mutex); req =lookup_get_request (mdsc, tid);if (!req) { +mutex_unlock (&mdsc->mutex);dout ("forward tid %llu to mds%d - req dne\n" , tid, next_mds); - goto out;/* dup reply? */ +return ;/* dup reply? */ }if (test_bit (CEPH_MDS_R_ABORTED , &req->r_req_flags)) {dout ("forward tid %llu aborted, unregistering\n" , tid);__unregister_request (mdsc, req); }else if (fwd_seq <= req->r_num_fwd) { -dout ("forward tid %llu to mds%d - old seq %d <= %d\n" , - tid, next_mds, req->r_num_fwd, fwd_seq);
fs/ceph/mds_client.c static void handle_forward(struct ceph_mds_client *mdsc, · line 3323
Quoted 10 lines
fs/ceph/mds_client.c +10
-0+/* +* The type of 'num_fwd' in ceph 'MClientRequestForward' +* is 'int32_t', while in 'ceph_mds_request_head' the +* type is '__u8'. So in case the request bounces between +* MDSes exceeding 256 times, the client will get stuck. +* +* In this case it's ususally a bug in MDS and continue +* bouncing the request makes no sense. +*/ +if (req->r_num_fwd ==256 ) {
Can you also fix this to be expressed as "> UCHAR_MAX"? Or preferably, if you have a way to get to the ceph_mds_request_head from here, then express it in terms of sizeof() the field that limits this.
Quoted 19 lines
fs/ceph/mds_client.c +11
-1+mutex_lock (&req->r_fill_mutex); + req->r_err = -EMULTIHOP ; +set_bit (CEPH_MDS_R_ABORTED , &req->r_req_flags); +mutex_unlock (&req->r_fill_mutex); + aborted = true; +pr_warn_ratelimited ("forward tid %llu seq overflow\n" , + tid); + }else { +dout ("forward tid %llu to mds%d - old seq %d <= %d\n" , + tid, next_mds, req->r_num_fwd, fwd_seq); + } }else {/* resend. forward race not possible; mds would drop */ dout ("forward tid %llu to mds%d (we resend)\n" , tid, next_mds);@@ -3322,9 +3343,12 @@ static void handle_forward(struct ceph_mds_client *mdsc,put_request_session (req);__do_request (mdsc, req); } -ceph_mdsc_put_request (req);
fs/ceph/mds_client.c static void handle_forward(struct ceph_mds_client *mdsc, · line 3354
Quoted 10 lines
fs/ceph/mds_client.c +5
-1-out:mutex_unlock (&mdsc->mutex); + +/* kick calling process */ +if (aborted) +complete_request (mdsc, req); +ceph_mdsc_put_request (req);return ; bad:
The code looks fine otherwise though...
Signature · 1 line
Jeff Layton <[email protected]>
Quoted 56 lines · Jeff Layton
Quoted 55 lines · On Tue, 2022-03-29 at 20:20 +0800, [email protected] wrote:
From: Xiubo Li <[email protected]>
The type of '
num_fwd' in ceph 'MClientRequestForward' is 'int32_t', while in 'ceph_mds_request_head' the type is '__u8'. So in case the request bounces between MDSes exceeding 256 times, the client will get stuck.In this case it's ususally a bug in MDS and continue bouncing the request makes no sense.
Signed-off-by Xiubo Li <[email protected]>---
V2:
- s/EIO/EMULTIHOP/
- Fixed dereferencing NULL seq bug
- Removed the out lablefs/ceph/mds_client.c | 34 +++++++++++++++++++++++++++++----- 1 file changed, 29 insertions(+), 5 deletions(-)fs/ceph/mds_client.c +8
-3@@ -3293,6 +3293,7 @@ static void handle_forward(struct ceph_mds_client *mdsc,int err = -EINVAL ;void *p = msg->front.iov_base;void *end = p + msg->front.iov_len; +bool aborted = false;ceph_decode_need (&p, end,2 *sizeof (u32), bad); next_mds =ceph_decode_32 (&p);@@ -3301,16 +3302,36 @@ static void handle_forward(struct ceph_mds_client *mdsc,mutex_lock (&mdsc->mutex); req =lookup_get_request (mdsc, tid);if (!req) { +mutex_unlock (&mdsc->mutex);dout ("forward tid %llu to mds%d - req dne\n" , tid, next_mds); - goto out;/* dup reply? */ +return ;/* dup reply? */ }if (test_bit (CEPH_MDS_R_ABORTED , &req->r_req_flags)) {dout ("forward tid %llu aborted, unregistering\n" , tid);__unregister_request (mdsc, req); }else if (fwd_seq <= req->r_num_fwd) { -dout ("forward tid %llu to mds%d - old seq %d <= %d\n" , - tid, next_mds, req->r_num_fwd, fwd_seq); +/* +* The type of 'num_fwd' in ceph 'MClientRequestForward' +* is 'int32_t', while in 'ceph_mds_request_head' the +* type is '__u8'. So in case the request bounces between +* MDSes exceeding 256 times, the client will get stuck.
Quoted 8 lines
fs/ceph/mds_client.c static void handle_forward(struct ceph_mds_client *mdsc, · line 3323
Quoted 5 lines
fs/ceph/mds_client.c +5
-0+* +* In this case it's ususally a bug in MDS and continue +* bouncing the request makes no sense. +*/ +if (req->r_num_fwd ==256 ) {Can you also fix this to be expressed as ">
UCHAR_MAX"? Or preferably, if you have a way to get to theceph_mds_request_headfrom here, then express it in terms ofsizeof()the field that limits this.
Sure, let me try the second one first.
-- Xiubo
Quoted 30 lines
Quoted 19 lines
fs/ceph/mds_client.c +11
-1+mutex_lock (&req->r_fill_mutex); + req->r_err = -EMULTIHOP ; +set_bit (CEPH_MDS_R_ABORTED , &req->r_req_flags); +mutex_unlock (&req->r_fill_mutex); + aborted = true; +pr_warn_ratelimited ("forward tid %llu seq overflow\n" , + tid); + }else { +dout ("forward tid %llu to mds%d - old seq %d <= %d\n" , + tid, next_mds, req->r_num_fwd, fwd_seq); + } }else {/* resend. forward race not possible; mds would drop */ dout ("forward tid %llu to mds%d (we resend)\n" , tid, next_mds);@@ -3322,9 +3343,12 @@ static void handle_forward(struct ceph_mds_client *mdsc,put_request_session (req);__do_request (mdsc, req); } -ceph_mdsc_put_request (req);fs/ceph/mds_client.c static void handle_forward(struct ceph_mds_client *mdsc, · line 3354
Quoted 10 lines
fs/ceph/mds_client.c +5
-1-out:mutex_unlock (&mdsc->mutex); + +/* kick calling process */ +if (aborted) +complete_request (mdsc, req); +ceph_mdsc_put_request (req);return ; bad:The code looks fine otherwise though...