nvmet-auth: complete a request only after freeing the dhchap pointers
Subsystems (3)
PATCH 1/1 nvmet-auth: complete a request only after freeing the dhchap pointers
It may happen that the work to destroy a queue
(for example nvmet_tcp_release_queue_work()) is started while an auth-send or auth-receive command is still completing.
nvmet_sq_destroy() will block, waiting for all the references to the sq to be dropped, the last reference is then dropped when nvmet_req_complete() is called.
When this happens, both nvmet_sq_destroy() and nvmet_execute_auth_send()/_receive() will free the dhchap pointers by calling nvmet_auth_sq_free().
Since there isn't any lock, the two threads may race against each other, causing double frees and memory corruptions, as reported by KASAN.
Reproduced by stress blktests nvme/041 nvme/042 nvme/043
nvme nvme2: qid 0: authenticated with hash hmac(sha512) dhgroup ffdhe4096
==================================================================
BUG: KASAN: double-free in kfree+0xec/0x4b0 Call Trace: <TASK> kfree+0xec/0x4b0 nvmet_auth_sq_free+0xe1/0x160 [nvmet] nvmet_execute_auth_send+0x482/0x16d0 [nvmet] process_one_work+0x8e5/0x1510 Allocated by task 191846: __kasan_kmalloc+0x81/0xa0 nvmet_auth_ctrl_sesskey+0xf6/0x380 [nvmet] nvmet_auth_reply+0x119/0x990 [nvmet] Freed by task 143270: kfree+0xec/0x4b0 nvmet_auth_sq_free+0xe1/0x160 [nvmet] process_one_work+0x8e5/0x1510
Fix this bug by calling nvmet_req_complete() only after freeing the pointers, so we will prevent the race by holding the sq reference.
--- drivers/nvme/target/fabrics-cmd-auth.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-)
drivers/nvme/target/fabrics-cmd-auth.c +7 -2
@@ -333,19 +333,21 @@ void nvmet_execute_auth_send(struct nvmet_req *req) __func__, ctrl->cntlid, req->sq->qid, status, req->error_loc); req->cqe->result.u64 =0 ; -nvmet_req_complete (req, status);if (req->sq->dhchap_step !=NVME_AUTH_DHCHAP_MESSAGE_SUCCESS2 && req->sq->dhchap_step !=NVME_AUTH_DHCHAP_MESSAGE_FAILURE2 ) {unsigned long auth_expire_secs = ctrl->kato ? ctrl->kato :120 ;mod_delayed_work (system_wq, &req->sq->auth_expired_work, auth_expire_secs *HZ ); + +nvmet_req_complete (req, status);return ; }/* Final states, clear up variables */ nvmet_auth_sq_free (req->sq);if (req->sq->dhchap_step ==NVME_AUTH_DHCHAP_MESSAGE_FAILURE2 )nvmet_ctrl_fatal_error (ctrl); +nvmet_req_complete (req, status); }static int nvmet_auth_challenge (struct nvmet_req *req,void *d,int al)
@@ -514,11 +516,14 @@ void nvmet_execute_auth_receive(struct nvmet_req *req)kfree (d); done: req->cqe->result.u64 =0 ; -nvmet_req_complete (req, status); +if (req->sq->dhchap_step ==NVME_AUTH_DHCHAP_MESSAGE_SUCCESS2 )nvmet_auth_sq_free (req->sq);else if (req->sq->dhchap_step ==NVME_AUTH_DHCHAP_MESSAGE_FAILURE1 ) {nvmet_auth_sq_free (req->sq);nvmet_ctrl_fatal_error (ctrl); +nvmet_req_complete (req, status); +return ; } +nvmet_req_complete (req, status); }
Signature · 1 line
2.39.3
Quoted 6 lines · Maurizio Lombardi
drivers/nvme/target/fabrics-cmd-auth.c +0
-1-nvmet_req_complete (req, status);if (req->sq->dhchap_step !=NVME_AUTH_DHCHAP_MESSAGE_SUCCESS2 && req->sq->dhchap_step !=NVME_AUTH_DHCHAP_MESSAGE_FAILURE2 ) {unsigned long auth_expire_secs = ctrl->kato ? ctrl->kato :120 ; mod_delayed_work(system_wq, &req->sq->auth_expired_work,
drivers/nvme/target/fabrics-cmd-auth.c void nvmet_execute_auth_send(struct nvmet_req *req) · line 350
Quoted 10 lines
drivers/nvme/target/fabrics-cmd-auth.c +3
-0auth_expire_secs * HZ); + +nvmet_req_complete (req, status);return ; }/* Final states, clear up variables */ nvmet_auth_sq_free (req->sq);if (req->sq->dhchap_step ==NVME_AUTH_DHCHAP_MESSAGE_FAILURE2 )nvmet_ctrl_fatal_error (ctrl); +nvmet_req_complete (req, status);
I'd be tempted to use a goto here to have a single completion path.
Quoted 8 lines
}
static int
nvmet_auth_challenge(struct nvmet_req *req, void *d, int al)drivers/nvme/target/fabrics-cmd-auth.c +0
-1@@ -514,11 +516,14 @@ void nvmet_execute_auth_receive(struct nvmet_req *req)kfree (d); done: req->cqe->result.u64 =0 ; -nvmet_req_complete (req, status);
drivers/nvme/target/fabrics-cmd-auth.c void nvmet_execute_auth_receive(struct nvmet_req *req) · line 528
Quoted 10 lines
drivers/nvme/target/fabrics-cmd-auth.c +4
-0+if (req->sq->dhchap_step ==NVME_AUTH_DHCHAP_MESSAGE_SUCCESS2 )nvmet_auth_sq_free (req->sq);else if (req->sq->dhchap_step ==NVME_AUTH_DHCHAP_MESSAGE_FAILURE1 ) {nvmet_auth_sq_free (req->sq);nvmet_ctrl_fatal_error (ctrl); +nvmet_req_complete (req, status); +return ; } +nvmet_req_complete (req, status);
And this looks really odd to me. Why do a manual complete and return in the branch just to skip the complete right after it?
po 16. 10. 2023 v 7:57 odesílatel Christoph Hellwig <[email protected]> napsal:
Quoted 19 lines · Maurizio Lombardi
Quoted 6 lines
drivers/nvme/target/fabrics-cmd-auth.c +0
-1-nvmet_req_complete (req, status);if (req->sq->dhchap_step !=NVME_AUTH_DHCHAP_MESSAGE_SUCCESS2 && req->sq->dhchap_step !=NVME_AUTH_DHCHAP_MESSAGE_FAILURE2 ) {unsigned long auth_expire_secs = ctrl->kato ? ctrl->kato :120 ; mod_delayed_work(system_wq, &req->sq->auth_expired_work,drivers/nvme/target/fabrics-cmd-auth.c void nvmet_execute_auth_send(struct nvmet_req *req) · line 350
Quoted 10 lines
drivers/nvme/target/fabrics-cmd-auth.c +3
-0auth_expire_secs * HZ); + +nvmet_req_complete (req, status);return ; }/* Final states, clear up variables */ nvmet_auth_sq_free (req->sq);if (req->sq->dhchap_step ==NVME_AUTH_DHCHAP_MESSAGE_FAILURE2 )nvmet_ctrl_fatal_error (ctrl); +nvmet_req_complete (req, status);
Quoted 1 line
I'd be tempted to use a goto here to have a single completion path.
Ok I will modify it.
Quoted 14 lines
Quoted 14 lines
}
static int
nvmet_auth_challenge(struct nvmet_req *req, void *d, int al)drivers/nvme/target/fabrics-cmd-auth.c +1
-1@@ -514,11 +516,14 @@ void nvmet_execute_auth_receive(struct nvmet_req *req)kfree (d); done: req->cqe->result.u64 =0 ; -nvmet_req_complete (req, status); +if (req->sq->dhchap_step ==NVME_AUTH_DHCHAP_MESSAGE_SUCCESS2 )nvmet_auth_sq_free (req->sq);else if (req->sq->dhchap_step ==NVME_AUTH_DHCHAP_MESSAGE_FAILURE1 ) {nvmet_auth_sq_free (req->sq);
Quoted 9 lines
drivers/nvme/target/fabrics-cmd-auth.c void nvmet_execute_auth_receive(struct nvmet_req *req) · line 528
Quoted 5 lines
drivers/nvme/target/fabrics-cmd-auth.c +3
-0nvmet_ctrl_fatal_error (ctrl); +nvmet_req_complete (req, status); +return ; } +nvmet_req_complete (req, status);And this looks really odd to me. Why do a manual complete and return in the branch just to skip the complete right after it?
You are right, it's a leftover of previous experiments and I didn't realize I could
simply remove this redundant code.
will submit a V2.
thanks,
Maurizio