Skip to contentSkip to conversation

nvmet-auth: complete a request only after freeing the dhchap pointers

PATCH Superseded Maurizio Lombardi 2 participants 3 messages Oct 13, 2023 – Oct 16, 2023

Subsystems (3)

v1 of 2 Superseded Latest is v2 →

lore
Share & tools
lore
#0 lore raw

PATCH 1/1 nvmet-auth: complete a request only after freeing the dhchap pointers

It may happen that the work to destroy a queue
(for example nvmet_tcp_release_queue_work()) is started while an auth-send or auth-receive command is still completing.

nvmet_sq_destroy() will block, waiting for all the references to the sq to be dropped, the last reference is then dropped when nvmet_req_complete() is called.

When this happens, both nvmet_sq_destroy() and nvmet_execute_auth_send()/_receive() will free the dhchap pointers by calling nvmet_auth_sq_free().
Since there isn't any lock, the two threads may race against each other, causing double frees and memory corruptions, as reported by KASAN.

Reproduced by stress blktests nvme/041 nvme/042 nvme/043

nvme nvme2: qid 0: authenticated with hash hmac(sha512) dhgroup ffdhe4096
==================================================================

 BUG: KASAN: double-free in kfree+0xec/0x4b0

 Call Trace:
  <TASK>
  kfree+0xec/0x4b0
  nvmet_auth_sq_free+0xe1/0x160 [nvmet]
  nvmet_execute_auth_send+0x482/0x16d0 [nvmet]
  process_one_work+0x8e5/0x1510

 Allocated by task 191846:
  __kasan_kmalloc+0x81/0xa0
  nvmet_auth_ctrl_sesskey+0xf6/0x380 [nvmet]
  nvmet_auth_reply+0x119/0x990 [nvmet]

 Freed by task 143270:
  kfree+0xec/0x4b0
  nvmet_auth_sq_free+0xe1/0x160 [nvmet]
  process_one_work+0x8e5/0x1510

Fix this bug by calling nvmet_req_complete() only after freeing the pointers, so we will prevent the race by holding the sq reference.

Fixes db1312dd9548 ("nvmet: implement basic In-Band Authentication")
---
 drivers/nvme/target/fabrics-cmd-auth.c | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)
drivers/nvme/target/fabrics-cmd-auth.c +7 -2
index 586458f765f1..7385b18a491c 100644
@@ -333,19 +333,21 @@ void nvmet_execute_auth_send(struct nvmet_req *req)
 			 __func__, ctrl->cntlid, req->sq->qid,
 			 status, req->error_loc);
 	req->cqe->result.u64 = 0;
-	nvmet_req_complete(req, status);
 	if (req->sq->dhchap_step != NVME_AUTH_DHCHAP_MESSAGE_SUCCESS2 &&
 	    req->sq->dhchap_step != NVME_AUTH_DHCHAP_MESSAGE_FAILURE2) {
 		unsigned long auth_expire_secs = ctrl->kato ? ctrl->kato : 120;
 
 		mod_delayed_work(system_wq, &req->sq->auth_expired_work,
 				 auth_expire_secs * HZ);
+
+		nvmet_req_complete(req, status);
 		return;
 	}
 	/* Final states, clear up variables */
 	nvmet_auth_sq_free(req->sq);
 	if (req->sq->dhchap_step == NVME_AUTH_DHCHAP_MESSAGE_FAILURE2)
 		nvmet_ctrl_fatal_error(ctrl);
+	nvmet_req_complete(req, status);
 }
 
 static int nvmet_auth_challenge(struct nvmet_req *req, void *d, int al)
@@ -514,11 +516,14 @@ void nvmet_execute_auth_receive(struct nvmet_req *req)
 	kfree(d);
 done:
 	req->cqe->result.u64 = 0;
-	nvmet_req_complete(req, status);
+
 	if (req->sq->dhchap_step == NVME_AUTH_DHCHAP_MESSAGE_SUCCESS2)
 		nvmet_auth_sq_free(req->sq);
 	else if (req->sq->dhchap_step == NVME_AUTH_DHCHAP_MESSAGE_FAILURE1) {
 		nvmet_auth_sq_free(req->sq);
 		nvmet_ctrl_fatal_error(ctrl);
+		nvmet_req_complete(req, status);
+		return;
 	}
+	nvmet_req_complete(req, status);
 }
Signature · 1 line
2.39.3
#1 lore raw
Quoted 6 lines · Maurizio Lombardi
drivers/nvme/target/fabrics-cmd-auth.c +0 -1
-	nvmet_req_complete(req, status);
 	if (req->sq->dhchap_step != NVME_AUTH_DHCHAP_MESSAGE_SUCCESS2 &&
 	    req->sq->dhchap_step != NVME_AUTH_DHCHAP_MESSAGE_FAILURE2) {
 		unsigned long auth_expire_secs = ctrl->kato ? ctrl->kato : 120;
 
 		mod_delayed_work(system_wq, &req->sq->auth_expired_work,

drivers/nvme/target/fabrics-cmd-auth.c void nvmet_execute_auth_send(struct nvmet_req *req) · line 350

Quoted 10 lines
drivers/nvme/target/fabrics-cmd-auth.c +3 -0
 				 auth_expire_secs * HZ);
+
+		nvmet_req_complete(req, status);
 		return;
 	}
 	/* Final states, clear up variables */
 	nvmet_auth_sq_free(req->sq);
 	if (req->sq->dhchap_step == NVME_AUTH_DHCHAP_MESSAGE_FAILURE2)
 		nvmet_ctrl_fatal_error(ctrl);
+	nvmet_req_complete(req, status);

I'd be tempted to use a goto here to have a single completion path.

Quoted 8 lines

}

static int nvmet_auth_challenge(struct nvmet_req *req, void *d, int al)

drivers/nvme/target/fabrics-cmd-auth.c +0 -1
@@ -514,11 +516,14 @@ void nvmet_execute_auth_receive(struct nvmet_req *req)
 	kfree(d);
 done:
 	req->cqe->result.u64 = 0;
-	nvmet_req_complete(req, status);

drivers/nvme/target/fabrics-cmd-auth.c void nvmet_execute_auth_receive(struct nvmet_req *req) · line 528

Quoted 10 lines
drivers/nvme/target/fabrics-cmd-auth.c +4 -0
+
 	if (req->sq->dhchap_step == NVME_AUTH_DHCHAP_MESSAGE_SUCCESS2)
 		nvmet_auth_sq_free(req->sq);
 	else if (req->sq->dhchap_step == NVME_AUTH_DHCHAP_MESSAGE_FAILURE1) {
 		nvmet_auth_sq_free(req->sq);
 		nvmet_ctrl_fatal_error(ctrl);
+		nvmet_req_complete(req, status);
+		return;
 	}
+	nvmet_req_complete(req, status);

And this looks really odd to me. Why do a manual complete and return in the branch just to skip the complete right after it?

#2 lore raw

po 16. 10. 2023 v 7:57 odesílatel Christoph Hellwig <[email protected]> napsal:

Quoted 19 lines · Maurizio Lombardi
Quoted 6 lines
drivers/nvme/target/fabrics-cmd-auth.c +0 -1
-     nvmet_req_complete(req, status);
      if (req->sq->dhchap_step != NVME_AUTH_DHCHAP_MESSAGE_SUCCESS2 &&
          req->sq->dhchap_step != NVME_AUTH_DHCHAP_MESSAGE_FAILURE2) {
              unsigned long auth_expire_secs = ctrl->kato ? ctrl->kato : 120;
 
              mod_delayed_work(system_wq, &req->sq->auth_expired_work,

drivers/nvme/target/fabrics-cmd-auth.c void nvmet_execute_auth_send(struct nvmet_req *req) · line 350

Quoted 10 lines
drivers/nvme/target/fabrics-cmd-auth.c +3 -0
                               auth_expire_secs * HZ);
+
+             nvmet_req_complete(req, status);
              return;
      }
      /* Final states, clear up variables */
      nvmet_auth_sq_free(req->sq);
      if (req->sq->dhchap_step == NVME_AUTH_DHCHAP_MESSAGE_FAILURE2)
              nvmet_ctrl_fatal_error(ctrl);
+     nvmet_req_complete(req, status);
Quoted 1 line

I'd be tempted to use a goto here to have a single completion path.

Ok I will modify it.

Quoted 14 lines
Quoted 14 lines

}

static int nvmet_auth_challenge(struct nvmet_req *req, void *d, int al)

drivers/nvme/target/fabrics-cmd-auth.c +1 -1
@@ -514,11 +516,14 @@ void nvmet_execute_auth_receive(struct nvmet_req *req)
      kfree(d);
 done:
      req->cqe->result.u64 = 0;
-     nvmet_req_complete(req, status);
+
      if (req->sq->dhchap_step == NVME_AUTH_DHCHAP_MESSAGE_SUCCESS2)
              nvmet_auth_sq_free(req->sq);
      else if (req->sq->dhchap_step == NVME_AUTH_DHCHAP_MESSAGE_FAILURE1) {
              nvmet_auth_sq_free(req->sq);
Quoted 9 lines

drivers/nvme/target/fabrics-cmd-auth.c void nvmet_execute_auth_receive(struct nvmet_req *req) · line 528

Quoted 5 lines
drivers/nvme/target/fabrics-cmd-auth.c +3 -0
              nvmet_ctrl_fatal_error(ctrl);
+             nvmet_req_complete(req, status);
+             return;
      }
+     nvmet_req_complete(req, status);

And this looks really odd to me. Why do a manual complete and return in the branch just to skip the complete right after it?

You are right, it's a leftover of previous experiments and I didn't realize I could
simply remove this redundant code.
will submit a V2.

thanks,
Maurizio