Skip to contentSkip to conversation

nvmet: Avoid writing fabric_ops, queue pointers on every request.

PATCH New Parav Pandit 3 participants 13 messages Feb 7, 2017 – Feb 15, 2017

Subsystems (3)
lore
Share & tools
lore
#0 lore raw

PATCH 1/1 nvmet: Avoid writing fabric_ops, queue pointers on every request.

Fabric operations are constants of a registered transport. They don't change with every target request that gets processed by the nvmet-core. Therefore this patch moves fabrics_ops initialization out of the hot request processing path for rdma and fc.
It continues to remain in same way for loop target through extra API.

Additionally this patch further avoid nvme cq and sq pointer initialization for every request during every request processing for rdma because nvme queue linking occurs during queue allocation time for AQ and IOQ.

As a result, in nvmet_req_init function, this change saves initialization of 24 bytes constants on every datapath request and also reduces function size by 32 bytes for rdma transport.

Reviewed-by Max Gurtovoy <maxg at mellanox.com>
Signed-off-by Parav Pandit <parav at mellanox.com>
---
 drivers/nvme/target/core.c  |  8 ++------
 drivers/nvme/target/fc.c    | 11 +++++------
 drivers/nvme/target/loop.c  | 15 +++++++++++----
 drivers/nvme/target/nvmet.h | 16 ++++++++++++++--
 drivers/nvme/target/rdma.c  |  7 +++++--
 5 files changed, 37 insertions(+), 20 deletions(-)
drivers/nvme/target/core.c +2 -6
index b1d66ed..e55d650 100644
@@ -463,15 +463,11 @@ int nvmet_sq_init(struct nvmet_sq *sq)
 }
 EXPORT_SYMBOL_GPL(nvmet_sq_init);
 
-bool nvmet_req_init(struct nvmet_req *req, struct nvmet_cq *cq,
-		struct nvmet_sq *sq, struct nvmet_fabrics_ops *ops)
+bool nvmet_req_init(struct nvmet_req *req)
 {
 	u8 flags = req->cmd->common.flags;
 	u16 status;
 
-	req->cq = cq;
-	req->sq = sq;
-	req->ops = ops;
 	req->sg = NULL;
 	req->sg_cnt = 0;
 	req->rsp->status = 0;
@@ -504,7 +500,7 @@ bool nvmet_req_init(struct nvmet_req *req, struct nvmet_cq *cq,
 	if (status)
 		goto fail;
 
-	if (unlikely(!percpu_ref_tryget_live(&sq->ref))) {
+	if (unlikely(!percpu_ref_tryget_live(&req->sq->ref))) {
 		status = NVME_SC_INVALID_FIELD | NVME_SC_DNR;
 		goto fail;
 	}
drivers/nvme/target/fc.c +5 -6
index 173e842..8b201c0 100644
@@ -30,7 +30,6 @@
 
 /* *************************** Data Structures/Defines ****************** */
 
-
 #define NVMET_LS_CTX_COUNT		4
 
 /* for this implementation, assume small single frame rqst/rsp */
@@ -210,6 +209,7 @@ struct nvmet_fc_tgt_assoc {
 static LIST_HEAD(nvmet_fc_target_list);
 static DEFINE_IDA(nvmet_fc_tgtport_cnt);
 
+static struct nvmet_fabrics_ops nvmet_fc_tgt_fcp_ops;
 
 static void nvmet_fc_handle_ls_rqst_work(struct work_struct *work);
 static void nvmet_fc_handle_fcp_rqst_work(struct work_struct *work);
@@ -417,6 +417,7 @@ struct nvmet_fc_tgt_assoc {
 		fod->tgtport = tgtport;
 		fod->queue = queue;
 		fod->active = false;
+		nvmet_req_fabric_ops_init(&fod->req, &nvmet_fc_tgt_fcp_ops);
 		list_add_tail(&fod->fcp_list, &queue->fod_list);
 		spin_lock_init(&fod->flock);
 
@@ -1403,8 +1404,6 @@ enum {
 
 static void nvmet_fc_fcp_nvme_cmd_done(struct nvmet_req *nvme_req);
 
-static struct nvmet_fabrics_ops nvmet_fc_tgt_fcp_ops;
-
 static void
 nvmet_fc_xmt_ls_rsp_done(struct nvmefc_tgt_ls_req *lsreq)
 {
@@ -2008,10 +2007,10 @@ enum {
 	/* clear any response payload */
 	memset(&fod->rspiubuf, 0, sizeof(fod->rspiubuf));
 
-	ret = nvmet_req_init(&fod->req,
+	nvmet_req_link_to_nvme_queues(&fod->req,
 				&fod->queue->nvme_cq,
-				&fod->queue->nvme_sq,
-				&nvmet_fc_tgt_fcp_ops);
+				&fod->queue->nvme_sq);
+	ret = nvmet_req_init(&fod->req);
 	if (!ret) {	/* bad SQE content */
 		nvmet_fc_abort_op(tgtport, fod->fcpreq);
 		return;
drivers/nvme/target/loop.c +11 -4
index 9aaa700..2f7ca34 100644
@@ -174,8 +174,12 @@ static int nvme_loop_queue_rq(struct blk_mq_hw_ctx *hctx,
 
 	iod->cmd.common.flags |= NVME_CMD_SGL_METABUF;
 	iod->req.port = nvmet_loop_port;
-	if (!nvmet_req_init(&iod->req, &queue->nvme_cq,
-			&queue->nvme_sq, &nvme_loop_ops)) {
+
+	nvmet_req_link_to_nvme_queues(&iod->req,
+				&queue->nvme_cq, &queue->nvme_sq);
+	nvmet_req_fabric_ops_init(&iod->req, &nvme_loop_ops);
+
+	if (!nvmet_req_init(&iod->req)) {
 		nvme_cleanup_cmd(req);
 		blk_mq_start_request(req);
 		nvme_loop_queue_response(&iod->req);
@@ -211,8 +215,11 @@ static void nvme_loop_submit_async_event(struct nvme_ctrl *arg, int aer_idx)
 	iod->cmd.common.command_id = NVME_LOOP_AQ_BLKMQ_DEPTH;
 	iod->cmd.common.flags |= NVME_CMD_SGL_METABUF;
 
-	if (!nvmet_req_init(&iod->req, &queue->nvme_cq, &queue->nvme_sq,
-			&nvme_loop_ops)) {
+	nvmet_req_link_to_nvme_queues(&iod->req,
+				&queue->nvme_cq, &queue->nvme_sq);
+	nvmet_req_fabric_ops_init(&iod->req, &nvme_loop_ops);
+
+	if (!nvmet_req_init(&iod->req)) {
 		dev_err(ctrl->ctrl.device, "failed async event work\n");
 		return;
 	}
drivers/nvme/target/nvmet.h +14 -2
index 23d5eb1..48486c2 100644
@@ -259,8 +259,20 @@ struct nvmet_async_event {
 int nvmet_parse_discovery_cmd(struct nvmet_req *req);
 int nvmet_parse_fabrics_cmd(struct nvmet_req *req);
 
-bool nvmet_req_init(struct nvmet_req *req, struct nvmet_cq *cq,
-		struct nvmet_sq *sq, struct nvmet_fabrics_ops *ops);
+static inline void nvmet_req_fabric_ops_init(struct nvmet_req *req,
+		struct nvmet_fabrics_ops *ops)
+{
+	req->ops = ops;
+}
+
+static inline void nvmet_req_link_to_nvme_queues(struct nvmet_req *req,
+	struct nvmet_cq *cq, struct nvmet_sq *sq)
+{
+	req->cq = cq;
+	req->sq = sq;
+}
+
+bool nvmet_req_init(struct nvmet_req *req);
 void nvmet_req_complete(struct nvmet_req *req, u16 status);
 
 void nvmet_cq_setup(struct nvmet_ctrl *ctrl, struct nvmet_cq *cq, u16 qid,
drivers/nvme/target/rdma.c +5 -2
index 8c3760a..1a57ab3 100644
@@ -402,6 +402,10 @@ static void nvmet_rdma_free_rsp(struct nvmet_rdma_device *ndev,
 		if (ret)
 			goto out_free;
 
+		nvmet_req_link_to_nvme_queues(&rsp->req,
+					&queue->nvme_cq, &queue->nvme_sq);
+		nvmet_req_fabric_ops_init(&rsp->req, &nvmet_rdma_ops);
+
 		list_add_tail(&rsp->free_list, &queue->free_rsps);
 	}
 
@@ -698,8 +702,7 @@ static void nvmet_rdma_handle_command(struct nvmet_rdma_queue *queue,
 	cmd->n_rdma = 0;
 	cmd->req.port = queue->port;
 
-	if (!nvmet_req_init(&cmd->req, &queue->nvme_cq,
-			&queue->nvme_sq, &nvmet_rdma_ops))
+	if (!nvmet_req_init(&cmd->req))
 		return;
 
 	status = nvmet_rdma_map_sgl(cmd);
Signature · 1 line
1.8.3.1
maintainer
#1 lore raw

PATCH 1/1 nvmet: Avoid writing fabric_ops, queue pointers on every request.

Quoted 5 lines · Parav Pandit

Fabric operations are constants of a registered transport. They don't change with every target request that gets processed by the nvmet-core. Therefore this patch moves fabrics_ops initialization out of the hot request processing path for rdma and fc.
It continues to remain in same way for loop target through extra API.

Can't you add it to nvme_loop_init_iod()?

Quoted 4 lines

Additionally this patch further avoid nvme cq and sq pointer initialization for every request during every request processing for rdma because nvme queue linking occurs during queue allocation time for AQ and IOQ.

This breaks SRQ mode where every nvmet_rdma_cmd serves different queues in it's lifetime..

#2 lore raw

PATCH 1/1 nvmet: Avoid writing fabric_ops, queue pointers on every request.

Hi Sagi,

Quoted 10 lines

-----Original Message-----
From: Sagi Grimberg [mailto:sagi at grimberg.me]
Sent: Wednesday, February 8, 2017 4:13 AM
To: Parav Pandit <parav at mellanox.com>; hch at lst.de; james.smart at broadcom.com; linux-nvme at lists.infradead.org
Subject: Re: [PATCH] nvmet: Avoid writing fabric_ops, queue pointers on every request.

Quoted 8 lines
Quoted 5 lines · Parav Pandit

Fabric operations are constants of a registered transport. They don't change with every target request that gets processed by the nvmet-core. Therefore this patch moves fabrics_ops initialization out of the hot request processing path for rdma and fc.
It continues to remain in same way for loop target through extra API.

Can't you add it to nvme_loop_init_iod()?

I didn't review that option when I first did it. I look it now and I believe it can be moved there.
I was under assumption that init_request() is done on every request processing and if that's the case we are not currently touching nvmet_loop_queue, so that minor overhead can be avoided by continue to do this in current location.

Quoted 8 lines
Quoted 5 lines

Additionally this patch further avoid nvme cq and sq pointer initialization for every request during every request processing for rdma because nvme queue linking occurs during queue allocation time for AQ and IOQ.

This breaks SRQ mode where every nvmet_rdma_cmd serves different queues in it's lifetime..

I fail to understand that.
nvmet_rdma_create_queue_ib() is call for as many QPs as we create; not based on number of SRQs we create. nvmet_rdma_queue stores cq and sq.
So there are as many cq and sq on the fabric side as QPs for fabric_connect command is called. queue is pulled out of cq context on which we received the command.
SRQ is just a place shared among this nvme queues to share the RQ buffer, right?
What did I miss?

maintainer
#3 lore raw

PATCH 1/1 nvmet: Avoid writing fabric_ops, queue pointers on every request.

Quoted 11 lines
Quoted 8 lines
Quoted 5 lines

Additionally this patch further avoid nvme cq and sq pointer initialization for every request during every request processing for rdma because nvme queue linking occurs during queue allocation time for AQ and IOQ.

This breaks SRQ mode where every nvmet_rdma_cmd serves different queues in it's lifetime..

I fail to understand that.
nvmet_rdma_create_queue_ib() is call for as many QPs as we create; not based on number of SRQs we create.

Correct.

Quoted 1 line

nvmet_rdma_queue stores cq and sq.

Correct.

Quoted 3 lines

So there are as many cq and sq on the fabric side as QPs for fabric_connect command is called. queue is pulled out of cq context on which we received the command.
SRQ is just a place shared among this nvme queues to share the RQ buffer, right?

Correct too, but we then assign the queue to the command, which is the context of the received SQE (maybe with in-capsule data). For the SRQ case we allocate the commands and pre-post them (before we have any queues), they are absolutely not bound to a given queue, they can't actually.

So for each new recv completion, the command context is now bound to the queue that it completed on, so it can be bound to different queues in its life-time.

#4 lore raw

PATCH 1/1 nvmet: Avoid writing fabric_ops, queue pointers on every request.

Hi Sagi,

Quoted 33 lines

-----Original Message-----
From: Sagi Grimberg [mailto:sagi at grimberg.me]
Sent: Wednesday, February 8, 2017 12:18 PM
To: Parav Pandit <parav at mellanox.com>; hch at lst.de; james.smart at broadcom.com; linux-nvme at lists.infradead.org
Subject: Re: [PATCH] nvmet: Avoid writing fabric_ops, queue pointers on every request.

Quoted 10 lines
Quoted 7 lines
Quoted 4 lines

Additionally this patch further avoid nvme cq and sq pointer initialization for every request during every request processing for rdma because nvme queue linking occurs during queue allocation time for AQ and IOQ.

This breaks SRQ mode where every nvmet_rdma_cmd serves different queues in it's lifetime..

I fail to understand that.
nvmet_rdma_create_queue_ib() is call for as many QPs as we create; not

based on number of SRQs we create.

Correct.

Quoted 1 line

nvmet_rdma_queue stores cq and sq.

Correct.

Quoted 1 line

So there are as many cq and sq on the fabric side as QPs for fabric_connect

command is called.

Quoted 2 lines

queue is pulled out of cq context on which we received the command. SRQ is just a place shared among this nvme queues to share the RQ buffer,

right?

Quoted 8 lines

Correct too, but we then assign the queue to the command, which is the context of the received SQE (maybe with in-capsule data). For the SRQ case we allocate the commands and pre-post them (before we have any queues), they are absolutely not bound to a given queue, they can't actually.

So for each new recv completion, the command context is now bound to the queue that it completed on, so it can be bound to different queues in its life- time.

Sorry, I am still not getting it.
nvmet_rdma_rsp structure contains nvmet_req. nvmet_rdma_rsp (and so nvmet_req) are per QP allocations. cq and sq pointers are initialized inside nvmet_req.

nvmet_rdma_cmd is per RQ/SRQ allocation.
When we do recv_done(), we bind rsp structure to cmd (cmd can be from RQ or SRQ). So I believe this is still good.
If we have cq and sq pointer inside the nvmet_rdma_cmd than I can understand that it can break.

On a side note: I tested the patch with use_srq flag (but didn't publish its performance numbers as they awaiting your per core SRQ fixes to match regular RQ numbers :-) ).

#5 lore raw

PATCH 1/1 nvmet: Avoid writing fabric_ops, queue pointers on every request.

In reply to Sagi Grimberg

Hi Sagi,

Quoted 57 lines

-----Original Message-----
From: Parav Pandit
Sent: Wednesday, February 8, 2017 2:03 PM
To: 'Sagi Grimberg' <sagi at grimberg.me>; hch at lst.de; james.smart at broadcom.com; linux-nvme at lists.infradead.org
Subject: RE: [PATCH] nvmet: Avoid writing fabric_ops, queue pointers on every request.

Quoted 25 lines
Quoted 11 lines
Quoted 7 lines
Quoted 4 lines

Additionally this patch further avoid nvme cq and sq pointer initialization for every request during every request processing for rdma because nvme queue linking occurs during queue allocation time for AQ and IOQ.

This breaks SRQ mode where every nvmet_rdma_cmd serves different queues in it's lifetime..

I fail to understand that.
nvmet_rdma_create_queue_ib() is call for as many QPs as we create; not

based on number of SRQs we create.

Correct.

Quoted 1 line

nvmet_rdma_queue stores cq and sq.

Correct.

Quoted 2 lines

So there are as many cq and sq on the fabric side as QPs for fabric_connect

command is called.

Quoted 3 lines

queue is pulled out of cq context on which we received the command. SRQ is just a place shared among this nvme queues to share the RQ buffer,

Quoted 6 lines

right?

Correct too, but we then assign the queue to the command, which is the context of the received SQE (maybe with in-capsule data). For the SRQ case we allocate the commands and pre-post them (before we have any queues), they are absolutely not bound to a given queue, they can't

actually.

Quoted 4 lines

So for each new recv completion, the command context is now bound to the queue that it completed on, so it can be bound to different queues in its life- time.

Sorry, I am still not getting it.
nvmet_rdma_rsp structure contains nvmet_req. nvmet_rdma_rsp (and so nvmet_req) are per QP allocations. cq and sq pointers are initialized inside nvmet_req.

nvmet_rdma_cmd is per RQ/SRQ allocation.
When we do recv_done(), we bind rsp structure to cmd (cmd can be from RQ or SRQ).
So I believe this is still good.
If we have cq and sq pointer inside the nvmet_rdma_cmd than I can understand that it can break.

Quoted 5 lines

On a side note: I tested the patch with use_srq flag (but didn't publish its performance numbers as they awaiting your per core SRQ fixes to match regular RQ numbers :-) ).

Did you get chance to review my comments? Can you please ack that this doesn't break the SRQ?

maintainer
#6 lore raw

PATCH 1/1 nvmet: Avoid writing fabric_ops, queue pointers on every request.

Quoted 2 lines

Hi Sagi,

Hi Parav,

Quoted 12 lines
Quoted 12 lines

Sorry, I am still not getting it.
nvmet_rdma_rsp structure contains nvmet_req. nvmet_rdma_rsp (and so nvmet_req) are per QP allocations. cq and sq pointers are initialized inside nvmet_req.

nvmet_rdma_cmd is per RQ/SRQ allocation.
When we do recv_done(), we bind rsp structure to cmd (cmd can be from RQ or SRQ).
So I believe this is still good.
If we have cq and sq pointer inside the nvmet_rdma_cmd than I can understand that it can break.

Quoted 3 lines
Quoted 3 lines

On a side note: I tested the patch with use_srq flag (but didn't publish its performance numbers as they awaiting your per core SRQ fixes to match regular RQ numbers :-) ).

Yep, forgot that the sq/cq are referenced from the rsp which is per-queue.

Christoph, James, can I get an ack from you guys?

#7 lore raw

PATCH 1/1 nvmet: Avoid writing fabric_ops, queue pointers on every request.

Quoted 4 lines · Sagi Grimberg

Yep, forgot that the sq/cq are referenced from the rsp which is per-queue.

Christoph, James, can I get an ack from you guys?

I really hate the nvmet_req_fabric_ops_init and nvmet_req_link_to_nvme_queues helpers, just use direct assignments instead of trivial helpers with crazy long names.

I need to look at it a bit more for the a real review.

maintainer
#8 lore raw

PATCH 1/1 nvmet: Avoid writing fabric_ops, queue pointers on every request.

Quoted 9 lines
Quoted 5 lines

Yep, forgot that the sq/cq are referenced from the rsp which is per-queue.

Christoph, James, can I get an ack from you guys?

I really hate the nvmet_req_fabric_ops_init and nvmet_req_link_to_nvme_queues helpers, just use direct assignments instead of trivial helpers with crazy long names.

Heh, I don't mind it all that much, but we can hold it off until something better (if at all) comes along..

#9 lore raw

PATCH 1/1 nvmet: Avoid writing fabric_ops, queue pointers on every request.

Quoted 15 lines

-----Original Message-----
From: Sagi Grimberg [mailto:sagi at grimberg.me]
Sent: Wednesday, February 15, 2017 10:29 AM
To: hch at lst.de
Cc: Parav Pandit <parav at mellanox.com>; james.smart at broadcom.com; linux-nvme at lists.infradead.org
Subject: Re: [PATCH] nvmet: Avoid writing fabric_ops, queue pointers on every request.

Quoted 3 lines
Quoted 3 lines

Yep, forgot that the sq/cq are referenced from the rsp which is per-queue.

Quoted 8 lines
Quoted 5 lines
Quoted 1 line

Christoph, James, can I get an ack from you guys?

I really hate the nvmet_req_fabric_ops_init and nvmet_req_link_to_nvme_queues helpers, just use direct assignments instead of trivial helpers with crazy long names.

Heh, I don't mind it all that much, but we can hold it off until something better (if at all) comes along..

How about
nvmet_ops_init()
nvmet_link_queues()

#10 lore raw

PATCH 1/1 nvmet: Avoid writing fabric_ops, queue pointers on every request.

Quoted 3 lines · Parav Pandit

How about
nvmet_ops_init()
nvmet_link_queues()

Just do the one respective two trivial assignment inline in the callers. That actually leads to smaller source and binary code.

#11 lore raw

PATCH 1/1 nvmet: Avoid writing fabric_ops, queue pointers on every request.

Quoted 11 lines

-----Original Message-----
From: hch at lst.de [mailto:hch at lst.de]
Sent: Wednesday, February 15, 2017 10:34 AM
To: Parav Pandit <parav at mellanox.com>
Cc: Sagi Grimberg <sagi at grimberg.me>; hch at lst.de; james.smart at broadcom.com; linux-nvme at lists.infradead.org
Subject: Re: [PATCH] nvmet: Avoid writing fabric_ops, queue pointers on every request.

Quoted 7 lines
Quoted 3 lines · Parav Pandit

How about
nvmet_ops_init()
nvmet_link_queues()

Just do the one respective two trivial assignment inline in the callers. That actually leads to smaller source and binary code.

o.k.

#12 lore raw

PATCH 1/1 nvmet: Avoid writing fabric_ops, queue pointers on every request.

In reply to Sagi Grimberg

Hi James,

Does this change look fine for FC?
I do not have infrastructure to test the FC side.

Parav

Quoted 73 lines

-----Original Message-----
From: Parav Pandit
Sent: Wednesday, February 8, 2017 2:03 PM
To: 'Sagi Grimberg' <sagi at grimberg.me>; hch at lst.de; james.smart at broadcom.com; linux-nvme at lists.infradead.org
Subject: RE: [PATCH] nvmet: Avoid writing fabric_ops, queue pointers on every request.

Hi Sagi,

Quoted 34 lines

-----Original Message-----
From: Sagi Grimberg [mailto:sagi at grimberg.me]
Sent: Wednesday, February 8, 2017 12:18 PM
To: Parav Pandit <parav at mellanox.com>; hch at lst.de; james.smart at broadcom.com; linux-nvme at lists.infradead.org
Subject: Re: [PATCH] nvmet: Avoid writing fabric_ops, queue pointers on every request.

Quoted 11 lines
Quoted 7 lines
Quoted 4 lines

Additionally this patch further avoid nvme cq and sq pointer initialization for every request during every request processing for rdma because nvme queue linking occurs during queue allocation time for AQ and IOQ.

This breaks SRQ mode where every nvmet_rdma_cmd serves different queues in it's lifetime..

I fail to understand that.
nvmet_rdma_create_queue_ib() is call for as many QPs as we create; not

based on number of SRQs we create.

Correct.

Quoted 1 line

nvmet_rdma_queue stores cq and sq.

Correct.

Quoted 2 lines

So there are as many cq and sq on the fabric side as QPs for fabric_connect

command is called.

Quoted 3 lines

queue is pulled out of cq context on which we received the command. SRQ is just a place shared among this nvme queues to share the RQ buffer,

Quoted 6 lines

right?

Correct too, but we then assign the queue to the command, which is the context of the received SQE (maybe with in-capsule data). For the SRQ case we allocate the commands and pre-post them (before we have any queues), they are absolutely not bound to a given queue, they can't

actually.

Quoted 4 lines

So for each new recv completion, the command context is now bound to the queue that it completed on, so it can be bound to different queues in its life- time.

Sorry, I am still not getting it.
nvmet_rdma_rsp structure contains nvmet_req. nvmet_rdma_rsp (and so nvmet_req) are per QP allocations. cq and sq pointers are initialized inside nvmet_req.

nvmet_rdma_cmd is per RQ/SRQ allocation.
When we do recv_done(), we bind rsp structure to cmd (cmd can be from RQ or SRQ).
So I believe this is still good.
If we have cq and sq pointer inside the nvmet_rdma_cmd than I can understand that it can break.

On a side note: I tested the patch with use_srq flag (but didn't publish its performance numbers as they awaiting your per core SRQ fixes to match regular RQ numbers :-) ).