nvmet: Avoid writing fabric_ops, queue pointers on every request.
Subsystems (3)
PATCH 1/1 nvmet: Avoid writing fabric_ops, queue pointers on every request.
Fabric operations are constants of a registered transport. They don't change with every target request that gets processed by the nvmet-core. Therefore this patch moves fabrics_ops initialization out of the hot request processing path for rdma and fc.
It continues to remain in same way for loop target through extra API.
Additionally this patch further avoid nvme cq and sq pointer initialization for every request during every request processing for rdma because nvme queue linking occurs during queue allocation time for AQ and IOQ.
As a result, in nvmet_req_init function, this change saves initialization of 24 bytes constants on every datapath request and also reduces function size by 32 bytes for rdma transport.
--- drivers/nvme/target/core.c | 8 ++------ drivers/nvme/target/fc.c | 11 +++++------ drivers/nvme/target/loop.c | 15 +++++++++++---- drivers/nvme/target/nvmet.h | 16 ++++++++++++++-- drivers/nvme/target/rdma.c | 7 +++++-- 5 files changed, 37 insertions(+), 20 deletions(-)
drivers/nvme/target/core.c +2 -6
@@ -463,15 +463,11 @@ int nvmet_sq_init(struct nvmet_sq *sq) }EXPORT_SYMBOL_GPL (nvmet_sq_init); -bool nvmet_req_init (struct nvmet_req *req,struct nvmet_cq *cq, -struct nvmet_sq *sq,struct nvmet_fabrics_ops *ops) +bool nvmet_req_init (struct nvmet_req *req) {u8 flags = req->cmd->common.flags;u16 status; - req->cq = cq; - req->sq = sq; - req->ops = ops; req->sg =NULL ; req->sg_cnt =0 ; req->rsp->status =0 ;
@@ -504,7 +500,7 @@ bool nvmet_req_init(struct nvmet_req *req, struct nvmet_cq *cq,if (status) goto fail; -if (unlikely (!percpu_ref_tryget_live (&sq->ref))) { +if (unlikely (!percpu_ref_tryget_live (&req->sq->ref))) { status =NVME_SC_INVALID_FIELD |NVME_SC_DNR ; goto fail; }
drivers/nvme/target/fc.c +5 -6
@@ -30,7 +30,6 @@/* *************************** Data Structures/Defines ****************** */ -#define NVMET_LS_CTX_COUNT 4/* for this implementation, assume small single frame rqst/rsp */
@@ -210,6 +209,7 @@ struct nvmet_fc_tgt_assoc {static LIST_HEAD (nvmet_fc_target_list);static DEFINE_IDA (nvmet_fc_tgtport_cnt); +static struct nvmet_fabrics_ops nvmet_fc_tgt_fcp_ops;static void nvmet_fc_handle_ls_rqst_work (struct work_struct *work);static void nvmet_fc_handle_fcp_rqst_work (struct work_struct *work);
@@ -417,6 +417,7 @@ struct nvmet_fc_tgt_assoc { fod->tgtport = tgtport; fod->queue = queue; fod->active = false; +nvmet_req_fabric_ops_init (&fod->req, &nvmet_fc_tgt_fcp_ops);list_add_tail (&fod->fcp_list, &queue->fod_list);spin_lock_init (&fod->flock);
@@ -1403,8 +1404,6 @@ enum {static void nvmet_fc_fcp_nvme_cmd_done (struct nvmet_req *nvme_req); -static struct nvmet_fabrics_ops nvmet_fc_tgt_fcp_ops; -static void nvmet_fc_xmt_ls_rsp_done (struct nvmefc_tgt_ls_req *lsreq) {
@@ -2008,10 +2007,10 @@ enum {/* clear any response payload */ memset (&fod->rspiubuf,0 ,sizeof (fod->rspiubuf)); - ret =nvmet_req_init (&fod->req, +nvmet_req_link_to_nvme_queues (&fod->req, &fod->queue->nvme_cq, - &fod->queue->nvme_sq, - &nvmet_fc_tgt_fcp_ops); + &fod->queue->nvme_sq); + ret =nvmet_req_init (&fod->req);if (!ret) {/* bad SQE content */ nvmet_fc_abort_op (tgtport, fod->fcpreq);return ;
drivers/nvme/target/loop.c +11 -4
@@ -174,8 +174,12 @@ static int nvme_loop_queue_rq(struct blk_mq_hw_ctx *hctx, iod->cmd.common.flags |=NVME_CMD_SGL_METABUF ; iod->req.port = nvmet_loop_port; -if (!nvmet_req_init (&iod->req, &queue->nvme_cq, - &queue->nvme_sq, &nvme_loop_ops)) { + +nvmet_req_link_to_nvme_queues (&iod->req, + &queue->nvme_cq, &queue->nvme_sq); +nvmet_req_fabric_ops_init (&iod->req, &nvme_loop_ops); + +if (!nvmet_req_init (&iod->req)) {nvme_cleanup_cmd (req);blk_mq_start_request (req);nvme_loop_queue_response (&iod->req);
@@ -211,8 +215,11 @@ static void nvme_loop_submit_async_event(struct nvme_ctrl *arg, int aer_idx) iod->cmd.common.command_id =NVME_LOOP_AQ_BLKMQ_DEPTH ; iod->cmd.common.flags |=NVME_CMD_SGL_METABUF ; -if (!nvmet_req_init (&iod->req, &queue->nvme_cq, &queue->nvme_sq, - &nvme_loop_ops)) { +nvmet_req_link_to_nvme_queues (&iod->req, + &queue->nvme_cq, &queue->nvme_sq); +nvmet_req_fabric_ops_init (&iod->req, &nvme_loop_ops); + +if (!nvmet_req_init (&iod->req)) {dev_err (ctrl->ctrl.device,"failed async event work\n" );return ; }
drivers/nvme/target/nvmet.h +14 -2
@@ -259,8 +259,20 @@ struct nvmet_async_event {int nvmet_parse_discovery_cmd (struct nvmet_req *req);int nvmet_parse_fabrics_cmd (struct nvmet_req *req); -bool nvmet_req_init (struct nvmet_req *req,struct nvmet_cq *cq, -struct nvmet_sq *sq,struct nvmet_fabrics_ops *ops); +static inline void nvmet_req_fabric_ops_init (struct nvmet_req *req, +struct nvmet_fabrics_ops *ops) +{ + req->ops = ops; +} + +static inline void nvmet_req_link_to_nvme_queues (struct nvmet_req *req, +struct nvmet_cq *cq,struct nvmet_sq *sq) +{ + req->cq = cq; + req->sq = sq; +} + +bool nvmet_req_init (struct nvmet_req *req);void nvmet_req_complete (struct nvmet_req *req,u16 status); void nvmet_cq_setup(struct nvmet_ctrl *ctrl, struct nvmet_cq *cq, u16 qid,
drivers/nvme/target/rdma.c +5 -2
@@ -402,6 +402,10 @@ static void nvmet_rdma_free_rsp(struct nvmet_rdma_device *ndev,if (ret) goto out_free; +nvmet_req_link_to_nvme_queues (&rsp->req, + &queue->nvme_cq, &queue->nvme_sq); +nvmet_req_fabric_ops_init (&rsp->req, &nvmet_rdma_ops); +list_add_tail (&rsp->free_list, &queue->free_rsps); }
@@ -698,8 +702,7 @@ static void nvmet_rdma_handle_command(struct nvmet_rdma_queue *queue, cmd->n_rdma =0 ; cmd->req.port = queue->port; -if (!nvmet_req_init (&cmd->req, &queue->nvme_cq, - &queue->nvme_sq, &nvmet_rdma_ops)) +if (!nvmet_req_init (&cmd->req))return ; status =nvmet_rdma_map_sgl (cmd);
Signature · 1 line
1.8.3.1
Sagi Grimberg
maintainerPATCH 1/1 nvmet: Avoid writing fabric_ops, queue pointers on every request.
Quoted 5 lines · Parav Pandit
Fabric operations are constants of a registered transport. They don't change with every target request that gets processed by the nvmet-core. Therefore this patch moves
fabrics_opsinitialization out of the hot request processing path for rdma and fc.
It continues to remain in same way for loop target through extra API.
Can't you add it to nvme_loop_init_iod()?
Quoted 4 lines
Additionally this patch further avoid nvme cq and sq pointer initialization for every request during every request processing for rdma because nvme queue linking occurs during queue allocation time for AQ and IOQ.
This breaks SRQ mode where every nvmet_rdma_cmd serves different queues in it's lifetime..
PATCH 1/1 nvmet: Avoid writing fabric_ops, queue pointers on every request.
Hi Sagi,
Quoted 10 lines
-----Original Message-----
From: Sagi Grimberg [mailto:sagi at grimberg.me]
Sent: Wednesday, February 8, 2017 4:13 AM
To: Parav Pandit <parav at mellanox.com>; hch at lst.de; james.smart at broadcom.com; linux-nvme at lists.infradead.org
Subject: Re: [PATCH] nvmet: Avoid writingfabric_ops, queue pointers on every request.
Quoted 8 lines
Quoted 5 lines · Parav Pandit
Fabric operations are constants of a registered transport. They don't change with every target request that gets processed by the nvmet-core. Therefore this patch moves
fabrics_opsinitialization out of the hot request processing path for rdma and fc.
It continues to remain in same way for loop target through extra API.Can't you add it to
nvme_loop_init_iod()?
I didn't review that option when I first did it. I look it now and I believe it can be moved there.
I was under assumption that init_request() is done on every request processing and if that's the case we are not currently touching nvmet_loop_queue, so that minor overhead can be avoided by continue to do this in current location.
Quoted 8 lines
Quoted 5 lines
Additionally this patch further avoid nvme cq and sq pointer initialization for every request during every request processing for rdma because nvme queue linking occurs during queue allocation time for AQ and IOQ.
This breaks SRQ mode where every
nvmet_rdma_cmdserves different queues in it's lifetime..
I fail to understand that.nvmet_rdma_create_queue_ib() is call for as many QPs as we create; not based on number of SRQs we create. nvmet_rdma_queue stores cq and sq.
So there are as many cq and sq on the fabric side as QPs for fabric_connect command is called. queue is pulled out of cq context on which we received the command.
SRQ is just a place shared among this nvme queues to share the RQ buffer, right?
What did I miss?
Sagi Grimberg
maintainerPATCH 1/1 nvmet: Avoid writing fabric_ops, queue pointers on every request.
Quoted 11 lines
Quoted 8 lines
Quoted 5 lines
Additionally this patch further avoid nvme cq and sq pointer initialization for every request during every request processing for rdma because nvme queue linking occurs during queue allocation time for AQ and IOQ.
This breaks SRQ mode where every
nvmet_rdma_cmdserves different queues in it's lifetime..I fail to understand that.
nvmet_rdma_create_queue_ib()is call for as many QPs as we create; not based on number of SRQs we create.
Correct.
Quoted 1 line
nvmet_rdma_queuestores cq and sq.
Correct.
Quoted 3 lines
So there are as many cq and sq on the fabric side as QPs for
fabric_connectcommand is called. queue is pulled out of cq context on which we received the command.
SRQ is just a place shared among this nvme queues to share the RQ buffer, right?
Correct too, but we then assign the queue to the command, which is the context of the received SQE (maybe with in-capsule data). For the SRQ case we allocate the commands and pre-post them (before we have any queues), they are absolutely not bound to a given queue, they can't actually.
So for each new recv completion, the command context is now bound to the queue that it completed on, so it can be bound to different queues in its life-time.
PATCH 1/1 nvmet: Avoid writing fabric_ops, queue pointers on every request.
Hi Sagi,
Quoted 33 lines
-----Original Message-----
From: Sagi Grimberg [mailto:sagi at grimberg.me]
Sent: Wednesday, February 8, 2017 12:18 PM
To: Parav Pandit <parav at mellanox.com>; hch at lst.de; james.smart at broadcom.com; linux-nvme at lists.infradead.org
Subject: Re: [PATCH] nvmet: Avoid writingfabric_ops, queue pointers on every request.Quoted 10 lines
Quoted 7 lines
Quoted 4 lines
Additionally this patch further avoid nvme cq and sq pointer initialization for every request during every request processing for rdma because nvme queue linking occurs during queue allocation time for AQ and IOQ.
This breaks SRQ mode where every
nvmet_rdma_cmdserves different queues in it's lifetime..I fail to understand that.
nvmet_rdma_create_queue_ib()is call for as many QPs as we create; notbased on number of SRQs we create.
Correct.
Quoted 1 line
nvmet_rdma_queuestores cq and sq.Correct.
Quoted 1 line
So there are as many cq and sq on the fabric side as QPs for
fabric_connectcommand is called.
Quoted 2 lines
queue is pulled out of cq context on which we received the command. SRQ is just a place shared among this nvme queues to share the RQ buffer,
right?
Quoted 8 lines
Correct too, but we then assign the queue to the command, which is the context of the received SQE (maybe with in-capsule data). For the SRQ case we allocate the commands and pre-post them (before we have any queues), they are absolutely not bound to a given queue, they can't actually.
So for each new recv completion, the command context is now bound to the queue that it completed on, so it can be bound to different queues in its life- time.
Sorry, I am still not getting it.nvmet_rdma_rsp structure contains nvmet_req. nvmet_rdma_rsp (and so nvmet_req) are per QP allocations. cq and sq pointers are initialized inside nvmet_req.
nvmet_rdma_cmd is per RQ/SRQ allocation.
When we do recv_done(), we bind rsp structure to cmd (cmd can be from RQ or SRQ). So I believe this is still good.
If we have cq and sq pointer inside the nvmet_rdma_cmd than I can understand that it can break.
On a side note: I tested the patch with use_srq flag (but didn't publish its performance numbers as they awaiting your per core SRQ fixes to match regular RQ numbers :-) ).
PATCH 1/1 nvmet: Avoid writing fabric_ops, queue pointers on every request.
In reply to Sagi GrimbergHi Sagi,
Quoted 57 lines
-----Original Message-----
From: Parav Pandit
Sent: Wednesday, February 8, 2017 2:03 PM
To: 'Sagi Grimberg' <sagi at grimberg.me>; hch at lst.de; james.smart at broadcom.com; linux-nvme at lists.infradead.org
Subject: RE: [PATCH] nvmet: Avoid writingfabric_ops, queue pointers on every request.Quoted 25 lines
Quoted 11 lines
Quoted 7 lines
Quoted 4 lines
Additionally this patch further avoid nvme cq and sq pointer initialization for every request during every request processing for rdma because nvme queue linking occurs during queue allocation time for AQ and IOQ.
This breaks SRQ mode where every
nvmet_rdma_cmdserves different queues in it's lifetime..I fail to understand that.
nvmet_rdma_create_queue_ib()is call for as many QPs as we create; notbased on number of SRQs we create.
Correct.
Quoted 1 line
nvmet_rdma_queuestores cq and sq.Correct.
Quoted 2 lines
So there are as many cq and sq on the fabric side as QPs for
fabric_connectcommand is called.
Quoted 3 lines
queue is pulled out of cq context on which we received the command. SRQ is just a place shared among this nvme queues to share the RQ buffer,
Quoted 6 lines
right?
Correct too, but we then assign the queue to the command, which is the context of the received SQE (maybe with in-capsule data). For the SRQ case we allocate the commands and pre-post them (before we have any queues), they are absolutely not bound to a given queue, they can't
actually.
Quoted 4 lines
So for each new recv completion, the command context is now bound to the queue that it completed on, so it can be bound to different queues in its life- time.
Sorry, I am still not getting it.
nvmet_rdma_rspstructure containsnvmet_req.nvmet_rdma_rsp(and sonvmet_req) are per QP allocations. cq and sq pointers are initialized insidenvmet_req.
nvmet_rdma_cmdis per RQ/SRQ allocation.
When we dorecv_done(), we bind rsp structure to cmd (cmd can be from RQ or SRQ).
So I believe this is still good.
If we have cq and sq pointer inside thenvmet_rdma_cmdthan I can understand that it can break.
Quoted 5 lines
On a side note: I tested the patch with
use_srqflag (but didn't publish its performance numbers as they awaiting your per core SRQ fixes to match regular RQ numbers :-) ).
Did you get chance to review my comments? Can you please ack that this doesn't break the SRQ?
Sagi Grimberg
maintainerPATCH 1/1 nvmet: Avoid writing fabric_ops, queue pointers on every request.
Quoted 2 lines
Hi Sagi,
Hi Parav,
Quoted 12 lines
Quoted 12 lines
Sorry, I am still not getting it.
nvmet_rdma_rspstructure containsnvmet_req.nvmet_rdma_rsp(and sonvmet_req) are per QP allocations. cq and sq pointers are initialized insidenvmet_req.
nvmet_rdma_cmdis per RQ/SRQ allocation.
When we dorecv_done(), we bind rsp structure to cmd (cmd can be from RQ or SRQ).
So I believe this is still good.
If we have cq and sq pointer inside thenvmet_rdma_cmdthan I can understand that it can break.
Quoted 3 lines
Quoted 3 lines
On a side note: I tested the patch with
use_srqflag (but didn't publish its performance numbers as they awaiting your per core SRQ fixes to match regular RQ numbers :-) ).
Yep, forgot that the sq/cq are referenced from the rsp which is per-queue.
Christoph, James, can I get an ack from you guys?
[email protected]
maintainerPATCH 1/1 nvmet: Avoid writing fabric_ops, queue pointers on every request.
Quoted 4 lines · Sagi Grimberg
Yep, forgot that the sq/cq are referenced from the rsp which is per-queue.
Christoph, James, can I get an ack from you guys?
I really hate the nvmet_req_fabric_ops_init and nvmet_req_link_to_nvme_queues helpers, just use direct assignments instead of trivial helpers with crazy long names.
I need to look at it a bit more for the a real review.
Sagi Grimberg
maintainerPATCH 1/1 nvmet: Avoid writing fabric_ops, queue pointers on every request.
Quoted 9 lines
Quoted 5 lines
Yep, forgot that the sq/cq are referenced from the rsp which is per-queue.
Christoph, James, can I get an ack from you guys?
I really hate the
nvmet_req_fabric_ops_initandnvmet_req_link_to_nvme_queueshelpers, just use direct assignments instead of trivial helpers with crazy long names.
Heh, I don't mind it all that much, but we can hold it off until something better (if at all) comes along..
PATCH 1/1 nvmet: Avoid writing fabric_ops, queue pointers on every request.
Quoted 15 lines
-----Original Message-----
From: Sagi Grimberg [mailto:sagi at grimberg.me]
Sent: Wednesday, February 15, 2017 10:29 AM
To: hch at lst.de
Cc: Parav Pandit <parav at mellanox.com>; james.smart at broadcom.com; linux-nvme at lists.infradead.org
Subject: Re: [PATCH] nvmet: Avoid writingfabric_ops, queue pointers on every request.Quoted 3 lines
Quoted 3 lines
Yep, forgot that the sq/cq are referenced from the rsp which is per-queue.
Quoted 8 lines
Quoted 5 lines
Quoted 1 line
Christoph, James, can I get an ack from you guys?
I really hate the
nvmet_req_fabric_ops_initandnvmet_req_link_to_nvme_queueshelpers, just use direct assignments instead of trivial helpers with crazy long names.Heh, I don't mind it all that much, but we can hold it off until something better (if at all) comes along..
How aboutnvmet_ops_init()nvmet_link_queues()
[email protected]
maintainerPATCH 1/1 nvmet: Avoid writing fabric_ops, queue pointers on every request.
Quoted 3 lines · Parav Pandit
How about
nvmet_ops_init()nvmet_link_queues()
Just do the one respective two trivial assignment inline in the callers. That actually leads to smaller source and binary code.
PATCH 1/1 nvmet: Avoid writing fabric_ops, queue pointers on every request.
Quoted 11 lines
-----Original Message-----
From: hch at lst.de [mailto:hch at lst.de]
Sent: Wednesday, February 15, 2017 10:34 AM
To: Parav Pandit <parav at mellanox.com>
Cc: Sagi Grimberg <sagi at grimberg.me>; hch at lst.de; james.smart at broadcom.com; linux-nvme at lists.infradead.org
Subject: Re: [PATCH] nvmet: Avoid writingfabric_ops, queue pointers on every request.
Quoted 7 lines
Quoted 3 lines · Parav Pandit
How about
nvmet_ops_init()nvmet_link_queues()Just do the one respective two trivial assignment inline in the callers. That actually leads to smaller source and binary code.
o.k.
PATCH 1/1 nvmet: Avoid writing fabric_ops, queue pointers on every request.
In reply to Sagi GrimbergHi James,
Does this change look fine for FC?
I do not have infrastructure to test the FC side.
Parav
Quoted 73 lines
-----Original Message-----
From: Parav Pandit
Sent: Wednesday, February 8, 2017 2:03 PM
To: 'Sagi Grimberg' <sagi at grimberg.me>; hch at lst.de; james.smart at broadcom.com; linux-nvme at lists.infradead.org
Subject: RE: [PATCH] nvmet: Avoid writingfabric_ops, queue pointers on every request.Hi Sagi,
Quoted 34 lines
-----Original Message-----
From: Sagi Grimberg [mailto:sagi at grimberg.me]
Sent: Wednesday, February 8, 2017 12:18 PM
To: Parav Pandit <parav at mellanox.com>; hch at lst.de; james.smart at broadcom.com; linux-nvme at lists.infradead.org
Subject: Re: [PATCH] nvmet: Avoid writingfabric_ops, queue pointers on every request.Quoted 11 lines
Quoted 7 lines
Quoted 4 lines
Additionally this patch further avoid nvme cq and sq pointer initialization for every request during every request processing for rdma because nvme queue linking occurs during queue allocation time for AQ and IOQ.
This breaks SRQ mode where every
nvmet_rdma_cmdserves different queues in it's lifetime..I fail to understand that.
nvmet_rdma_create_queue_ib()is call for as many QPs as we create; notbased on number of SRQs we create.
Correct.
Quoted 1 line
nvmet_rdma_queuestores cq and sq.Correct.
Quoted 2 lines
So there are as many cq and sq on the fabric side as QPs for
fabric_connectcommand is called.
Quoted 3 lines
queue is pulled out of cq context on which we received the command. SRQ is just a place shared among this nvme queues to share the RQ buffer,
Quoted 6 lines
right?
Correct too, but we then assign the queue to the command, which is the context of the received SQE (maybe with in-capsule data). For the SRQ case we allocate the commands and pre-post them (before we have any queues), they are absolutely not bound to a given queue, they can't
actually.
Quoted 4 lines
So for each new recv completion, the command context is now bound to the queue that it completed on, so it can be bound to different queues in its life- time.
Sorry, I am still not getting it.
nvmet_rdma_rspstructure containsnvmet_req.nvmet_rdma_rsp(and sonvmet_req) are per QP allocations. cq and sq pointers are initialized insidenvmet_req.
nvmet_rdma_cmdis per RQ/SRQ allocation.
When we dorecv_done(), we bind rsp structure to cmd (cmd can be from RQ or SRQ).
So I believe this is still good.
If we have cq and sq pointer inside thenvmet_rdma_cmdthan I can understand that it can break.On a side note: I tested the patch with
use_srqflag (but didn't publish its performance numbers as they awaiting your per core SRQ fixes to match regular RQ numbers :-) ).