nvme-fc: trace sq head updates
Subsystems (2)
PATCH 1/1 nvme-fc: trace sq head updates
FC-NVMe has some detailed rules if and when SQ head pointer updates should be send, but the spec is a tad unclear about the values. So add tracing to the FC-NVMe driver to track this.
--- drivers/nvme/host/fc.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-)
drivers/nvme/host/fc.c +7 -1
@@ -12,6 +12,7 @@#include "nvme.h" #include "fabrics.h" +#include "trace.h" #include <linux/nvme-fc-driver.h> #include <linux/nvme-fc.h>
@@ -35,7 +36,8 @@ struct nvme_fc_queue {u32 qnum;u32 rqcnt;u32 seqno; - +u16 sq_head; +u16 sq_tail;u64 connection_id;atomic_t csn;
@@ -1653,6 +1655,7 @@ nvme_fc_fcpio_done(struct nvmefc_fcp_req *req) status =cpu_to_le16 (NVME_SC_INTERNAL <<1 ); goto done; } + queue->sq_head =cpu_to_le16 (cqe->sq_head); result = cqe->result; status = cqe->status;break ;
@@ -1675,6 +1678,8 @@ nvme_fc_fcpio_done(struct nvmefc_fcp_req *req) }__nvme_fc_fcpop_chk_teardowns (ctrl, op, opstate); +trace_nvme_sq (rq, queue->sq_head, + queue->sq_tail % ctrl->ctrl.sqsize);nvme_end_request (rq, status, result); check_error:
@@ -2215,6 +2220,7 @@ nvme_fc_start_fcp_op(struct nvme_fc_ctrl *ctrl, struct nvme_fc_queue *queue, op->fcp_req.rcv_rsplen =0 ; op->fcp_req.status =NVME_SC_SUCCESS ; op->fcp_req.sqid =cpu_to_le16 (queue->qnum); + queue->sq_tail++;/* * validate per fabric rules, set fields mandated by fabric spec
Signature · 1 line
2.16.4
Keith Busch
maintainerPATCH 1/1 nvme-fc: trace sq head updates
drivers/nvme/host/fc.c nvme_fc_fcpio_done(struct nvmefc_fcp_req *req) · line 1682
Quoted 6 lines · Hannes Reinecke
drivers/nvme/host/fc.c +2
-0@@ -1675,6 +1678,8 @@ nvme_fc_fcpio_done(struct nvmefc_fcp_req *req) }__nvme_fc_fcpop_chk_teardowns (ctrl, op, opstate); +trace_nvme_sq (rq, queue->sq_head, + queue->sq_tail % ctrl->ctrl.sqsize);
Correct me if I'm wrong, but don't you need a +1 on that modulo value? It looks like ctrl.sqsize is set to the last valid index rather than the actual size.
PATCH 1/1 nvme-fc: trace sq head updates
In reply to Hannes ReineckeQuoted 49 lines · Hannes Reinecke
FC-NVMe has some detailed rules if and when SQ head pointer updates should be send, but the spec is a tad unclear about the values. So add tracing to the FC-NVMe driver to track this.
Signed-off-by Hannes Reinecke <hare at suse.com>--- drivers/nvme/host/fc.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-)drivers/nvme/host/fc.c +6
-1@@ -12,6 +12,7 @@#include "nvme.h" #include "fabrics.h" +#include "trace.h" #include <linux/nvme-fc-driver.h> #include <linux/nvme-fc.h> @@ -35,7 +36,8 @@ struct nvme_fc_queue {u32 qnum;u32 rqcnt;u32 seqno; - +u16 sq_head; +u16 sq_tail;u64 connection_id;atomic_t csn;@@ -1653,6 +1655,7 @@ nvme_fc_fcpio_done(struct nvmefc_fcp_req *req) status =cpu_to_le16 (NVME_SC_INTERNAL <<1 ); goto done; } + queue->sq_head =cpu_to_le16 (cqe->sq_head); result = cqe->result; status = cqe->status;break ;@@ -1675,6 +1678,8 @@ nvme_fc_fcpio_done(struct nvmefc_fcp_req *req) }__nvme_fc_fcpop_chk_teardowns (ctrl, op, opstate); +trace_nvme_sq (rq, queue->sq_head, + queue->sq_tail % ctrl->ctrl.sqsize);nvme_end_request (rq, status, result); check_error:
drivers/nvme/host/fc.c nvme_fc_start_fcp_op(struct nvme_fc_ctrl *ctrl, struct nvme_fc_queue *queue, · line 2226
Quoted 7 lines
drivers/nvme/host/fc.c +1
-0@@ -2215,6 +2220,7 @@ nvme_fc_start_fcp_op(struct nvme_fc_ctrl *ctrl, struct nvme_fc_queue *queue, op->fcp_req.rcv_rsplen =0 ; op->fcp_req.status =NVME_SC_SUCCESS ; op->fcp_req.sqid =cpu_to_le16 (queue->qnum); + queue->sq_tail++;/* * validate per fabric rules, set fields mandated by fabric spec
this is fine, but....?? to make this be meaningful/work, you're going to need to queue/serialize the processing of ERSP IU's based on the response sequence number (rsn).? Otherwise, there's no guarantee that sqhd comes back in the proper sequential order.?? And I'd recommend, as there's no other reason for ERSP serialization, that you don't hold up the completion but move the rsn/sqhd values to sideband tracking with the tracking doing the serializing and updating of sqhd.
-- james
PATCH 1/1 nvme-fc: trace sq head updates
Quoted 55 lines · James Smart
Quoted 49 lines · Hannes Reinecke
FC-NVMe has some detailed rules if and when SQ head pointer updates should be send, but the spec is a tad unclear about the values. So add tracing to the FC-NVMe driver to track this.
Signed-off-by Hannes Reinecke <hare at suse.com>---
?drivers/nvme/host/fc.c| 8 +++++++-
? 1 file changed, 7 insertions(+), 1 deletion(-)drivers/nvme/host/fc.c +0
-0@@ -12,6 +12,7 @@? #include "
nvme.h"
? #include "fabrics.h"
+#include "trace.h"
? #include <linux/nvme-fc-driver.h>
? #include <linux/nvme-fc.h>drivers/nvme/host/fc.c +5
-1@@ -35,7 +36,8 @@ struct nvme_fc_queue { ????? u32??????????? qnum; ????? u32??????????? rqcnt; ????? u32??????????? seqno; - +??? u16??????????? sq_head; +??? u16??????????? sq_tail; ????? u64??????????? connection_id; ????? atomic_t??????? csn;@@ -1653,6 +1655,7 @@ nvme_fc_fcpio_done(struct nvmefc_fcp_req *req) ????????????? status = cpu_to_le16(NVME_SC_INTERNAL << 1); ????????????? goto done; ????????? } +??????? queue->sq_head = cpu_to_le16(cqe->sq_head); ????????? result = cqe->result; ????????? status = cqe->status; ????????? break;@@ -1675,6 +1678,8 @@ nvme_fc_fcpio_done(struct nvmefc_fcp_req *req) ????? } ????? __nvme_fc_fcpop_chk_teardowns(ctrl, op, opstate); +??? trace_nvme_sq(rq, queue->sq_head, +????????????? queue->sq_tail % ctrl->ctrl.sqsize); ????? nvme_end_request(rq, status, result);?
check_error:drivers/nvme/host/fc.c +1
-0@@ -2215,6 +2220,7 @@ nvme_fc_start_fcp_op(struct nvme_fc_ctrl *ctrl,struct nvme_fc_queue *queue, ????? op->fcp_req.rcv_rsplen = 0; ????? op->fcp_req.status = NVME_SC_SUCCESS; ????? op->fcp_req.sqid = cpu_to_le16(queue->qnum); +??? queue->sq_tail++;Quoted 2 lines
????? /*
?????? * validate per fabric rules, set fields mandated by fabric spec
Quoted 8 lines
this is fine, but....?? to make this be meaningful/work, you're going to need to queue/serialize the processing of ERSP IU's based on the response sequence number (rsn).? Otherwise, there's no guarantee that sqhd comes back in the proper sequential order.?? And I'd recommend, as there's no other reason for ERSP serialization, that you don't hold up the completion but move the rsn/sqhd values to sideband tracking with the tracking doing the serializing and updating of sqhd.
Right, see, what you mean.
Would also have the beauty of moving sqhd tracking off the hot path; after all, we don't have to be 100% correct in tracking sqhd, and can allow for lazy updates.
Will be working on an updated patch.
Cheers,
Hannes