nvme-fc: sync with FC-NVME-2 standard
Subsystems (3)
cover nvme-fc: sync with FC-NVME-2 standard
The following 3 patches start the sync up with FC-NVME-2 rev 1.06.
The first patch updates the nvme-fc.h header to match the spec The second patch updates the existing transport to match the
header - where field names may have changed.
The last patch updates the CMD IU for a new "category" field that
identifies the command set type for the io - needed by analyzer guys to decode the sqe.
Subsequent patches will be coming to add new behavioral changes. Older implementations should continue to work fine, with new behaviors enabling new features.
James Smart (3):
nvme-fc: Sync nvme-fc header to FC-NVME-2 nvme-fc and nvmet-fc: sync with FC-NVME-2 header changes
nvme-fc: Set new cmd set indicator in nvme-fc cmnd iu
drivers/nvme/host/fc.c | 29 ++++---- drivers/nvme/target/fc.c | 27 +++---- include/linux/nvme-fc.h | 182 +++++++++++++++++++++++++++++++++++------------ 3 files changed, 168 insertions(+), 70 deletions(-)
Signature · 7 lines
2.13.7 _______________________________________________ Linux-nvme mailing list [email protected] http://lists.infradead.org/mailman/listinfo/linux-nvme
PATCH 1/3 nvme-fc: Sync nvme-fc header to FC-NVME-2
Sync the header to FC-NVME-2 r1.06 (T11-2019-00210-v001).
Includes some minor mods where pre-release field names changed by the time the spec was released.
--- include/linux/nvme-fc.h | 182 ++++++++++++++++++++++++++++++++++++------------ 1 file changed, 137 insertions(+), 45 deletions(-)
include/linux/nvme-fc.h +137 -45
@@ -4,33 +4,60 @@*/ /* -* This file contains definitions relative to FC-NVME r1.14 (16-020vB). -* The fcnvme_lsdesc_cr_assoc_cmd struct reflects expected r1.16 content. +* This file contains definitions relative to FC-NVME-2 r1.06 +* (T11-2019-00210-v001). */ #ifndef _NVME_FC_H#define _NVME_FC_H 1 +#include <uapi/scsi/fc/fc_fs.h> -#define NVME_CMD_SCSI_ID 0xFD +#define NVME_CMD_FORMAT_ID 0xFD#define NVME_CMD_FC_ID FC_TYPE_NVME/* FC-NVME Cmd IU Flags */ -#define FCNVME_CMD_FLAGS_DIRMASK 0x03 -#define FCNVME_CMD_FLAGS_WRITE 0x01 -#define FCNVME_CMD_FLAGS_READ 0x02 +enum { +FCNVME_CMD_FLAGS_DIRMASK =0x03 , +FCNVME_CMD_FLAGS_WRITE = (1 <<0 ), +FCNVME_CMD_FLAGS_READ = (1 <<1 ), + +FCNVME_CMD_FLAGS_PICWP = (1 <<2 ), +}; + +enum { +FCNVME_CMD_CAT_MASK =0x0F , +FCNVME_CMD_CAT_ADMINQ =0x01 , +FCNVME_CMD_CAT_CSSMASK =0x07 , +FCNVME_CMD_CAT_CSSFLAG =0x08 , +}; + +static inline __u8 fccmnd_set_cat_admin (__u8 rsv_cat) +{ +return (rsv_cat & ~FCNVME_CMD_CAT_MASK ) |FCNVME_CMD_CAT_ADMINQ ; +} + +static inline __u8 fccmnd_set_cat_css (__u8 rsv_cat,__u8 css) +{ +return (rsv_cat & ~FCNVME_CMD_CAT_MASK ) |FCNVME_CMD_CAT_CSSFLAG | + (css &FCNVME_CMD_CAT_CSSMASK ); +}struct nvme_fc_cmd_iu { -__u8 scsi_id; +__u8 format_id;__u8 fc_id;__be16 iu_len; -__u8 rsvd4[3 ]; +__u8 rsvd4[2 ]; +__u8 rsv_cat;__u8 flags;__be64 connection_id;__be32 csn;__be32 data_len;struct nvme_command sqe; -__be32 rsvd88[2 ]; +__u8 dps; +__u8 lbads; +__be16 ms; +__be32 rsvd92; };#define NVME_FC_SIZEOF_ZEROS_RSP 12
@@ -38,11 +65,12 @@ struct nvme_fc_cmd_iu {enum {FCNVME_SC_SUCCESS =0 ,FCNVME_SC_INVALID_FIELD =1 , -FCNVME_SC_INVALID_CONNID =2 , +/* reserved 2 */ +FCNVME_SC_ILL_CONN_PARAMS =3 , };struct nvme_fc_ersp_iu { -__u8 status_code; +__u8 ersp_result;__u8 rsvd1;__be16 iu_len;__be32 rsn;
@@ -53,14 +81,44 @@ struct nvme_fc_ersp_iu { }; -/* FC-NVME Link Services */ +#define FCNVME_NVME_SR_OPCODE 0x01 + +struct nvme_fc_nvme_sr_iu { +__u8 fc_id; +__u8 opcode; +__u8 rsvd2; +__u8 retry_rctl; +__be32 rsvd4; +}; + + +enum { +FCNVME_SRSTAT_ACC =0x0 , +FCNVME_SRSTAT_INV_FCID =0x1 , +/* reserved 0x2 */ +FCNVME_SRSTAT_LOGICAL_ERR =0x3 , +FCNVME_SRSTAT_INV_QUALIF =0x4 , +FCNVME_SRSTAT_UNABL2PERFORM =0x9 , +}; + +struct nvme_fc_nvme_sr_rsp_iu { +__u8 fc_id; +__u8 opcode; +__u8 rsvd2; +__u8 status; +__be32 rsvd4; +}; + + +/* FC-NVME Link Services - LS cmd values (w0 bits 31:24) */ enum {FCNVME_LS_RSVD =0 ,FCNVME_LS_RJT =1 ,FCNVME_LS_ACC =2 , -FCNVME_LS_CREATE_ASSOCIATION =3 , -FCNVME_LS_CREATE_CONNECTION =4 , -FCNVME_LS_DISCONNECT =5 , +FCNVME_LS_CREATE_ASSOCIATION =3 ,/* Create Association */ +FCNVME_LS_CREATE_CONNECTION =4 ,/* Create I/O Connection */ +FCNVME_LS_DISCONNECT_ASSOC =5 ,/* Disconnect Association */ +FCNVME_LS_DISCONNECT_CONN =6 ,/* Disconnect Connection */ };/* FC-NVME Link Service Descriptors */
@@ -117,14 +175,17 @@ enum fcnvme_ls_rjt_reason {FCNVME_RJT_RC_UNSUP =0x0b ,/* command not supported */ -FCNVME_RJT_RC_INPROG =0x0e , -/* command already in progress */ -FCNVME_RJT_RC_INV_ASSOC =0x40 , -/* Invalid Association ID*/ +/* Invalid Association ID */ FCNVME_RJT_RC_INV_CONN =0x41 , -/* Invalid Connection ID*/ +/* Invalid Connection ID */ + +FCNVME_RJT_RC_INV_PARAM =0x42 , +/* Invalid Parameters */ + +FCNVME_RJT_RC_INSUF_RES =0x43 , +/* Insufficient Resources */ FCNVME_RJT_RC_VENDOR = 0xff,/* vendor specific error */
@@ -138,14 +199,32 @@ enum fcnvme_ls_rjt_explan {FCNVME_RJT_EXP_OXID_RXID =0x17 ,/* invalid OX_ID-RX_ID combination */ -FCNVME_RJT_EXP_INSUF_RES =0x29 , -/* insufficient resources */ -FCNVME_RJT_EXP_UNAB_DATA =0x2a ,/* unable to supply requested data */ FCNVME_RJT_EXP_INV_LEN =0x2d ,/* Invalid payload length */ + +FCNVME_RJT_EXP_INV_ERSP_RAT =0x40 , +/* Invalid NVMe_ERSP Ratio */ + +FCNVME_RJT_EXP_INV_CTLR_ID =0x41 , +/* Invalid Controller ID */ + +FCNVME_RJT_EXP_INV_QUEUE_ID =0x42 , +/* Invalid Queue ID */ + +FCNVME_RJT_EXP_INV_SQSIZE =0x43 , +/* Invalid Submission Queue Size */ + +FCNVME_RJT_EXP_INV_HOSTID =0x44 , +/* Invalid HOST ID */ + +FCNVME_RJT_EXP_INV_HOSTNQN =0x45 , +/* Invalid HOSTNQN */ + + FCNVME_RJT_EXP_INV_SUBNQN = 0x46, + /* Invalid SUBNQN */ };/* FCNVME_LSDESC_RJT */
@@ -209,21 +288,11 @@ struct fcnvme_lsdesc_cr_conn_cmd {__be32 rsvd52; }; -/* Disconnect Scope Values */ -enum { -FCNVME_DISCONN_ASSOCIATION =0 , -FCNVME_DISCONN_CONNECTION =1 , -}; -/* FCNVME_LSDESC_DISCONN_CMD */ struct fcnvme_lsdesc_disconn_cmd {__be32 desc_tag;/* FCNVME_LSDESC_xxx */ __be32 desc_len; -u8 rsvd8[3 ]; -/* note: scope is really a 1 bit field */ -u8 scope;/* FCNVME_DISCONN_xxx */ -__be32 rsvd12; -__be64 id; +__be32 rsvd8[4 ]; };/* FCNVME_LSDESC_CONN_ID */
@@ -242,9 +311,14 @@ struct fcnvme_lsdesc_assoc_id {/* r_ctl values */ enum { -FCNVME_RS_RCTL_DATA =1 , -FCNVME_RS_RCTL_XFER_RDY =5 , -FCNVME_RS_RCTL_RSP =8 , +FCNVME_RS_RCTL_CMND =0x6 , +FCNVME_RS_RCTL_DATA =0x1 , +FCNVME_RS_RCTL_CONF =0x3 , +FCNVME_RS_RCTL_SR =0x9 , +FCNVME_RS_RCTL_XFER_RDY =0x5 , +FCNVME_RS_RCTL_RSP =0x7 , +FCNVME_RS_RCTL_ERSP =0x8 , +FCNVME_RS_RCTL_SR_RSP =0xA , };
@@ -264,7 +338,10 @@ struct fcnvme_ls_acc_hdr {struct fcnvme_ls_rqst_w0 w0;__be32 desc_list_len;struct fcnvme_lsdesc_rqst rqst; -/* Followed by cmd-specific ACC descriptors, see next definitions */ +/* +* Followed by cmd-specific ACCEPT descriptors, see xxx_acc +* definitions below +*/ };/* FCNVME_LS_CREATE_ASSOCIATION */
@@ -302,25 +379,39 @@ struct fcnvme_ls_cr_conn_acc {struct fcnvme_lsdesc_conn_id connectid; }; -/* FCNVME_LS_DISCONNECT */ -struct fcnvme_ls_disconnect_rqst { +/* FCNVME_LS_DISCONNECT_ASSOC */ +struct fcnvme_ls_disconnect_assoc_rqst {struct fcnvme_ls_rqst_w0 w0;__be32 desc_list_len;struct fcnvme_lsdesc_assoc_id associd;struct fcnvme_lsdesc_disconn_cmd discon_cmd; }; -struct fcnvme_ls_disconnect_acc { +struct fcnvme_ls_disconnect_assoc_acc { +struct fcnvme_ls_acc_hdr hdr; +}; + + +/* FCNVME_LS_DISCONNECT_CONN */ +struct fcnvme_ls_disconnect_conn_rqst { +struct fcnvme_ls_rqst_w0 w0; +__be32 desc_list_len; +struct fcnvme_lsdesc_assoc_id associd; +struct fcnvme_lsdesc_disconn_cmd connectid; +}; + +struct fcnvme_ls_disconnect_conn_acc {struct fcnvme_ls_acc_hdr hdr; };/* -* Yet to be defined in FC-NVME: +* Default R_A_TOV is pulled in from fc_fs.h but needs conversion +* from ms to seconds for our use. */ -#define NVME_FC_CONNECT_TIMEOUT_SEC 2/* 2 seconds */ -#define NVME_FC_LS_TIMEOUT_SEC 2/* 2 seconds */ -#define NVME_FC_TGTOP_TIMEOUT_SEC 2/* 2 seconds */ +#define FC_TWO_TIMES_R_A_TOV (2 * (FC_DEF_R_A_TOV / 1000)) +#define NVME_FC_LS_TIMEOUT_SEC FC_TWO_TIMES_R_A_TOV +#define NVME_FC_TGTOP_TIMEOUT_SEC FC_TWO_TIMES_R_A_TOV/* * TRADDR string must be of form "nn-<16hexdigits>:pn-<16hexdigits>"
@@ -328,6 +419,7 @@ struct fcnvme_ls_disconnect_acc {* infront of the <16hexdigits>. Without is considered the "min" string * and with is considered the "max" string. The hexdigits may be upper * or lower case. +* Note: FC-NVME-2 standard requires a "0x" prefix. */ #define NVME_FC_TRADDR_NNLEN 3/* "?n-" */ #define NVME_FC_TRADDR_OXNNLEN 5/* "?n-0x" */
Signature · 7 lines
2.13.7 _______________________________________________ Linux-nvme mailing list [email protected] http://lists.infradead.org/mailman/listinfo/linux-nvme
PATCH 2/3 nvme-fc and nvmet-fc: sync with FC-NVME-2 header changes
In reply to James SmartSync sources with revised structure and field names to correspond with FC-NVME-2 header sync-up.
Tested interoperability with success:
- prior initiator with new target
- prior target with new initiator
- new on new
--- drivers/nvme/host/fc.c | 24 +++++++++++------------- drivers/nvme/target/fc.c | 27 +++++++++++++++------------ 2 files changed, 26 insertions(+), 25 deletions(-)
drivers/nvme/host/fc.c +11 -13
@@ -1224,7 +1224,7 @@ nvme_fc_connect_admin_queue(struct nvme_fc_ctrl *ctrl, lsreq->rqstlen =sizeof (*assoc_rqst); lsreq->rspaddr = assoc_acc; lsreq->rsplen =sizeof (*assoc_acc); - lsreq->timeout =NVME_FC_CONNECT_TIMEOUT_SEC ; + lsreq->timeout =NVME_FC_LS_TIMEOUT_SEC ; ret =nvme_fc_send_ls_req (ctrl->rport, lsop);if (ret)
@@ -1332,7 +1332,7 @@ nvme_fc_connect_queue(struct nvme_fc_ctrl *ctrl, struct nvme_fc_queue *queue, lsreq->rqstlen =sizeof (*conn_rqst); lsreq->rspaddr = conn_acc; lsreq->rsplen =sizeof (*conn_acc); - lsreq->timeout =NVME_FC_CONNECT_TIMEOUT_SEC ; + lsreq->timeout =NVME_FC_LS_TIMEOUT_SEC ; ret =nvme_fc_send_ls_req (ctrl->rport, lsop);if (ret)
@@ -1413,8 +1413,8 @@ nvme_fc_disconnect_assoc_done(struct nvmefc_ls_req *lsreq, int status)static void nvme_fc_xmt_disconnect_assoc (struct nvme_fc_ctrl *ctrl) { -struct fcnvme_ls_disconnect_rqst *discon_rqst; -struct fcnvme_ls_disconnect_acc *discon_acc; +struct fcnvme_ls_disconnect_assoc_rqst *discon_rqst; +struct fcnvme_ls_disconnect_assoc_acc *discon_acc;struct nvmefc_ls_req_op *lsop;struct nvmefc_ls_req *lsreq;int ret;
@@ -1430,11 +1430,11 @@ nvme_fc_xmt_disconnect_assoc(struct nvme_fc_ctrl *ctrl) lsreq = &lsop->ls_req; lsreq->private = (void *)&lsop[1 ]; - discon_rqst = (struct fcnvme_ls_disconnect_rqst *) + discon_rqst = (struct fcnvme_ls_disconnect_assoc_rqst *) (lsreq->private + ctrl->lport->ops->lsrqst_priv_sz); - discon_acc = (struct fcnvme_ls_disconnect_acc *)&discon_rqst[1 ]; + discon_acc = (struct fcnvme_ls_disconnect_assoc_acc *)&discon_rqst[1 ]; - discon_rqst->w0.ls_cmd =FCNVME_LS_DISCONNECT ; + discon_rqst->w0.ls_cmd =FCNVME_LS_DISCONNECT_ASSOC ; discon_rqst->desc_list_len =cpu_to_be32 (sizeof (struct fcnvme_lsdesc_assoc_id ) +sizeof (struct fcnvme_lsdesc_disconn_cmd ));
@@ -1451,14 +1451,12 @@ nvme_fc_xmt_disconnect_assoc(struct nvme_fc_ctrl *ctrl) discon_rqst->discon_cmd.desc_len =fcnvme_lsdesc_len (sizeof (struct fcnvme_lsdesc_disconn_cmd )); - discon_rqst->discon_cmd.scope =FCNVME_DISCONN_ASSOCIATION ; - discon_rqst->discon_cmd.id =cpu_to_be64 (ctrl->association_id); lsreq->rqstaddr = discon_rqst; lsreq->rqstlen =sizeof (*discon_rqst); lsreq->rspaddr = discon_acc; lsreq->rsplen =sizeof (*discon_acc); - lsreq->timeout =NVME_FC_CONNECT_TIMEOUT_SEC ; + lsreq->timeout =NVME_FC_LS_TIMEOUT_SEC ; ret =nvme_fc_send_ls_req_async (ctrl->rport, lsop, nvme_fc_disconnect_assoc_done);
@@ -1659,7 +1657,7 @@ nvme_fc_fcpio_done(struct nvmefc_fcp_req *req) (freq->rcv_rsplen / 4) || be32_to_cpu(op->rsp_iu.xfrd_len) != freq->transferred_length || - op->rsp_iu.status_code || + op->rsp_iu.ersp_result || sqe->common.command_id != cqe->command_id)) { status =cpu_to_le16 (NVME_SC_HOST_PATH_ERROR <<1 ); dev_info(ctrl->ctrl.device,
@@ -1669,7 +1667,7 @@ nvme_fc_fcpio_done(struct nvmefc_fcp_req *req) ctrl->cnum,be16_to_cpu (op->rsp_iu.iu_len),be32_to_cpu (op->rsp_iu.xfrd_len), freq->transferred_length, - op->rsp_iu.status_code, + op->rsp_iu.ersp_result, sqe->common.command_id, cqe->command_id); goto done;
@@ -1728,7 +1726,7 @@ __nvme_fc_init_request(struct nvme_fc_ctrl *ctrl, op->rq = rq; op->rqno = rqno; - cmdiu->scsi_id =NVME_CMD_SCSI_ID ; + cmdiu->format_id =NVME_CMD_FORMAT_ID ; cmdiu->fc_id =NVME_CMD_FC_ID ; cmdiu->iu_len =cpu_to_be16 (sizeof (*cmdiu) /sizeof (u32));
drivers/nvme/target/fc.c +15 -12
@@ -1495,20 +1495,20 @@ static void nvmet_fc_ls_disconnect(struct nvmet_fc_tgtport *tgtport, struct nvmet_fc_ls_iod *iod) { - struct fcnvme_ls_disconnect_rqst *rqst = - (struct fcnvme_ls_disconnect_rqst *)iod->rqstbuf; -struct fcnvme_ls_disconnect_acc *acc = - (struct fcnvme_ls_disconnect_acc *)iod->rspbuf; + struct fcnvme_ls_disconnect_assoc_rqst *rqst = + (struct fcnvme_ls_disconnect_assoc_rqst *)iod->rqstbuf; +struct fcnvme_ls_disconnect_assoc_acc *acc = + (struct fcnvme_ls_disconnect_assoc_acc *)iod->rspbuf;struct nvmet_fc_tgt_assoc *assoc;int ret =0 ;memset (acc,0 ,sizeof (*acc)); -if (iod->rqstdatalen <sizeof (struct fcnvme_ls_disconnect_rqst )) +if (iod->rqstdatalen <sizeof (struct fcnvme_ls_disconnect_assoc_rqst )) ret =VERR_DISCONN_LEN ;else if (rqst->desc_list_len !=fcnvme_lsdesc_len ( -sizeof (struct fcnvme_ls_disconnect_rqst ))) +sizeof (struct fcnvme_ls_disconnect_assoc_rqst ))) ret =VERR_DISCONN_RQST_LEN ;else if (rqst->associd.desc_tag !=cpu_to_be32 (FCNVME_LSDESC_ASSOC_ID )) ret =VERR_ASSOC_ID ;
@@ -1523,8 +1523,11 @@ nvmet_fc_ls_disconnect(struct nvmet_fc_tgtport *tgtport, fcnvme_lsdesc_len( sizeof(struct fcnvme_lsdesc_disconn_cmd))) ret =VERR_DISCONN_CMD_LEN ; - else if ((rqst->discon_cmd.scope != FCNVME_DISCONN_ASSOCIATION) && - (rqst->discon_cmd.scope != FCNVME_DISCONN_CONNECTION)) +/* +* As the standard changed on the LS, check if old format and scope +* something other than Association (e.g. 0). +*/ + else if (rqst->discon_cmd.rsvd8[0]) ret =VERR_DISCONN_SCOPE ; else {/* match an active association */
@@ -1556,8 +1559,8 @@ nvmet_fc_ls_disconnect(struct nvmet_fc_tgtport *tgtport,nvmet_fc_format_rsp_hdr (acc,FCNVME_LS_ACC ,fcnvme_lsdesc_len ( -sizeof (struct fcnvme_ls_disconnect_acc )), -FCNVME_LS_DISCONNECT ); +sizeof (struct fcnvme_ls_disconnect_assoc_acc )), +FCNVME_LS_DISCONNECT_ASSOC );/* release get taken in nvmet_fc_find_target_assoc */ nvmet_fc_tgt_a_put (iod->assoc);
@@ -1632,7 +1635,7 @@ nvmet_fc_handle_ls_rqst(struct nvmet_fc_tgtport *tgtport,/* Creates an IO Queue/Connection */ nvmet_fc_ls_create_connection (tgtport, iod);break ; -case FCNVME_LS_DISCONNECT : +case FCNVME_LS_DISCONNECT_ASSOC :/* Terminate a Queue/Connection or the Association */ nvmet_fc_ls_disconnect (tgtport, iod);break ;
@@ -2299,7 +2302,7 @@ nvmet_fc_rcv_fcp_req(struct nvmet_fc_target_port *target_port,/* validate iu, so the connection id can be used to find the queue */ if ((cmdiubuf_len !=sizeof (*cmdiu)) || - (cmdiu->scsi_id !=NVME_CMD_SCSI_ID ) || + (cmdiu->format_id !=NVME_CMD_FORMAT_ID ) || (cmdiu->fc_id !=NVME_CMD_FC_ID ) || (be16_to_cpu (cmdiu->iu_len) != (sizeof (*cmdiu)/4 )))return -EIO ;
Signature · 7 lines
2.13.7 _______________________________________________ Linux-nvme mailing list [email protected] http://lists.infradead.org/mailman/listinfo/linux-nvme
PATCH 3/3 nvme-fc: Set new cmd set indicator in nvme-fc cmnd iu
In reply to James SmartSet the new category field in the FC-NVME CMND_IU based on queue number.
--- drivers/nvme/host/fc.c | 5 +++++ 1 file changed, 5 insertions(+)
drivers/nvme/host/fc.c +5 -0
@@ -1729,6 +1729,11 @@ __nvme_fc_init_request(struct nvme_fc_ctrl *ctrl, cmdiu->format_id =NVME_CMD_FORMAT_ID ; cmdiu->fc_id =NVME_CMD_FC_ID ; cmdiu->iu_len =cpu_to_be16 (sizeof (*cmdiu) /sizeof (u32)); +if (queue->qnum) + cmdiu->rsv_cat =fccmnd_set_cat_css (0 , + (NVME_CC_CSS_NVM >>NVME_CC_CSS_SHIFT )); +else + cmdiu->rsv_cat =fccmnd_set_cat_admin (0 ); op->fcp_req.cmddma =fc_dma_map_single (ctrl->lport->dev, &op->cmd_iu,sizeof (op->cmd_iu),DMA_TO_DEVICE );
Signature · 7 lines
2.13.7 _______________________________________________ Linux-nvme mailing list [email protected] http://lists.infradead.org/mailman/listinfo/linux-nvme
Sagi Grimberg
maintainerThis looks good James, I'll queue these up to nvme-4.5
_______________________________________________
Linux-nvme mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-nvme
Keith Busch
maintainerQuoted 1 line · Sagi Grimberg
This looks good James, I'll queue these up to nvme-4.5
I've queued these up in nvme-5.5.
_______________________________________________
Linux-nvme mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-nvme