nvmet/nvmet_fc: add events for discovery controller rescan
Subsystems (5)
cover nvmet/nvmet_fc: add events for discovery controller rescan
Note: These patches were posted to the nvme list this morning. I should have cc'd the scsi list as well - so I'm not posting to the scsi list. I fully expect all patches to be pulled via the nvme-4.15 tree then the block tree.
A transport may have a transport-specific mechanism that can signal when discovery controller content has changed and request a host to reconnect to the discovery controller.
FC is such a transport. RSCNs may be generated by the FC port with the discovery server, with the RSCNs then broadcast to the FC-NVME hosts. A host, upon receiving the RSCN, would validate connectivity then initiate a discovery controller rescan, allowing new subsystems to be connected to or updating subsystem connectivity tables.
These patches:
- Modify the nvmet core layer to call a transport callback on every subsystem add or remove from a transport port.
- Modify the nvmet-fc transport to support the callback, and add its own internal lldd api to generate RSCN's via the lldd.
- Modify the lpfc driver to send/receive RSCNs for FC-NVME: transmit the changed attribute RSCN on the target, receiving the RSCN on the initiator and invoking the nvmet-fc transport rescan api. Also adds manual sysfs mechanism to generate the RSCN on the target.
Dick Kennedy (1):
lpfc: Add sysfs interface to post NVME RSCN
James Smart (4):
nvmet: call transport on subsystem add and delete
nvmet_fc: support transport subsystem events
lpfc: Add support to generate RSCN events for nport
lpfc: Add NVME rescan support via RSCNs
drivers/nvme/target/configfs.c | 2 + drivers/nvme/target/core.c | 10 ++++ drivers/nvme/target/fc.c | 10 ++++ drivers/nvme/target/nvmet.h | 2 + drivers/scsi/lpfc/lpfc.h | 2 + drivers/scsi/lpfc/lpfc_attr.c | 62 ++++++++++++++++++++ drivers/scsi/lpfc/lpfc_crtn.h | 4 ++ drivers/scsi/lpfc/lpfc_els.c | 118 +++++++++++++++++++++++++++++++++++++++ drivers/scsi/lpfc/lpfc_hbadisc.c | 35 ++++++++++++ drivers/scsi/lpfc/lpfc_hw.h | 9 +++ drivers/scsi/lpfc/lpfc_nvme.c | 42 ++++++++++++++ drivers/scsi/lpfc/lpfc_nvmet.c | 18 ++++++ drivers/scsi/lpfc/lpfc_sli.c | 1 + include/linux/nvme-fc-driver.h | 6 ++ 14 files changed, 321 insertions(+)
Signature · 1 line
2.13.1
PATCH 1/5 nvmet: call transport on subsystem add and delete
A transport may have a transport-specific mechanism that can signal when discovery controller content has changed and request a host to reconnect to the discovery controller.
FC is such a transport. RSCNs may be generated by the FC port with the discovery server, with the RSCNs then broadcast to the FC-NVME hosts. A host, upon receiving the RSCN, would validate connectivity then initiate a discovery controller rescan, allowing new subsystems to be connected to or updating subsystem connectivity tables.
Modify the subsystem link/unlink routines to always call a new nvmet_port_subsystem_chg() routine whenever a new subsystem is linked to a port or removed from a port.
The nvmet_port_subsystem_chg() routine will look to see if the transport supports a discovery change callback, and if so, calls the callback.
--- drivers/nvme/target/configfs.c | 2 ++ drivers/nvme/target/core.c | 10 ++++++++++ drivers/nvme/target/nvmet.h | 2 ++ 3 files changed, 14 insertions(+)
drivers/nvme/target/configfs.c +2 -0
@@ -503,6 +503,7 @@ static int nvmet_port_subsys_allow_link(struct config_item *parent,list_add_tail (&link->entry, &port->subsystems); nvmet_genctr++;up_write (&nvmet_config_sem); +nvmet_port_subsystem_chg (port);return 0 ; out_free_link:
@@ -529,6 +530,7 @@ static void nvmet_port_subsys_drop_link(struct config_item *parent, found:list_del (&p->entry); nvmet_genctr++; +nvmet_port_subsystem_chg (port);if (list_empty (&port->subsystems))nvmet_disable_port (port);up_write (&nvmet_config_sem);
drivers/nvme/target/core.c +10 -0
@@ -206,6 +206,16 @@ void nvmet_disable_port(struct nvmet_port *port)module_put (ops->owner); } +void nvmet_port_subsystem_chg (struct nvmet_port *port) +{ +struct nvmet_fabrics_ops *ops; + + ops = nvmet_transports[port->disc_addr.trtype]; + +if (ops->discov_chg) + ops->discov_chg (port); +} +static void nvmet_keep_alive_timer (struct work_struct *work) { struct nvmet_ctrl *ctrl = container_of(to_delayed_work(work),
drivers/nvme/target/nvmet.h +2 -0
@@ -209,6 +209,7 @@ struct nvmet_fabrics_ops {int (*add_port)(struct nvmet_port *port);void (*remove_port)(struct nvmet_port *port);void (*delete_ctrl)(struct nvmet_ctrl *ctrl); +void (*discov_chg)(struct nvmet_port *port); };#define NVMET_MAX_INLINE_BIOVEC 8
@@ -302,6 +303,7 @@ void nvmet_unregister_transport(struct nvmet_fabrics_ops *ops);int nvmet_enable_port (struct nvmet_port *port);void nvmet_disable_port (struct nvmet_port *port); +void nvmet_port_subsystem_chg (struct nvmet_port *port);void nvmet_referral_enable (struct nvmet_port *parent,struct nvmet_port *port);void nvmet_referral_disable (struct nvmet_port *port);
Signature · 1 line
2.13.1
PATCH 2/5 nvmet_fc: support transport subsystem events
In reply to James SmartThis patch adds support for transport events to signal discovery controller changes.
Add new lldd api for a subsystem change. The lldd is to generate the appropriate RSCN for the FC-NVME targetport.
Add transport op for discov_chg callback from the nvmet layer.
--- drivers/nvme/target/fc.c | 10 ++++++++++ include/linux/nvme-fc-driver.h | 6 ++++++ 2 files changed, 16 insertions(+)
drivers/nvme/target/fc.c +10 -0
@@ -2546,6 +2546,15 @@ nvmet_fc_remove_port(struct nvmet_port *port)nvmet_fc_tgtport_put (tgtport); } +static void +nvmet_fc_port_subsys_event (struct nvmet_port *port) +{ +struct nvmet_fc_tgtport *tgtport = port->priv; + +if (tgtport && tgtport->ops->nvme_subsystem_change) + tgtport->ops->nvme_subsystem_change (&tgtport->fc_target_port); +} +static struct nvmet_fabrics_ops nvmet_fc_tgt_fcp_ops = { .owner =THIS_MODULE , .type =NVMF_TRTYPE_FC ,
@@ -2554,6 +2563,7 @@ static struct nvmet_fabrics_ops nvmet_fc_tgt_fcp_ops = { .remove_port = nvmet_fc_remove_port, .queue_response = nvmet_fc_fcp_nvme_cmd_done, .delete_ctrl = nvmet_fc_delete_ctrl, + .discov_chg = nvmet_fc_port_subsys_event, };static int __init nvmet_fc_init_module (void )
include/linux/nvme-fc-driver.h +6 -0
@@ -344,6 +344,11 @@ struct nvme_fc_remote_port {* indicating an FC transport Aborted status. * Entrypoint is Mandatory. * +* @nvme_subsystem_change: Called by the transport to generate FC state change +* notifications to NVME initiators. The state change notifications should +* cause the initiator to reconnect to the discovery controller on the +* targetport to look for new discovery log records. +* * @max_hw_queues: indicates the maximum number of hw queues the LLDD * supports for cpu affinitization. * Value is Mandatory. Must be at least 1.
@@ -856,6 +861,7 @@ struct nvmet_fc_target_template { struct nvmefc_tgt_fcp_req *fcpreq);void (*defer_rcv)(struct nvmet_fc_target_port *tgtport,struct nvmefc_tgt_fcp_req *fcpreq); +void (*nvme_subsystem_change)(struct nvmet_fc_target_port *tgtport);u32 max_hw_queues;u16 max_sgl_segments;
Signature · 1 line
2.13.1
PATCH 3/5 lpfc: Add support to generate RSCN events for nport
In reply to James SmartThis patch adds general Changed Attribute RSCN support, both transmit and receive, to the driver. This support will be used with NVME to request initiators to perform nvme discovery.
This patch:
adds RSCN definitions
adds statistics on RSCN transmissions
creates a routine that will transmit a Changed Attribute RSCN Handle RSCN receipt
--- drivers/scsi/lpfc/lpfc.h | 1 + drivers/scsi/lpfc/lpfc_crtn.h | 2 + drivers/scsi/lpfc/lpfc_els.c | 114 +++++++++++++++++++++++++++++++++++++++ drivers/scsi/lpfc/lpfc_hbadisc.c | 35 ++++++++++++ drivers/scsi/lpfc/lpfc_hw.h | 9 ++++ drivers/scsi/lpfc/lpfc_sli.c | 1 + 6 files changed, 162 insertions(+)
drivers/scsi/lpfc/lpfc.h +1 -0
@@ -277,6 +277,7 @@ struct lpfc_stats {uint32_t elsXmitADISC;uint32_t elsXmitLOGO;uint32_t elsXmitSCR; +uint32_t elsXmitRSCN;uint32_t elsXmitRNID;uint32_t elsXmitFARP;uint32_t elsXmitFARPR;
drivers/scsi/lpfc/lpfc_crtn.h +2 -0
@@ -142,6 +142,7 @@ int lpfc_issue_els_adisc(struct lpfc_vport *, struct lpfc_nodelist *, uint8_t);int lpfc_issue_els_logo (struct lpfc_vport *,struct lpfc_nodelist *,uint8_t );int lpfc_issue_els_npiv_logo (struct lpfc_vport *,struct lpfc_nodelist *);int lpfc_issue_els_scr (struct lpfc_vport *,uint32_t ,uint8_t ); +int lpfc_issue_els_rscn (struct lpfc_vport *vport,uint8_t retry);int lpfc_issue_fabric_reglogin (struct lpfc_vport *);int lpfc_els_free_iocb (struct lpfc_hba *,struct lpfc_iocbq *);int lpfc_ct_free_iocb (struct lpfc_hba *,struct lpfc_iocbq *);
@@ -357,6 +358,7 @@ void lpfc_mbox_timeout_handler(struct lpfc_hba *);struct lpfc_nodelist *lpfc_findnode_did (struct lpfc_vport *,uint32_t );struct lpfc_nodelist *lpfc_findnode_wwpn (struct lpfc_vport *,struct lpfc_name *); +struct lpfc_nodelist *lpfc_findnode_mapped (struct lpfc_vport *vport);int lpfc_sli_issue_mbox_wait (struct lpfc_hba *,LPFC_MBOXQ_t *,uint32_t );
drivers/scsi/lpfc/lpfc_els.c +114 -0
132 lines not on this page · raw message
drivers/scsi/lpfc/lpfc_hbadisc.c +35 -0
@@ -5119,6 +5119,41 @@ __lpfc_findnode_did(struct lpfc_vport *vport, uint32_t did) }struct lpfc_nodelist * +lpfc_findnode_mapped (struct lpfc_vport *vport) +{ +struct Scsi_Host *shost =lpfc_shost_from_vport (vport); +struct lpfc_nodelist *ndlp; +uint32_t data1; +unsigned long iflags; + +spin_lock_irqsave (shost->host_lock, iflags); + +list_for_each_entry (ndlp, &vport->fc_nodes, nlp_listp) { +if (ndlp->nlp_state ==NLP_STE_UNMAPPED_NODE || + ndlp->nlp_state ==NLP_STE_MAPPED_NODE ) { + data1 = (((uint32_t )ndlp->nlp_state <<24 ) | + ((uint32_t )ndlp->nlp_xri <<16 ) | + ((uint32_t )ndlp->nlp_type <<8 ) | + ((uint32_t )ndlp->nlp_rpi &0xff )); +spin_unlock_irqrestore (shost->host_lock, iflags); +lpfc_printf_vlog (vport,KERN_INFO ,LOG_NODE , +"2025 FIND node DID " +"Data: x%p x%x x%x x%x %p\n" , + ndlp, ndlp->nlp_DID, + ndlp->nlp_flag, data1, + ndlp->active_rrqs_xri_bitmap); +return ndlp; + } + } +spin_unlock_irqrestore (shost->host_lock, iflags); + +/* FIND node did <did> NOT FOUND */ +lpfc_printf_vlog (vport,KERN_INFO ,LOG_NODE , +"2026 FIND mapped did NOT FOUND.\n" ); +return NULL ; +} + +struct lpfc_nodelist *lpfc_findnode_did (struct lpfc_vport *vport,uint32_t did) {struct Scsi_Host *shost =lpfc_shost_from_vport (vport);
drivers/scsi/lpfc/lpfc_hw.h +9 -0
@@ -561,6 +561,8 @@ struct fc_vft_header {/* * Extended Link Service LS_COMMAND codes (Payload Word 0) +* The supported payload size is also hard-coded into this word +* when necessary. */ #ifdef __BIG_ENDIAN_BITFIELD#define ELS_CMD_MASK 0xffff0000
@@ -597,6 +599,7 @@ struct fc_vft_header {#define ELS_CMD_RPS 0x56000000#define ELS_CMD_RPL 0x57000000#define ELS_CMD_FAN 0x60000000 +#define ELS_CMD_RSCN_XMT 0x61040008#define ELS_CMD_RSCN 0x61040000#define ELS_CMD_SCR 0x62000000#define ELS_CMD_RNID 0x78000000
@@ -637,6 +640,7 @@ struct fc_vft_header {#define ELS_CMD_RPS 0x56#define ELS_CMD_RPL 0x57#define ELS_CMD_FAN 0x60 +#define ELS_CMD_RSCN_XMT 0x08000461#define ELS_CMD_RSCN 0x0461#define ELS_CMD_SCR 0x62#define ELS_CMD_RNID 0x78
@@ -1088,6 +1092,11 @@ struct fc_lcb_res_frame {uint16_t lcb_duration;/* LCB Payload Word 2, bit 15:0 */ }; +struct fc_RSCN {/* RSCN ELS frame */ +#define RSCN_EQ_PORT_ATTRIBUTE 2 +uint32_t NPort_ID; +}; +/* * Read Diagnostic Parameters (RDP) ELS frame. */
drivers/scsi/lpfc/lpfc_sli.c +1 -0
@@ -8761,6 +8761,7 @@ lpfc_sli4_iocb2wqe(struct lpfc_hba *phba, struct lpfc_iocbq *iocbq,if (if_type ==LPFC_SLI_INTF_IF_TYPE_2 ) {if (pcmd && (*pcmd ==ELS_CMD_FLOGI || *pcmd ==ELS_CMD_SCR || + *pcmd ==ELS_CMD_RSCN_XMT || *pcmd ==ELS_CMD_FDISC || *pcmd ==ELS_CMD_LOGO || *pcmd ==ELS_CMD_PLOGI )) {
Signature · 1 line
2.13.1
PATCH 4/5 lpfc: Add NVME rescan support via RSCNs
In reply to James SmartThis patch adds NVME rescan support via RSCNs
NVME Target:
Adds support for the new nvme_subsystem_change callback. The callback can be invoked by the nvmet_fc transport when it detects conditions that would like to have nvme discovery invoked again. Typically this is upon the addition of new subsystems added to the nvmet configuration (thus the transport tying the callback being invoked to port add calls).
The callback routine will generate an RSCN to the fabric in order to cause RSCN events to all initiators with view of the nport.
NVME Host:
Upon reception of an RSCN event for an nport which supports NVME initiator and which is currently logged in and PRLI'd, the initiator will call the nvme_fc transport nvme_fc_rescan_remoteport() routine to request nvme discovery to be performed again on the port pair.
--- drivers/scsi/lpfc/lpfc_crtn.h | 2 ++ drivers/scsi/lpfc/lpfc_els.c | 4 ++++ drivers/scsi/lpfc/lpfc_nvme.c | 42 ++++++++++++++++++++++++++++++++++++++++++ drivers/scsi/lpfc/lpfc_nvmet.c | 18 ++++++++++++++++++ 4 files changed, 66 insertions(+)
drivers/scsi/lpfc/lpfc_crtn.h +2 -0
@@ -546,6 +546,8 @@ int lpfc_sli4_dump_page_a0(struct lpfc_hba *phba, struct lpfcMboxq *mbox);void lpfc_mbx_cmpl_rdp_page_a0 (struct lpfc_hba *phba,LPFC_MBOXQ_t *pmb);/* NVME interfaces. */ +void lpfc_nvme_rescan_port (struct lpfc_vport *vport, +struct lpfc_nodelist *ndlp);void lpfc_nvme_unregister_port (struct lpfc_vport *vport,struct lpfc_nodelist *ndlp); int lpfc_nvme_register_port(struct lpfc_vport *vport,
drivers/scsi/lpfc/lpfc_els.c +4 -0
@@ -6071,6 +6071,10 @@ lpfc_rscn_recovery_check(struct lpfc_vport *vport)if (vport->phba->nvmet_support)continue ; +/* Check to see if we need to NVME rescan this remoteport. */ +if (ndlp->nlp_fc4_type &NLP_FC4_NVME ) +lpfc_nvme_rescan_port (vport, ndlp); +lpfc_disc_state_machine (vport, ndlp,NULL ,NLP_EVT_DEVICE_RECOVERY );lpfc_cancel_retry_delay_tmo (vport, ndlp);
drivers/scsi/lpfc/lpfc_nvme.c +42 -0
48 lines not on this page · raw message
drivers/scsi/lpfc/lpfc_nvmet.c +18 -0
@@ -866,6 +866,23 @@ lpfc_nvmet_defer_rcv(struct nvmet_fc_target_port *tgtport,lpfc_rq_buf_free (phba, &nvmebuf->hbuf);/* repost */ } +static void +lpfc_nvmet_subsystem_change (struct nvmet_fc_target_port *tgtport) +{ +struct lpfc_nvmet_tgtport *tgtp; +struct lpfc_hba *phba; +uint32_t rc; + + tgtp = tgtport->private; + phba = tgtp->phba; + + rc =lpfc_issue_els_rscn (phba->pport,0 ); +lpfc_printf_log (phba,KERN_ERR ,LOG_NVME , +"6420 NVMET subsystem change: " +"Notification %s\n" , + (rc) ?"Failed" :"Sent" ); +} +static struct nvmet_fc_target_template lpfc_tgttemplate = { .targetport_delete = lpfc_nvmet_targetport_delete, .xmt_ls_rsp = lpfc_nvmet_xmt_ls_rsp,
@@ -873,6 +890,7 @@ static struct nvmet_fc_target_template lpfc_tgttemplate = { .fcp_abort = lpfc_nvmet_xmt_fcp_abort, .fcp_req_release = lpfc_nvmet_xmt_fcp_release, .defer_rcv = lpfc_nvmet_defer_rcv, + .nvme_subsystem_change = lpfc_nvmet_subsystem_change, .max_hw_queues = 1, .max_sgl_segments = LPFC_NVMET_DEFAULT_SEGS,
Signature · 1 line
2.13.1
PATCH 5/5 lpfc: Add sysfs interface to post NVME RSCN
In reply to James SmartFrom: Dick Kennedy <[email protected]>
To support scenarios which aren't bound to nvmetcli add port scenarios, which is currently where the transport invokes the nvme_subsystem_changed callback, add a syfs attribute to lpfc which can be written to cause an RSCN to be generated for the nport.
--- drivers/scsi/lpfc/lpfc.h | 1 + drivers/scsi/lpfc/lpfc_attr.c | 62 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 63 insertions(+)
drivers/scsi/lpfc/lpfc.h +1 -0
@@ -781,6 +781,7 @@ struct lpfc_hba {uint32_t cfg_use_msi;uint32_t cfg_auto_imax;uint32_t cfg_fcp_imax; +uint32_t cfg_force_rscn;uint32_t cfg_fcp_cpu_map;uint32_t cfg_fcp_io_channel;uint32_t cfg_suppress_rsp;
drivers/scsi/lpfc/lpfc_attr.c +62 -0
80 lines not on this page · raw message
Signature · 1 line
2.13.1
cover nvmet/nvmet_fc: add events for discovery controller rescan
In reply to James SmartJames,
Quoted 4 lines
Note: These patches were posted to the nvme list this morning. I should have cc'd the scsi list as well - so I'm not posting to the scsi list. I fully expect all patches to be pulled via the nvme-4.15 tree then the block tree.
The SCSI pieces look OK to me.
Signature · 1 line
Martin K. Petersen Oracle Linux Engineering