lpfc updates for 11.2.0.14
Subsystems (3)
v2 cover lpfc updates for 11.2.0.14
This patch set provides a number of bug fixes, code cleanups, and error handling, mostly in the nvme area of lpfc.
There is one new feature in the series to add MDS diagnostics support
The patches were cut against the Martin's 4.12/scsi-fixes tree. There are no outside dependencies. The patches should merge via Martin's tree.
v2 addresses comments from Hannes.
James Smart (15):
lpfc: Fix used-RPI accounting problem.
lpfc: Fix system crash when port is reset.
lpfc: Adding additional stats counters for nvme.
lpfc: Fix nvmet RQ resource needs for large block writes.
lpfc: Fix NVMEI driver not decrementing counter causing bad rport
state.
lpfc: Fix NMI watchdog assertions when running nvmet IOPS tests
lpfc: Separate NVMET data buffer pool fir ELS/CT.
lpfc: Separate NVMET RQ buffer posting from IO resources
SGL/iocbq/context
lpfc: Added recovery logic for running out of NVMET IO context
resources
lpfc: Fix NVME I+T not registering NVME as a supported FC4 type
lpfc: Fix debugfs root inode "lpfc" not getting deleted on driver
unload.
lpfc: Cleanup entry_repost settings on SLI4 queues
lpfc: Fix NVMEI's handling of NVMET's PRLI response attributes
lpfc: Add MDS Diagnostic support.
lpfc: update version to 11.2.0.14
drivers/scsi/lpfc/lpfc.h | 23 ++- drivers/scsi/lpfc/lpfc_attr.c | 47 +++-- drivers/scsi/lpfc/lpfc_crtn.h | 11 +- drivers/scsi/lpfc/lpfc_ct.c | 1 + drivers/scsi/lpfc/lpfc_debugfs.c | 69 ++++--- drivers/scsi/lpfc/lpfc_disc.h | 1 + drivers/scsi/lpfc/lpfc_els.c | 19 +- drivers/scsi/lpfc/lpfc_hbadisc.c | 9 +- drivers/scsi/lpfc/lpfc_hw4.h | 16 +- drivers/scsi/lpfc/lpfc_init.c | 137 ++++-------- drivers/scsi/lpfc/lpfc_mem.c | 100 +++------ drivers/scsi/lpfc/lpfc_nportdisc.c | 6 + drivers/scsi/lpfc/lpfc_nvmet.c | 412 ++++++++++++++++++++++++++++++------- drivers/scsi/lpfc/lpfc_nvmet.h | 14 +- drivers/scsi/lpfc/lpfc_sli.c | 357 +++++++++++++++++++++++++------- drivers/scsi/lpfc/lpfc_sli4.h | 19 +- drivers/scsi/lpfc/lpfc_version.h | 2 +- 17 files changed, 830 insertions(+), 413 deletions(-)
Signature · 1 line
2.11.0
PATCH v2 1/15 lpfc: Fix used-RPI accounting problem.
With 255 vports created a link trasition can casue a crash.
When going through discovery after a link bounce the driver is using rpis before the cmd FCOE_POST_HDR_TEMPLATES completes. By doing that the next rpi bumps the rpi range out of the boundary.
The fix it to increment the next_rpi only when the FCOE_POST_HDR_TEMPLATE succeeds.
--- drivers/scsi/lpfc/lpfc_els.c | 3 ++- drivers/scsi/lpfc/lpfc_init.c | 24 +++++------------------- drivers/scsi/lpfc/lpfc_sli.c | 8 ++++++++ drivers/scsi/lpfc/lpfc_sli4.h | 1 + 4 files changed, 16 insertions(+), 20 deletions(-)
drivers/scsi/lpfc/lpfc_els.c +2 -1
@@ -8667,7 +8667,8 @@ lpfc_cmpl_els_fdisc(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb,lpfc_do_scr_ns_plogi (phba, vport); goto out; fdisc_failed: -if (vport->fc_vport->vport_state !=FC_VPORT_NO_FABRIC_RSCS ) +if (vport->fc_vport && + (vport->fc_vport->vport_state !=FC_VPORT_NO_FABRIC_RSCS ))lpfc_vport_set_state (vport,FC_VPORT_FAILED );/* Cancel discovery timer */ lpfc_can_disctmo (vport);
drivers/scsi/lpfc/lpfc_init.c +5 -19
@@ -6525,7 +6525,6 @@ lpfc_sli4_create_rpi_hdr(struct lpfc_hba *phba)uint16_t rpi_limit, curr_rpi_range;struct lpfc_dmabuf *dmabuf;struct lpfc_rpi_hdr *rpi_hdr; -uint32_t rpi_count;/* * If the SLI4 port supports extents, posting the rpi header isn't
@@ -6538,8 +6537,7 @@ lpfc_sli4_create_rpi_hdr(struct lpfc_hba *phba)return NULL ;/* The limit on the logical index is just the max_rpi count. */ - rpi_limit = phba->sli4_hba.max_cfg_param.rpi_base + - phba->sli4_hba.max_cfg_param.max_rpi -1 ; + rpi_limit = phba->sli4_hba.max_cfg_param.max_rpi;spin_lock_irq (&phba->hbalock);/*
@@ -6550,18 +6548,10 @@ lpfc_sli4_create_rpi_hdr(struct lpfc_hba *phba) curr_rpi_range = phba->sli4_hba.next_rpi;spin_unlock_irq (&phba->hbalock); -/* -* The port has a limited number of rpis. The increment here -* is LPFC_RPI_HDR_COUNT - 1 to account for the starting value -* and to allow the full max_rpi range per port. -*/ -if ((curr_rpi_range + (LPFC_RPI_HDR_COUNT -1 )) > rpi_limit) - rpi_count = rpi_limit - curr_rpi_range; -else - rpi_count =LPFC_RPI_HDR_COUNT ; - -if (!rpi_count) +/* Reached full RPI range */ +if (curr_rpi_range == rpi_limit)return NULL ; +/* * First allocate the protocol header region for the port. The * port expects a 4KB DMA-mapped memory region that is 4K aligned.
@@ -6595,13 +6585,9 @@ lpfc_sli4_create_rpi_hdr(struct lpfc_hba *phba)/* The rpi_hdr stores the logical index only. */ rpi_hdr->start_rpi = curr_rpi_range; + rpi_hdr->next_rpi = phba->sli4_hba.next_rpi +LPFC_RPI_HDR_COUNT ;list_add_tail (&rpi_hdr->list, &phba->sli4_hba.lpfc_rpi_hdr_list); -/* -* The next_rpi stores the next logical module-64 rpi value used -* to post physical rpis in subsequent rpi postings. -*/ - phba->sli4_hba.next_rpi += rpi_count;spin_unlock_irq (&phba->hbalock);return rpi_hdr;
drivers/scsi/lpfc/lpfc_sli.c +8 -0
@@ -17137,6 +17137,14 @@ lpfc_sli4_post_rpi_hdr(struct lpfc_hba *phba, struct lpfc_rpi_hdr *rpi_page)"status x%x add_status x%x, mbx status x%x\n" , shdr_status, shdr_add_status, rc); rc = -ENXIO ; + }else { +/* +* The next_rpi stores the next logical module-64 rpi value used +* to post physical rpis in subsequent rpi postings. +*/ +spin_lock_irq (&phba->hbalock); + phba->sli4_hba.next_rpi = rpi_page->next_rpi; +spin_unlock_irq (&phba->hbalock); }return rc; }
drivers/scsi/lpfc/lpfc_sli4.h +1 -0
@@ -698,6 +698,7 @@ struct lpfc_rpi_hdr {struct lpfc_dmabuf *dmabuf;uint32_t page_count;uint32_t start_rpi; +uint16_t next_rpi; };struct lpfc_rsrc_blks {
Signature · 1 line
2.11.0
PATCH v2 2/15 lpfc: Fix system crash when port is reset.
In reply to James SmartThe driver panic when using the els_wq during port reset.
Check for NULL els_wq before dereferencing.
--- drivers/scsi/lpfc/lpfc.h | 8 ++++++-- drivers/scsi/lpfc/lpfc_hbadisc.c | 6 +++--- 2 files changed, 9 insertions(+), 5 deletions(-)
drivers/scsi/lpfc/lpfc.h +6 -2
@@ -1228,7 +1228,11 @@ lpfc_sli_read_hs(struct lpfc_hba *phba)static inline struct lpfc_sli_ring *lpfc_phba_elsring (struct lpfc_hba *phba) { -if (phba->sli_rev ==LPFC_SLI_REV4 ) -return phba->sli4_hba.els_wq->pring; +if (phba->sli_rev ==LPFC_SLI_REV4 ) { +if (phba->sli4_hba.els_wq) +return phba->sli4_hba.els_wq->pring; +else +return NULL ; + }return &phba->sli.sli3_ring[LPFC_ELS_RING ]; }
drivers/scsi/lpfc/lpfc_hbadisc.c +3 -3
@@ -693,9 +693,9 @@ lpfc_work_done(struct lpfc_hba *phba) pring =lpfc_phba_elsring (phba); status = (ha_copy & (HA_RXMASK << (4 *LPFC_ELS_RING ))); status >>= (4 *LPFC_ELS_RING ); -if ((status &HA_RXMASK ) || - (pring->flag &LPFC_DEFERRED_RING_EVENT ) || - (phba->hba_flag &HBA_SP_QUEUE_EVT )) { +if (pring && (status &HA_RXMASK || + pring->flag &LPFC_DEFERRED_RING_EVENT || + phba->hba_flag &HBA_SP_QUEUE_EVT )) {if (pring->flag &LPFC_STOP_IOCB_EVENT ) { pring->flag |=LPFC_DEFERRED_RING_EVENT ;/* Set the lpfc data pending flag */
Signature · 1 line
2.11.0
PATCH v2 2/15 lpfc: Fix system crash when port is reset.
Quoted 3 lines · James Smart
The driver panic when using the
els_wqduring port reset.Check for NULL
els_wqbefore dereferencing.
James/Dick, if possible please take a look on:
https://marc.info/?l=linux-scsi&m=149487670315697
For some reason that email didn't get threaded as I expect. And also I forgot to include James' Gmail address, sorry.
Thanks in advance,
Guilherme
Quoted 45 lines
Signed-off-by Dick Kennedy <dick.kennedy at broadcom.com>Signed-off-by James Smart <james.smart at broadcom.com>Reviewed-by Hannes Reinecke <hare at suse.com>--- drivers/scsi/lpfc/lpfc.h | 8 ++++++-- drivers/scsi/lpfc/lpfc_hbadisc.c | 6 +++--- 2 files changed, 9 insertions(+), 5 deletions(-)drivers/scsi/lpfc/lpfc.h +6
-2@@ -1228,7 +1228,11 @@ lpfc_sli_read_hs(struct lpfc_hba *phba)static inline struct lpfc_sli_ring *lpfc_phba_elsring (struct lpfc_hba *phba) { -if (phba->sli_rev ==LPFC_SLI_REV4 ) -return phba->sli4_hba.els_wq->pring; +if (phba->sli_rev ==LPFC_SLI_REV4 ) { +if (phba->sli4_hba.els_wq) +return phba->sli4_hba.els_wq->pring; +else +return NULL ; + }return &phba->sli.sli3_ring[LPFC_ELS_RING ]; }drivers/scsi/lpfc/lpfc_hbadisc.c +3
-3@@ -693,9 +693,9 @@ lpfc_work_done(struct lpfc_hba *phba) pring =lpfc_phba_elsring (phba); status = (ha_copy & (HA_RXMASK << (4 *LPFC_ELS_RING ))); status >>= (4 *LPFC_ELS_RING ); -if ((status &HA_RXMASK ) || - (pring->flag &LPFC_DEFERRED_RING_EVENT ) || - (phba->hba_flag &HBA_SP_QUEUE_EVT )) { +if (pring && (status &HA_RXMASK || + pring->flag &LPFC_DEFERRED_RING_EVENT || + phba->hba_flag &HBA_SP_QUEUE_EVT )) {if (pring->flag &LPFC_STOP_IOCB_EVENT ) { pring->flag |=LPFC_DEFERRED_RING_EVENT ;/* Set the lpfc data pending flag */
PATCH v2 2/15 lpfc: Fix system crash when port is reset.
Quoted 8 lines · Guilherme G. Piccoli
Quoted 3 lines · James Smart
The driver panic when using the
els_wqduring port reset.Check for NULL
els_wqbefore dereferencing.James/Dick, if possible please take a look on:
https://marc.info/?l=linux-scsi&m=149487670315697
Quoted 7 lines
For some reason that email didn't get threaded as I expect. And also I forgot to include James' Gmail address, sorry.
Thanks in advance,
Guilherme
Gilherme,
Your patch is still needed. Can you repost your patch with a new subject and add my signature:
Sorry, I didn't see it to roll it into the set.
-- james
PATCH v2 2/15 lpfc: Fix system crash when port is reset.
Quoted 6 lines · James Smart
Your patch is still needed. Can you repost your patch with a new subject and add my signature:
Signed-off-by James Smart <james.smart at broadcom.com>Sorry, I didn't see it to roll it into the set.
No problem James, already sent it to linux-scsi:
https://marc.info/?l=linux-scsi&m=149505857715991
Cheers,
Guilherme
Quoted 10 lines
-- james
_______________________________________________
Linux-nvme mailing list
Linux-nvme at lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-nvme
PATCH v2 3/15 lpfc: Adding additional stats counters for nvme.
In reply to James SmartMore debug messages added for nvme statistics.
--- drivers/scsi/lpfc/lpfc_attr.c | 24 ++++++++++++--------- drivers/scsi/lpfc/lpfc_debugfs.c | 27 +++++++++++++---------- drivers/scsi/lpfc/lpfc_nvmet.c | 46 ++++++++++++++++++++++++++++++---------- drivers/scsi/lpfc/lpfc_nvmet.h | 12 ++++++----- drivers/scsi/lpfc/lpfc_sli.c | 38 ++++++++++++++++++++++++++++----- drivers/scsi/lpfc/lpfc_sli4.h | 2 +- 6 files changed, 106 insertions(+), 43 deletions(-)
drivers/scsi/lpfc/lpfc_attr.c +14 -10
@@ -205,8 +205,9 @@ lpfc_nvme_info_show(struct device *dev, struct device_attribute *attr, atomic_read(&tgtp->xmt_ls_rsp_error)); len +=snprintf (buf+len,PAGE_SIZE -len, -"FCP: Rcv %08x Drop %08x\n" , +"FCP: Rcv %08x Release %08x Drop %08x\n" ,atomic_read (&tgtp->rcv_fcp_cmd_in), +atomic_read (&tgtp->xmt_fcp_release),atomic_read (&tgtp->rcv_fcp_cmd_drop)); if (atomic_read(&tgtp->rcv_fcp_cmd_in) !=
@@ -218,15 +219,12 @@ lpfc_nvme_info_show(struct device *dev, struct device_attribute *attr, } len +=snprintf (buf+len,PAGE_SIZE -len, -"FCP Rsp: RD %08x rsp %08x WR %08x rsp %08x\n" , +"FCP Rsp: RD %08x rsp %08x WR %08x rsp %08x " +"drop %08x\n" ,atomic_read (&tgtp->xmt_fcp_read),atomic_read (&tgtp->xmt_fcp_read_rsp),atomic_read (&tgtp->xmt_fcp_write), -atomic_read (&tgtp->xmt_fcp_rsp)); - - len +=snprintf (buf+len,PAGE_SIZE -len, -"FCP Rsp: abort %08x drop %08x\n" , -atomic_read (&tgtp->xmt_fcp_abort), +atomic_read (&tgtp->xmt_fcp_rsp),atomic_read (&tgtp->xmt_fcp_drop)); len += snprintf(buf+len, PAGE_SIZE-len,
@@ -236,10 +234,16 @@ lpfc_nvme_info_show(struct device *dev, struct device_attribute *attr, atomic_read(&tgtp->xmt_fcp_rsp_drop)); len +=snprintf (buf+len,PAGE_SIZE -len, -"ABORT: Xmt %08x Err %08x Cmpl %08x" , +"ABORT: Xmt %08x Cmpl %08x\n" , +atomic_read (&tgtp->xmt_fcp_abort), +atomic_read (&tgtp->xmt_fcp_abort_cmpl)); + + len +=snprintf (buf + len,PAGE_SIZE - len, +"ABORT: Sol %08x Usol %08x Err %08x Cmpl %08x" , +atomic_read (&tgtp->xmt_abort_sol), +atomic_read (&tgtp->xmt_abort_unsol),atomic_read (&tgtp->xmt_abort_rsp), -atomic_read (&tgtp->xmt_abort_rsp_error), -atomic_read (&tgtp->xmt_abort_cmpl)); +atomic_read (&tgtp->xmt_abort_rsp_error)); len +=snprintf (buf+len,PAGE_SIZE -len,"\n" );return len;
drivers/scsi/lpfc/lpfc_debugfs.c +16 -11
@@ -798,21 +798,22 @@ lpfc_debugfs_nvmestat_data(struct lpfc_vport *vport, char *buf, int size) atomic_read(&tgtp->xmt_fcp_rsp)); len +=snprintf (buf + len, size - len, -"FCP Rsp: abort %08x drop %08x\n" , -atomic_read (&tgtp->xmt_fcp_abort), -atomic_read (&tgtp->xmt_fcp_drop)); - - len +=snprintf (buf + len, size - len,"FCP Rsp Cmpl: %08x err %08x drop %08x\n" ,atomic_read (&tgtp->xmt_fcp_rsp_cmpl),atomic_read (&tgtp->xmt_fcp_rsp_error),atomic_read (&tgtp->xmt_fcp_rsp_drop)); len +=snprintf (buf + len, size - len, -"ABORT: Xmt %08x Err %08x Cmpl %08x" , +"ABORT: Xmt %08x Cmpl %08x\n" , +atomic_read (&tgtp->xmt_fcp_abort), +atomic_read (&tgtp->xmt_fcp_abort_cmpl)); + + len +=snprintf (buf + len, size - len, +"ABORT: Sol %08x Usol %08x Err %08x Cmpl %08x" , +atomic_read (&tgtp->xmt_abort_sol), +atomic_read (&tgtp->xmt_abort_unsol),atomic_read (&tgtp->xmt_abort_rsp), -atomic_read (&tgtp->xmt_abort_rsp_error), -atomic_read (&tgtp->xmt_abort_cmpl)); +atomic_read (&tgtp->xmt_abort_rsp_error)); len +=snprintf (buf + len, size - len,"\n" );
@@ -1959,6 +1960,7 @@ lpfc_debugfs_nvmestat_write(struct file *file, const char __user *buf,atomic_set (&tgtp->rcv_ls_req_out,0 );atomic_set (&tgtp->rcv_ls_req_drop,0 );atomic_set (&tgtp->xmt_ls_abort,0 ); +atomic_set (&tgtp->xmt_ls_abort_cmpl,0 );atomic_set (&tgtp->xmt_ls_rsp,0 );atomic_set (&tgtp->xmt_ls_drop,0 );atomic_set (&tgtp->xmt_ls_rsp_error,0 );
@@ -1967,19 +1969,22 @@ lpfc_debugfs_nvmestat_write(struct file *file, const char __user *buf,atomic_set (&tgtp->rcv_fcp_cmd_in,0 );atomic_set (&tgtp->rcv_fcp_cmd_out,0 );atomic_set (&tgtp->rcv_fcp_cmd_drop,0 ); -atomic_set (&tgtp->xmt_fcp_abort,0 );atomic_set (&tgtp->xmt_fcp_drop,0 );atomic_set (&tgtp->xmt_fcp_read_rsp,0 );atomic_set (&tgtp->xmt_fcp_read,0 );atomic_set (&tgtp->xmt_fcp_write,0 );atomic_set (&tgtp->xmt_fcp_rsp,0 ); +atomic_set (&tgtp->xmt_fcp_release,0 );atomic_set (&tgtp->xmt_fcp_rsp_cmpl,0 );atomic_set (&tgtp->xmt_fcp_rsp_error,0 );atomic_set (&tgtp->xmt_fcp_rsp_drop,0 ); +atomic_set (&tgtp->xmt_fcp_abort,0 ); +atomic_set (&tgtp->xmt_fcp_abort_cmpl,0 ); +atomic_set (&tgtp->xmt_abort_sol,0 ); +atomic_set (&tgtp->xmt_abort_unsol,0 );atomic_set (&tgtp->xmt_abort_rsp,0 );atomic_set (&tgtp->xmt_abort_rsp_error,0 ); -atomic_set (&tgtp->xmt_abort_cmpl,0 ); }return nbytes; }
@@ -3143,7 +3148,7 @@ __lpfc_idiag_print_rqpair(struct lpfc_queue *qp, struct lpfc_queue *datqp, "\t\t%s RQ info: ", rqtype); len +=snprintf (pbuffer + len,LPFC_QUE_INFO_GET_BUF_SIZE - len,"AssocCQID[%02d]: RQ-STAT[nopost:x%x nobuf:x%x " -"trunc:x%x rcv:x%llx]\n" , +"posted:x%x rcv:x%llx]\n" , qp->assoc_qid, qp->q_cnt_1, qp->q_cnt_2, qp->q_cnt_3, (unsigned long long )qp->q_cnt_4); len += snprintf(pbuffer + len, LPFC_QUE_INFO_GET_BUF_SIZE - len,
drivers/scsi/lpfc/lpfc_nvmet.c +35 -11
188 lines not on this page · raw message
drivers/scsi/lpfc/lpfc_nvmet.h +7 -5
@@ -34,6 +34,7 @@ struct lpfc_nvmet_tgtport {atomic_t rcv_ls_req_out;atomic_t rcv_ls_req_drop;atomic_t xmt_ls_abort; +atomic_t xmt_ls_abort_cmpl;/* Stats counters - lpfc_nvmet_xmt_ls_rsp */ atomic_t xmt_ls_rsp;
@@ -47,9 +48,9 @@ struct lpfc_nvmet_tgtport {atomic_t rcv_fcp_cmd_in;atomic_t rcv_fcp_cmd_out;atomic_t rcv_fcp_cmd_drop; +atomic_t xmt_fcp_release;/* Stats counters - lpfc_nvmet_xmt_fcp_op */ -atomic_t xmt_fcp_abort;atomic_t xmt_fcp_drop;atomic_t xmt_fcp_read_rsp;atomic_t xmt_fcp_read;
@@ -62,12 +63,13 @@ struct lpfc_nvmet_tgtport {atomic_t xmt_fcp_rsp_drop; -/* Stats counters - lpfc_nvmet_unsol_issue_abort */ +/* Stats counters - lpfc_nvmet_xmt_fcp_abort */ +atomic_t xmt_fcp_abort; +atomic_t xmt_fcp_abort_cmpl; +atomic_t xmt_abort_sol; +atomic_t xmt_abort_unsol;atomic_t xmt_abort_rsp;atomic_t xmt_abort_rsp_error; - -/* Stats counters - lpfc_nvmet_xmt_abort_cmp */ -atomic_t xmt_abort_cmpl; };struct lpfc_nvmet_rcv_ctx {
drivers/scsi/lpfc/lpfc_sli.c +33 -5
@@ -512,6 +512,7 @@ lpfc_sli4_rq_put(struct lpfc_queue *hq, struct lpfc_queue *dq, }else {return -EINVAL ; } + hq->RQ_buf_posted += hq->entry_repost;writel (doorbell.word0, hq->db_regaddr); }return put_index;
@@ -12788,6 +12789,7 @@ lpfc_sli4_sp_handle_rcqe(struct lpfc_hba *phba, struct lpfc_rcqe *rcqe)struct fc_frame_header *fc_hdr;struct lpfc_queue *hrq = phba->sli4_hba.hdr_rq;struct lpfc_queue *drq = phba->sli4_hba.dat_rq; +struct lpfc_nvmet_tgtport *tgtp;struct hbq_dmabuf *dma_buf;uint32_t status, rq_id;unsigned long iflags;
@@ -12808,7 +12810,6 @@ lpfc_sli4_sp_handle_rcqe(struct lpfc_hba *phba, struct lpfc_rcqe *rcqe)case FC_STATUS_RQ_BUF_LEN_EXCEEDED :lpfc_printf_log (phba,KERN_ERR ,LOG_SLI ,"2537 Receive Frame Truncated!!\n" ); - hrq->RQ_buf_trunc++;case FC_STATUS_RQ_SUCCESS :lpfc_sli4_rq_release (hrq, drq);spin_lock_irqsave (&phba->hbalock, iflags);
@@ -12819,6 +12820,7 @@ lpfc_sli4_sp_handle_rcqe(struct lpfc_hba *phba, struct lpfc_rcqe *rcqe) goto out; } hrq->RQ_rcv_buf++; + hrq->RQ_buf_posted--;memcpy (&dma_buf->cq_event.cqe.rcqe_cmpl, rcqe,sizeof (*rcqe));/* If a NVME LS event (type 0x28), treat it as Fast path */
@@ -12832,8 +12834,21 @@ lpfc_sli4_sp_handle_rcqe(struct lpfc_hba *phba, struct lpfc_rcqe *rcqe)spin_unlock_irqrestore (&phba->hbalock, iflags); workposted = true;break ; -case FC_STATUS_INSUFF_BUF_NEED_BUF :case FC_STATUS_INSUFF_BUF_FRM_DISC : +if (phba->nvmet_support) { + tgtp = phba->targetport->private; +lpfc_printf_log (phba,KERN_ERR ,LOG_SLI |LOG_NVME , +"6402 RQE Error x%x, posted %d err_cnt " +"%d: %x %x %x\n" , + status, hrq->RQ_buf_posted, + hrq->RQ_no_posted_buf, +atomic_read (&tgtp->rcv_fcp_cmd_in), +atomic_read (&tgtp->rcv_fcp_cmd_out), +atomic_read (&tgtp->xmt_fcp_release)); + } +/* fallthrough */ + +case FC_STATUS_INSUFF_BUF_NEED_BUF : hrq->RQ_no_posted_buf++;/* Post more buffers if possible */ spin_lock_irqsave (&phba->hbalock, iflags);
@@ -13135,6 +13150,7 @@ lpfc_sli4_nvmet_handle_rcqe(struct lpfc_hba *phba, struct lpfc_queue *cq,struct lpfc_queue *drq;struct rqb_dmabuf *dma_buf;struct fc_frame_header *fc_hdr; +struct lpfc_nvmet_tgtport *tgtp;uint32_t status, rq_id;unsigned long iflags;uint32_t fctl, idx;
@@ -13165,8 +13181,6 @@ lpfc_sli4_nvmet_handle_rcqe(struct lpfc_hba *phba, struct lpfc_queue *cq,case FC_STATUS_RQ_BUF_LEN_EXCEEDED :lpfc_printf_log (phba,KERN_ERR ,LOG_SLI ,"6126 Receive Frame Truncated!!\n" ); - hrq->RQ_buf_trunc++; -break ;case FC_STATUS_RQ_SUCCESS :lpfc_sli4_rq_release (hrq, drq);spin_lock_irqsave (&phba->hbalock, iflags);
@@ -13178,6 +13192,7 @@ lpfc_sli4_nvmet_handle_rcqe(struct lpfc_hba *phba, struct lpfc_queue *cq, }spin_unlock_irqrestore (&phba->hbalock, iflags); hrq->RQ_rcv_buf++; + hrq->RQ_buf_posted--; fc_hdr = (struct fc_frame_header *)dma_buf->hbuf.virt;/* Just some basic sanity checks on FCP Command frame */
@@ -13200,8 +13215,21 @@ lpfc_sli4_nvmet_handle_rcqe(struct lpfc_hba *phba, struct lpfc_queue *cq, drop:lpfc_in_buf_free (phba, &dma_buf->dbuf);break ; -case FC_STATUS_INSUFF_BUF_NEED_BUF :case FC_STATUS_INSUFF_BUF_FRM_DISC : +if (phba->nvmet_support) { + tgtp = phba->targetport->private; +lpfc_printf_log (phba,KERN_ERR ,LOG_SLI |LOG_NVME , +"6401 RQE Error x%x, posted %d err_cnt " +"%d: %x %x %x\n" , + status, hrq->RQ_buf_posted, + hrq->RQ_no_posted_buf, +atomic_read (&tgtp->rcv_fcp_cmd_in), +atomic_read (&tgtp->rcv_fcp_cmd_out), +atomic_read (&tgtp->xmt_fcp_release)); + } +/* fallthrough */ + +case FC_STATUS_INSUFF_BUF_NEED_BUF : hrq->RQ_no_posted_buf++;/* Post more buffers if possible */ spin_lock_irqsave (&phba->hbalock, iflags);
drivers/scsi/lpfc/lpfc_sli4.h +1 -1
@@ -195,7 +195,7 @@ struct lpfc_queue {/* defines for RQ stats */ #define RQ_no_posted_buf q_cnt_1#define RQ_no_buf_found q_cnt_2 -#define RQ_buf_trunc q_cnt_3 +#define RQ_buf_posted q_cnt_3#define RQ_rcv_buf q_cnt_4uint64_t isr_timestamp;
Signature · 1 line
2.11.0
PATCH v2 4/15 lpfc: Fix nvmet RQ resource needs for large block writes.
In reply to James SmartLarge block writes to the nvme target were failing because the default number of RQs posted was insufficient.
Expand the NVMET RQs to 2048 RQEs and ensure a minimum of 512 RQEs are posted, no matter how many MRQs are configured.
---
v2:
set minimum mrq post to 512
merged parts of fixup patch 0012 into the patch:
rq/mrq entry_repost values
drivers/scsi/lpfc/lpfc_attr.c | 6 +++--- drivers/scsi/lpfc/lpfc_init.c | 23 +++++++++++++------- drivers/scsi/lpfc/lpfc_nvmet.c | 2 +- drivers/scsi/lpfc/lpfc_nvmet.h | 1 + drivers/scsi/lpfc/lpfc_sli.c | 49 +++++++++--------------------------------- drivers/scsi/lpfc/lpfc_sli4.h | 2 +- 6 files changed, 31 insertions(+), 52 deletions(-)
drivers/scsi/lpfc/lpfc_attr.c +3 -3
@@ -60,9 +60,9 @@#define LPFC_MIN_DEVLOSS_TMO 1#define LPFC_MAX_DEVLOSS_TMO 255 -#define LPFC_DEF_MRQ_POST 256 -#define LPFC_MIN_MRQ_POST 32 -#define LPFC_MAX_MRQ_POST 512 +#define LPFC_DEF_MRQ_POST 512 +#define LPFC_MIN_MRQ_POST 512 +#define LPFC_MAX_MRQ_POST 2048/* * Write key size should be multiple of 4. If write key is changed
drivers/scsi/lpfc/lpfc_init.c +15 -8
@@ -3390,6 +3390,11 @@ lpfc_sli4_nvmet_sgl_update(struct lpfc_hba *phba)*/ els_xri_cnt =lpfc_sli4_get_els_iocb_cnt (phba); nvmet_xri_cnt = phba->cfg_nvmet_mrq * phba->cfg_nvmet_mrq_post; + +/* Ensure we at least meet the minimun for the system */ +if (nvmet_xri_cnt <LPFC_NVMET_RQE_DEF_COUNT ) + nvmet_xri_cnt =LPFC_NVMET_RQE_DEF_COUNT ; + tot_cnt = phba->sli4_hba.max_cfg_param.max_xri - els_xri_cnt;if (nvmet_xri_cnt > tot_cnt) { phba->cfg_nvmet_mrq_post = tot_cnt / phba->cfg_nvmet_mrq;
@@ -8158,7 +8163,7 @@ lpfc_sli4_queue_create(struct lpfc_hba *phba)/* Create NVMET Receive Queue for header */ qdesc =lpfc_sli4_queue_alloc (phba, phba->sli4_hba.rq_esize, - phba->sli4_hba.rq_ecount); +LPFC_NVMET_RQE_DEF_COUNT );if (!qdesc) {lpfc_printf_log (phba,KERN_ERR ,LOG_INIT , "3146 Failed allocate "
@@ -8180,7 +8185,7 @@ lpfc_sli4_queue_create(struct lpfc_hba *phba)/* Create NVMET Receive Queue for data */ qdesc =lpfc_sli4_queue_alloc (phba, phba->sli4_hba.rq_esize, - phba->sli4_hba.rq_ecount); +LPFC_NVMET_RQE_DEF_COUNT );if (!qdesc) {lpfc_printf_log (phba,KERN_ERR ,LOG_INIT , "3156 Failed allocate "
@@ -8770,9 +8775,6 @@ lpfc_sli4_queue_setup(struct lpfc_hba *phba) goto out_destroy; } -lpfc_rq_adjust_repost (phba, phba->sli4_hba.hdr_rq,LPFC_ELS_HBQ ); -lpfc_rq_adjust_repost (phba, phba->sli4_hba.dat_rq,LPFC_ELS_HBQ ); - rc =lpfc_rq_create (phba, phba->sli4_hba.hdr_rq, phba->sli4_hba.dat_rq, phba->sli4_hba.els_cq,LPFC_USOL );if (rc) {
@@ -11096,7 +11098,7 @@ lpfc_pci_probe_one_s4(struct pci_dev *pdev, const struct pci_device_id *pid)struct lpfc_hba *phba;struct lpfc_vport *vport =NULL ;struct Scsi_Host *shost =NULL ; -int error, cnt; +int error, cnt, num;uint32_t cfg_mode, intr_mode;/* Allocate memory for HBA structure */
@@ -11131,8 +11133,13 @@ lpfc_pci_probe_one_s4(struct pci_dev *pdev, const struct pci_device_id *pid) } cnt = phba->cfg_iocb_cnt *1024 ; -if (phba->nvmet_support) - cnt += phba->cfg_nvmet_mrq_post * phba->cfg_nvmet_mrq; +if (phba->nvmet_support) { +/* Ensure we at least meet the minimun for the system */ + num = (phba->cfg_nvmet_mrq_post * phba->cfg_nvmet_mrq); +if (num <LPFC_NVMET_RQE_DEF_COUNT ) + num =LPFC_NVMET_RQE_DEF_COUNT ; + cnt += num; + }/* Initialize and populate the iocb list per host */ lpfc_printf_log(phba, KERN_INFO, LOG_INIT,
drivers/scsi/lpfc/lpfc_nvmet.c +1 -1
@@ -614,9 +614,9 @@ lpfc_nvmet_xmt_fcp_op(struct nvmet_fc_target_port *tgtport,lpfc_nvmeio_data (phba,"NVMET FCP CMND: xri x%x op x%x len x%x\n" , ctxp->oxid, rsp->op, rsp->rsplen); + ctxp->flag |=LPFC_NVMET_IO_INP ; rc =lpfc_sli4_issue_wqe (phba,LPFC_FCP_RING , nvmewqeq);if (rc ==WQE_SUCCESS ) { - ctxp->flag |=LPFC_NVMET_IO_INP ;#ifdef CONFIG_SCSI_LPFC_DEBUG_FS if (!phba->ktime_on)return 0 ;
drivers/scsi/lpfc/lpfc_nvmet.h +1 -0
@@ -22,6 +22,7 @@********************************************************************/ #define LPFC_NVMET_DEFAULT_SEGS (64 + 1)/* 256K IOs */ +#define LPFC_NVMET_RQE_DEF_COUNT 512#define LPFC_NVMET_SUCCESS_LEN 12/* Used for NVME Target */
drivers/scsi/lpfc/lpfc_sli.c +10 -39
111 lines not on this page · raw message
drivers/scsi/lpfc/lpfc_sli4.h +1 -1
@@ -156,6 +156,7 @@ struct lpfc_queue {uint32_t entry_size;/* Size of each queue entry. */ uint32_t entry_repost;/* Count of entries before doorbell is rung */ #define LPFC_QUEUE_MIN_REPOST 8 +#define LPFC_RQ_REPOST 64uint32_t queue_id;/* Queue ID assigned by the hardware */ uint32_t assoc_qid;/* Queue ID associated with, for CQ/WQ/MQ */ uint32_t page_count;/* Number of pages allocated for this queue */
@@ -763,7 +764,6 @@ int lpfc_rq_create(struct lpfc_hba *, struct lpfc_queue *,int lpfc_mrq_create (struct lpfc_hba *phba,struct lpfc_queue **hrqp,struct lpfc_queue **drqp,struct lpfc_queue **cqp,uint32_t subtype); -void lpfc_rq_adjust_repost (struct lpfc_hba *,struct lpfc_queue *,int );int lpfc_eq_destroy (struct lpfc_hba *,struct lpfc_queue *);int lpfc_cq_destroy (struct lpfc_hba *,struct lpfc_queue *);int lpfc_mq_destroy (struct lpfc_hba *,struct lpfc_queue *);
Signature · 1 line
2.11.0
PATCH v2 4/15 lpfc: Fix nvmet RQ resource needs for large block writes.
Quoted 15 lines · James Smart
Large block writes to the nvme target were failing because the default number of RQs posted was insufficient.
Expand the NVMET RQs to 2048 RQEs and ensure a minimum of 512 RQEs are posted, no matter how many MRQs are configured.
Signed-off-by Dick Kennedy <dick.kennedy at broadcom.com>Signed-off-by James Smart <james.smart at broadcom.com>---
v2:
set minimum mrq post to 512
merged parts of fixup patch 0012 into the patch:
rq/mrqentry_repostvalues
Quoted 8 lines
drivers/scsi/lpfc/lpfc_attr.c | 6 +++--- drivers/scsi/lpfc/lpfc_init.c | 23 +++++++++++++------- drivers/scsi/lpfc/lpfc_nvmet.c | 2 +- drivers/scsi/lpfc/lpfc_nvmet.h | 1 + drivers/scsi/lpfc/lpfc_sli.c | 49 +++++++++--------------------------------- drivers/scsi/lpfc/lpfc_sli4.h | 2 +- 6 files changed, 31 insertions(+), 52 deletions(-)
Cheers,
Hannes
Signature · 5 lines
Dr. Hannes Reinecke Teamlead Storage & Networking hare at suse.de +49 911 74053 688 SUSE LINUX GmbH, Maxfeldstr. 5, 90409 N?rnberg GF: F. Imend?rffer, J. Smithard, J. Guild, D. Upmanyu, G. Norton HRB 21284 (AG N?rnberg)
PATCH v2 5/15 lpfc: Fix NVMEI driver not decrementing counter causing bad rport state.
In reply to James SmartDuring driver boot, a latency in the NVMET driver side causes the incoming NVMEI PRLI to get rejected by the NVMET driver. When this happens, the NVMEI driver runs out of PRLI retries. Bouncing the link does not fix the situation.
If the NVMEI driver decides, on PRLI completion failures, to retry the PRLI, always decrement the fc4_prli_sent counter. This allows the PRLI completion to resolve to UNMAPPED when NVMET rejects the PRLI.
--- drivers/scsi/lpfc/lpfc_els.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-)
drivers/scsi/lpfc/lpfc_els.c +6 -3
@@ -2077,16 +2077,19 @@ lpfc_cmpl_els_prli(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb,if (irsp->ulpStatus) {/* Check for retry */ + ndlp->fc4_prli_sent--;if (lpfc_els_retry (phba, cmdiocb, rspiocb)) {/* ELS command is being retried */ - ndlp->fc4_prli_sent--; goto out; } +/* PRLI failed */ lpfc_printf_vlog (vport,KERN_ERR ,LOG_ELS , -"2754 PRLI failure DID:%06X Status:x%x/x%x\n" , +"2754 PRLI failure DID:%06X Status:x%x/x%x, " +"data: x%x\n" , ndlp->nlp_DID, irsp->ulpStatus, - irsp->un.ulpWord[4 ]); + irsp->un.ulpWord[4 ], ndlp->fc4_prli_sent); +/* Do not call DSM for lpfc_els_abort'ed ELS cmds */ if (lpfc_error_lost_link (irsp)) goto out;
Signature · 1 line
2.11.0
PATCH v2 6/15 lpfc: Fix NMI watchdog assertions when running nvmet IOPS tests
In reply to James SmartAfter running IOPS test for 30 second we get kernel:NMI watchdog: Watchdog detected hard LOCKUP on cpu 0
The driver is speend too much time in it's ISR.
In ISR EQ and CQ processing routines, if we hit the entry_repost numbers of EQE/CQEs just break out of the routine as opposed to hitting the doorbell with NOARM and continue processing.
---
v2:
merged parts of fixup patch 0012 into the patch:
eq/cq processing breaks
drivers/scsi/lpfc/lpfc_debugfs.c | 30 +++++++++++++++--------------- drivers/scsi/lpfc/lpfc_sli.c | 16 ++++++---------- 2 files changed, 21 insertions(+), 25 deletions(-)
drivers/scsi/lpfc/lpfc_debugfs.c +15 -15
@@ -3075,11 +3075,11 @@ __lpfc_idiag_print_wq(struct lpfc_queue *qp, char *wqtype, qp->assoc_qid, qp->q_cnt_1, (unsigned long long)qp->q_cnt_4); len +=snprintf (pbuffer + len,LPFC_QUE_INFO_GET_BUF_SIZE - len, -"\t\tWQID[%02d], QE-CNT[%04d], QE-SIZE[%04d], " -"HOST-IDX[%04d], PORT-IDX[%04d]" , +"\t\tWQID[%02d], QE-CNT[%04d], QE-SZ[%04d], " +"HST-IDX[%04d], PRT-IDX[%04d], PST[%03d]" , qp->queue_id, qp->entry_count, qp->entry_size, qp->host_index, - qp->hba_index); + qp->hba_index, qp->entry_repost); len +=snprintf (pbuffer + len,LPFC_QUE_INFO_GET_BUF_SIZE - len,"\n" );return len;
@@ -3126,11 +3126,11 @@ __lpfc_idiag_print_cq(struct lpfc_queue *qp, char *cqtype, qp->assoc_qid, qp->q_cnt_1, qp->q_cnt_2, qp->q_cnt_3, (unsigned long long)qp->q_cnt_4); len +=snprintf (pbuffer + len,LPFC_QUE_INFO_GET_BUF_SIZE - len, -"\tCQID[%02d], QE-CNT[%04d], QE-SIZE[%04d], " -"HOST-IDX[%04d], PORT-IDX[%04d]" , +"\tCQID[%02d], QE-CNT[%04d], QE-SZ[%04d], " +"HST-IDX[%04d], PRT-IDX[%04d], PST[%03d]" , qp->queue_id, qp->entry_count, qp->entry_size, qp->host_index, - qp->hba_index); + qp->hba_index, qp->entry_repost); len +=snprintf (pbuffer + len,LPFC_QUE_INFO_GET_BUF_SIZE - len,"\n" );
@@ -3152,16 +3152,16 @@ __lpfc_idiag_print_rqpair(struct lpfc_queue *qp, struct lpfc_queue *datqp, qp->assoc_qid, qp->q_cnt_1, qp->q_cnt_2, qp->q_cnt_3, (unsigned long long)qp->q_cnt_4); len +=snprintf (pbuffer + len,LPFC_QUE_INFO_GET_BUF_SIZE - len, -"\t\tHQID[%02d], QE-CNT[%04d], QE-SIZE[%04d], " -"HOST-IDX[%04d], PORT-IDX[%04d]\n" , +"\t\tHQID[%02d], QE-CNT[%04d], QE-SZ[%04d], " +"HST-IDX[%04d], PRT-IDX[%04d], PST[%03d]\n" , qp->queue_id, qp->entry_count, qp->entry_size, - qp->host_index, qp->hba_index); + qp->host_index, qp->hba_index, qp->entry_repost); len +=snprintf (pbuffer + len,LPFC_QUE_INFO_GET_BUF_SIZE - len, -"\t\tDQID[%02d], QE-CNT[%04d], QE-SIZE[%04d], " -"HOST-IDX[%04d], PORT-IDX[%04d]\n" , +"\t\tDQID[%02d], QE-CNT[%04d], QE-SZ[%04d], " +"HST-IDX[%04d], PRT-IDX[%04d], PST[%03d]\n" , datqp->queue_id, datqp->entry_count, datqp->entry_size, datqp->host_index, - datqp->hba_index); + datqp->hba_index, datqp->entry_repost);return len; }
@@ -3247,10 +3247,10 @@ __lpfc_idiag_print_eq(struct lpfc_queue *qp, char *eqtype, eqtype, qp->q_cnt_1, qp->q_cnt_2, qp->q_cnt_3, (unsigned long long)qp->q_cnt_4); len +=snprintf (pbuffer + len,LPFC_QUE_INFO_GET_BUF_SIZE - len, -"EQID[%02d], QE-CNT[%04d], QE-SIZE[%04d], " -"HOST-IDX[%04d], PORT-IDX[%04d]" , +"EQID[%02d], QE-CNT[%04d], QE-SZ[%04d], " +"HST-IDX[%04d], PRT-IDX[%04d], PST[%03d]" , qp->queue_id, qp->entry_count, qp->entry_size, - qp->host_index, qp->hba_index); + qp->host_index, qp->hba_index, qp->entry_repost); len +=snprintf (pbuffer + len,LPFC_QUE_INFO_GET_BUF_SIZE - len,"\n" );return len;
drivers/scsi/lpfc/lpfc_sli.c +6 -10
@@ -12961,7 +12961,7 @@ lpfc_sli4_sp_handle_eqe(struct lpfc_hba *phba, struct lpfc_eqe *eqe,while ((cqe =lpfc_sli4_cq_get (cq))) { workposted |=lpfc_sli4_sp_handle_mcqe (phba, cqe);if (!(++ecount % cq->entry_repost)) -lpfc_sli4_cq_release (cq,LPFC_QUEUE_NOARM ); +break ; cq->CQ_mbox++; }break ;
@@ -12975,7 +12975,7 @@ lpfc_sli4_sp_handle_eqe(struct lpfc_hba *phba, struct lpfc_eqe *eqe, workposted |=lpfc_sli4_sp_handle_cqe (phba, cq, cqe);if (!(++ecount % cq->entry_repost)) -lpfc_sli4_cq_release (cq,LPFC_QUEUE_NOARM ); +break ; }/* Track the max number of CQEs processed in 1 EQ */
@@ -13227,10 +13227,6 @@ lpfc_sli4_nvmet_handle_rcqe(struct lpfc_hba *phba, struct lpfc_queue *cq,case FC_STATUS_INSUFF_BUF_NEED_BUF : hrq->RQ_no_posted_buf++;/* Post more buffers if possible */ -spin_lock_irqsave (&phba->hbalock, iflags); - phba->hba_flag |=HBA_POST_RECEIVE_BUFFER ; -spin_unlock_irqrestore (&phba->hbalock, iflags); - workposted = true;break ; } out:
@@ -13384,7 +13380,7 @@ lpfc_sli4_hba_handle_eqe(struct lpfc_hba *phba, struct lpfc_eqe *eqe,while ((cqe =lpfc_sli4_cq_get (cq))) { workposted |=lpfc_sli4_fp_handle_cqe (phba, cq, cqe);if (!(++ecount % cq->entry_repost)) -lpfc_sli4_cq_release (cq,LPFC_QUEUE_NOARM ); +break ; }/* Track the max number of CQEs processed in 1 EQ */
@@ -13475,7 +13471,7 @@ lpfc_sli4_fof_handle_eqe(struct lpfc_hba *phba, struct lpfc_eqe *eqe)while ((cqe =lpfc_sli4_cq_get (cq))) { workposted |=lpfc_sli4_fp_handle_cqe (phba, cq, cqe);if (!(++ecount % cq->entry_repost)) -lpfc_sli4_cq_release (cq,LPFC_QUEUE_NOARM ); +break ; }/* Track the max number of CQEs processed in 1 EQ */
@@ -13557,7 +13553,7 @@ lpfc_sli4_fof_intr_handler(int irq, void *dev_id)while ((eqe =lpfc_sli4_eq_get (eq))) {lpfc_sli4_fof_handle_eqe (phba, eqe);if (!(++ecount % eq->entry_repost)) -lpfc_sli4_eq_release (eq,LPFC_QUEUE_NOARM ); +break ; eq->EQ_processed++; }
@@ -13674,7 +13670,7 @@ lpfc_sli4_hba_intr_handler(int irq, void *dev_id)lpfc_sli4_hba_handle_eqe (phba, eqe, hba_eqidx);if (!(++ecount % fpeq->entry_repost)) -lpfc_sli4_eq_release (fpeq,LPFC_QUEUE_NOARM ); +break ; fpeq->EQ_processed++; }
Signature · 1 line
2.11.0
PATCH v2 7/15 lpfc: Separate NVMET data buffer pool fir ELS/CT.
In reply to James SmartUsing 2048 byte buffer and onle 128 bytes is needed.
Create nee LFPC_NVMET_DATA_BUF_SIZE define to use
for NVMET RQ/MRQs.
--- drivers/scsi/lpfc/lpfc.h | 1 + drivers/scsi/lpfc/lpfc_crtn.h | 1 + drivers/scsi/lpfc/lpfc_hw4.h | 1 + drivers/scsi/lpfc/lpfc_init.c | 7 ++++++- drivers/scsi/lpfc/lpfc_mem.c | 33 ++++++++++++++++++++++++++------- drivers/scsi/lpfc/lpfc_sli.c | 19 +++++++++++++++---- 6 files changed, 50 insertions(+), 12 deletions(-)
drivers/scsi/lpfc/lpfc.h +1 -0
@@ -943,6 +943,7 @@ struct lpfc_hba {struct pci_pool *lpfc_mbuf_pool;struct pci_pool *lpfc_hrb_pool;/* header receive buffer pool */ struct pci_pool *lpfc_drb_pool;/* data receive buffer pool */ +struct pci_pool *lpfc_nvmet_drb_pool;/* data receive buffer pool */ struct pci_pool *lpfc_hbq_pool;/* SLI3 hbq buffer pool */ struct pci_pool *txrdy_payload_pool;struct lpfc_dma_pool lpfc_mbuf_safety_pool;
drivers/scsi/lpfc/lpfc_crtn.h +1 -0
@@ -271,6 +271,7 @@ int lpfc_sli4_fcf_rr_next_proc(struct lpfc_vport *, uint16_t);void lpfc_sli4_clear_fcf_rr_bmask (struct lpfc_hba *);int lpfc_mem_alloc (struct lpfc_hba *,int align); +int lpfc_nvmet_mem_alloc (struct lpfc_hba *phba);int lpfc_mem_alloc_active_rrq_pool_s4 (struct lpfc_hba *);void lpfc_mem_free (struct lpfc_hba *);void lpfc_mem_free_all (struct lpfc_hba *);
drivers/scsi/lpfc/lpfc_hw4.h +1 -0
@@ -1356,6 +1356,7 @@ struct lpfc_mbx_wq_destroy {#define LPFC_HDR_BUF_SIZE 128#define LPFC_DATA_BUF_SIZE 2048 +#define LPFC_NVMET_DATA_BUF_SIZE 128struct rq_context {uint32_t word0;#define lpfc_rq_context_rqe_count_SHIFT 16/* Version 0 Only */
drivers/scsi/lpfc/lpfc_init.c +6 -1
@@ -5956,16 +5956,21 @@ lpfc_sli4_driver_resource_setup(struct lpfc_hba *phba)for (i =0 ; i < lpfc_enable_nvmet_cnt; i++) {if (wwn == lpfc_enable_nvmet[i]) {#if (IS_ENABLED (CONFIG_NVME_TARGET_FC )) +if (lpfc_nvmet_mem_alloc (phba)) +break ; + + phba->nvmet_support =1 ;/* a match */ +lpfc_printf_log (phba,KERN_ERR ,LOG_INIT ,"6017 NVME Target %016llx\n" , wwn); - phba->nvmet_support =1 ;/* a match */ #else lpfc_printf_log (phba,KERN_ERR ,LOG_INIT ,"6021 Can't enable NVME Target." " NVME_TARGET_FC infrastructure" " is not in kernel\n" );#endif +break ; } } }
drivers/scsi/lpfc/lpfc_mem.c +26 -7
@@ -214,6 +214,21 @@ lpfc_mem_alloc(struct lpfc_hba *phba, int align)return -ENOMEM ; } +int +lpfc_nvmet_mem_alloc (struct lpfc_hba *phba) +{ + phba->lpfc_nvmet_drb_pool = +pci_pool_create ("lpfc_nvmet_drb_pool" , + phba->pcidev,LPFC_NVMET_DATA_BUF_SIZE , +SGL_ALIGN_SZ ,0 ); +if (!phba->lpfc_nvmet_drb_pool) { +lpfc_printf_log (phba,KERN_ERR ,LOG_INIT , +"6024 Can't enable NVME Target - no memory\n" ); +return -ENOMEM ; + } +return 0 ; +} +/** * lpfc_mem_free - Frees memory allocated by lpfc_mem_alloc * @phba: HBA to free memory for
@@ -232,6 +247,9 @@ lpfc_mem_free(struct lpfc_hba *phba)/* Free HBQ pools */ lpfc_sli_hbqbuf_free_all (phba); +if (phba->lpfc_nvmet_drb_pool) +pci_pool_destroy (phba->lpfc_nvmet_drb_pool); + phba->lpfc_nvmet_drb_pool =NULL ;if (phba->lpfc_drb_pool)pci_pool_destroy (phba->lpfc_drb_pool); phba->lpfc_drb_pool =NULL ;
@@ -624,20 +642,20 @@ lpfc_sli4_nvmet_alloc(struct lpfc_hba *phba)kfree (dma_buf);return NULL ; } - dma_buf->dbuf.virt =pci_pool_alloc (phba->lpfc_drb_pool,GFP_KERNEL , - &dma_buf->dbuf.phys); + dma_buf->dbuf.virt =pci_pool_alloc (phba->lpfc_nvmet_drb_pool, +GFP_KERNEL , &dma_buf->dbuf.phys);if (!dma_buf->dbuf.virt) {pci_pool_free (phba->lpfc_hrb_pool, dma_buf->hbuf.virt, dma_buf->hbuf.phys);kfree (dma_buf);return NULL ; } - dma_buf->total_size =LPFC_DATA_BUF_SIZE ; + dma_buf->total_size =LPFC_NVMET_DATA_BUF_SIZE ; dma_buf->context =kzalloc (sizeof (struct lpfc_nvmet_rcv_ctx ),GFP_KERNEL );if (!dma_buf->context) { -pci_pool_free (phba->lpfc_drb_pool, dma_buf->dbuf.virt, +pci_pool_free (phba->lpfc_nvmet_drb_pool, dma_buf->dbuf.virt, dma_buf->dbuf.phys);pci_pool_free (phba->lpfc_hrb_pool, dma_buf->hbuf.virt, dma_buf->hbuf.phys);
@@ -648,7 +666,7 @@ lpfc_sli4_nvmet_alloc(struct lpfc_hba *phba) dma_buf->iocbq =lpfc_sli_get_iocbq (phba);if (!dma_buf->iocbq) {kfree (dma_buf->context); -pci_pool_free (phba->lpfc_drb_pool, dma_buf->dbuf.virt, +pci_pool_free (phba->lpfc_nvmet_drb_pool, dma_buf->dbuf.virt, dma_buf->dbuf.phys);pci_pool_free (phba->lpfc_hrb_pool, dma_buf->hbuf.virt, dma_buf->hbuf.phys);
@@ -678,7 +696,7 @@ lpfc_sli4_nvmet_alloc(struct lpfc_hba *phba)if (!dma_buf->sglq) {lpfc_sli_release_iocbq (phba, dma_buf->iocbq);kfree (dma_buf->context); -pci_pool_free (phba->lpfc_drb_pool, dma_buf->dbuf.virt, +pci_pool_free (phba->lpfc_nvmet_drb_pool, dma_buf->dbuf.virt, dma_buf->dbuf.phys);pci_pool_free (phba->lpfc_hrb_pool, dma_buf->hbuf.virt, dma_buf->hbuf.phys);
@@ -718,7 +736,8 @@ lpfc_sli4_nvmet_free(struct lpfc_hba *phba, struct rqb_dmabuf *dmab)lpfc_sli_release_iocbq (phba, dmab->iocbq);kfree (dmab->context);pci_pool_free (phba->lpfc_hrb_pool, dmab->hbuf.virt, dmab->hbuf.phys); -pci_pool_free (phba->lpfc_drb_pool, dmab->dbuf.virt, dmab->dbuf.phys); +pci_pool_free (phba->lpfc_nvmet_drb_pool, + dmab->dbuf.virt, dmab->dbuf.phys);kfree (dmab); }
drivers/scsi/lpfc/lpfc_sli.c +15 -4
@@ -15079,7 +15079,12 @@ lpfc_rq_create(struct lpfc_hba *phba, struct lpfc_queue *hrq,if (phba->sli4_hba.pc_sli4_params.rqv ==LPFC_Q_CREATE_VERSION_1 ) {bf_set (lpfc_rq_context_rqe_count_1, &rq_create->u.request.context, hrq->entry_count); - rq_create->u.request.context.buffer_size =LPFC_DATA_BUF_SIZE ; +if (subtype ==LPFC_NVMET ) + rq_create->u.request.context.buffer_size = +LPFC_NVMET_DATA_BUF_SIZE ; +else + rq_create->u.request.context.buffer_size = +LPFC_DATA_BUF_SIZE ;bf_set (lpfc_rq_context_rqe_size, &rq_create->u.request.context,LPFC_RQE_SIZE_8 ); bf_set(lpfc_rq_context_page_size, &rq_create->u.request.context,
@@ -15116,8 +15121,14 @@ lpfc_rq_create(struct lpfc_hba *phba, struct lpfc_queue *hrq, LPFC_RQ_RING_SIZE_4096);break ; } -bf_set (lpfc_rq_context_buf_size, &rq_create->u.request.context, -LPFC_DATA_BUF_SIZE ); +if (subtype ==LPFC_NVMET ) +bf_set (lpfc_rq_context_buf_size, + &rq_create->u.request.context, +LPFC_NVMET_DATA_BUF_SIZE ); +else +bf_set (lpfc_rq_context_buf_size, + &rq_create->u.request.context, +LPFC_DATA_BUF_SIZE ); }bf_set (lpfc_rq_context_cq_id, &rq_create->u.request.context, cq->queue_id);
@@ -15263,7 +15274,7 @@ lpfc_mrq_create(struct lpfc_hba *phba, struct lpfc_queue **hrqp, cq->queue_id);bf_set (lpfc_rq_context_data_size, &rq_create->u.request.context, -LPFC_DATA_BUF_SIZE ); +LPFC_NVMET_DATA_BUF_SIZE );bf_set (lpfc_rq_context_hdr_size, &rq_create->u.request.context,LPFC_HDR_BUF_SIZE );
Signature · 1 line
2.11.0
PATCH v2 8/15 lpfc: Separate NVMET RQ buffer posting from IO resources SGL/iocbq/context
In reply to James SmartCurrently IO resources are mapped 1 to 1 with RQ buffers posted
Added logic to separate RQE buffers from IO op resources (sgl/iocbq/context). During initialization, the driver will determine how many SGLs it will allocate for NVMET (based on what the firmware reports) and associate a NVMET IOCBq and NVMET context structure with each one.
Now that hdr/data buffers are immediately reposted back to the RQ, 512 RQEs for each MRQ is sufficient. Also, since NVMET data buffers are now 128 bytes, lpfc_nvmet_mrq_post is not necessary anymore as we will always post the max (512) buffers per NVMET MRQ.
---
v2:
original patch added code that was then removed in a later
patch. Removed the code addition.
drivers/scsi/lpfc/lpfc.h | 11 +- drivers/scsi/lpfc/lpfc_attr.c | 11 -- drivers/scsi/lpfc/lpfc_crtn.h | 8 +- drivers/scsi/lpfc/lpfc_init.c | 92 ++------------- drivers/scsi/lpfc/lpfc_mem.c | 73 +----------- drivers/scsi/lpfc/lpfc_nvmet.c | 246 +++++++++++++++++++++++++++++++---------- drivers/scsi/lpfc/lpfc_nvmet.h | 1 + drivers/scsi/lpfc/lpfc_sli.c | 78 ++++++++++++- drivers/scsi/lpfc/lpfc_sli4.h | 4 +- 9 files changed, 291 insertions(+), 233 deletions(-)
drivers/scsi/lpfc/lpfc.h +7 -4
@@ -141,6 +141,13 @@ struct lpfc_dmabuf {uint32_t buffer_tag;/* used for tagged queue ring */ }; +struct lpfc_nvmet_ctxbuf { +struct list_head list; +struct lpfc_nvmet_rcv_ctx *context; +struct lpfc_iocbq *iocbq; +struct lpfc_sglq *sglq; +}; +struct lpfc_dma_pool {struct lpfc_dmabuf *elements;uint32_t max_count;
@@ -163,9 +170,6 @@ struct rqb_dmabuf {struct lpfc_dmabuf dbuf;uint16_t total_size;uint16_t bytes_recv; -void *context; -struct lpfc_iocbq *iocbq; -struct lpfc_sglq *sglq;struct lpfc_queue *hrq;/* ptr to associated Header RQ */ struct lpfc_queue *drq;/* ptr to associated Data RQ */ };
@@ -777,7 +781,6 @@ struct lpfc_hba {uint32_t cfg_nvme_oas;uint32_t cfg_nvme_io_channel;uint32_t cfg_nvmet_mrq; -uint32_t cfg_nvmet_mrq_post;uint32_t cfg_enable_nvmet;uint32_t cfg_nvme_enable_fb;uint32_t cfg_nvmet_fb_size;
drivers/scsi/lpfc/lpfc_attr.c +0 -11
@@ -3316,14 +3316,6 @@ LPFC_ATTR_R(nvmet_mrq, "Specify number of RQ pairs for processing NVMET cmds");/* -* lpfc_nvmet_mrq_post: Specify number buffers to post on every MRQ -* -*/ -LPFC_ATTR_R (nvmet_mrq_post,LPFC_DEF_MRQ_POST , -LPFC_MIN_MRQ_POST ,LPFC_MAX_MRQ_POST , -"Specify number of buffers to post on every MRQ" ); - -/* * lpfc_enable_fc4_type: Defines what FC4 types are supported. * Supported Values: 1 - register just FCP * 3 - register both FCP and NVME
@@ -5158,7 +5150,6 @@ struct device_attribute *lpfc_hba_attrs[] = { &dev_attr_lpfc_suppress_rsp, &dev_attr_lpfc_nvme_io_channel, &dev_attr_lpfc_nvmet_mrq, - &dev_attr_lpfc_nvmet_mrq_post, &dev_attr_lpfc_nvme_enable_fb, &dev_attr_lpfc_nvmet_fb_size, &dev_attr_lpfc_enable_bg,
@@ -6198,7 +6189,6 @@ lpfc_get_cfgparam(struct lpfc_hba *phba)lpfc_enable_fc4_type_init (phba, lpfc_enable_fc4_type);lpfc_nvmet_mrq_init (phba, lpfc_nvmet_mrq); -lpfc_nvmet_mrq_post_init (phba, lpfc_nvmet_mrq_post);/* Initialize first burst. Target vs Initiator are different. */ lpfc_nvme_enable_fb_init (phba, lpfc_nvme_enable_fb);
@@ -6295,7 +6285,6 @@ lpfc_nvme_mod_param_dep(struct lpfc_hba *phba)/* Not NVME Target mode. Turn off Target parameters. */ phba->nvmet_support =0 ; phba->cfg_nvmet_mrq =0 ; - phba->cfg_nvmet_mrq_post =0 ; phba->cfg_nvmet_fb_size =0 ; }
drivers/scsi/lpfc/lpfc_crtn.h +4 -4
@@ -75,6 +75,8 @@ void lpfc_init_vpi_cmpl(struct lpfc_hba *, LPFC_MBOXQ_t *);void lpfc_cancel_all_vport_retry_delay_timer (struct lpfc_hba *);void lpfc_retry_pport_discovery (struct lpfc_hba *);void lpfc_release_rpi (struct lpfc_hba *,struct lpfc_vport *,uint16_t ); +int lpfc_init_iocb_list (struct lpfc_hba *phba,int cnt); +void lpfc_free_iocb_list (struct lpfc_hba *phba);void lpfc_mbx_cmpl_local_config_link (struct lpfc_hba *,LPFC_MBOXQ_t *);void lpfc_mbx_cmpl_reg_login (struct lpfc_hba *,LPFC_MBOXQ_t *);
@@ -246,16 +248,14 @@ struct hbq_dmabuf *lpfc_sli4_rb_alloc(struct lpfc_hba *);void lpfc_sli4_rb_free (struct lpfc_hba *,struct hbq_dmabuf *);struct rqb_dmabuf *lpfc_sli4_nvmet_alloc (struct lpfc_hba *phba);void lpfc_sli4_nvmet_free (struct lpfc_hba *phba,struct rqb_dmabuf *dmab); -void lpfc_nvmet_rq_post (struct lpfc_hba *phba,struct lpfc_nvmet_rcv_ctx *ctxp, -struct lpfc_dmabuf *mp); +void lpfc_nvmet_ctxbuf_post (struct lpfc_hba *phba, +struct lpfc_nvmet_ctxbuf *ctxp);int lpfc_nvmet_rcv_unsol_abort (struct lpfc_vport *vport,struct fc_frame_header *fc_hdr);void lpfc_sli4_build_dflt_fcf_record (struct lpfc_hba *,struct fcf_record *,uint16_t );int lpfc_sli4_rq_put (struct lpfc_queue *hq,struct lpfc_queue *dq,struct lpfc_rqe *hrqe,struct lpfc_rqe *drqe); -int lpfc_post_rq_buffer (struct lpfc_hba *phba,struct lpfc_queue *hq, -struct lpfc_queue *dq,int count);int lpfc_free_rq_buffer (struct lpfc_hba *phba,struct lpfc_queue *hq);void lpfc_unregister_fcf (struct lpfc_hba *);void lpfc_unregister_fcf_rescan (struct lpfc_hba *);
drivers/scsi/lpfc/lpfc_init.c +10 -82
159 lines not on this page · raw message
drivers/scsi/lpfc/lpfc_mem.c +5 -68
97 lines not on this page · raw message
drivers/scsi/lpfc/lpfc_nvmet.c +188 -58
445 lines not on this page · raw message
drivers/scsi/lpfc/lpfc_nvmet.h +1 -0
@@ -106,6 +106,7 @@ struct lpfc_nvmet_rcv_ctx {#define LPFC_NVMET_CTX_RLS 0x8/* ctx free requested */ #define LPFC_NVMET_ABTS_RCV 0x10/* ABTS received on exchange */ struct rqb_dmabuf *rqb_buffer; +struct lpfc_nvmet_ctxbuf *ctxbuf;#ifdef CONFIG_SCSI_LPFC_DEBUG_FS uint64_t ts_isr_cmd;
drivers/scsi/lpfc/lpfc_sli.c +74 -4
123 lines not on this page · raw message
drivers/scsi/lpfc/lpfc_sli4.h +2 -2
@@ -618,10 +618,12 @@ struct lpfc_sli4_hba {uint16_t scsi_xri_start;uint16_t els_xri_cnt;uint16_t nvmet_xri_cnt; +uint16_t nvmet_ctx_cnt;struct list_head lpfc_els_sgl_list;struct list_head lpfc_abts_els_sgl_list;struct list_head lpfc_nvmet_sgl_list;struct list_head lpfc_abts_nvmet_ctx_list; +struct list_head lpfc_nvmet_ctx_list;struct list_head lpfc_abts_scsi_buf_list;struct list_head lpfc_abts_nvme_buf_list;struct lpfc_sglq **lpfc_sglq_active_list;
@@ -662,8 +664,6 @@ struct lpfc_sli4_hba {uint16_t num_online_cpu;uint16_t num_present_cpu;uint16_t curr_disp_cpu; - -uint16_t nvmet_mrq_post_idx; };enum lpfc_sge_type {
Signature · 1 line
2.11.0
PATCH v2 9/15 lpfc: Added recovery logic for running out of NVMET IO context resources
In reply to James SmartPrevious logic would just drop the IO
Added logic to queue the IO to wait for an IO context resource from an IO thats already in progress.
--- drivers/scsi/lpfc/lpfc.h | 1 + drivers/scsi/lpfc/lpfc_attr.c | 6 ++ drivers/scsi/lpfc/lpfc_crtn.h | 2 + drivers/scsi/lpfc/lpfc_debugfs.c | 6 ++ drivers/scsi/lpfc/lpfc_init.c | 2 + drivers/scsi/lpfc/lpfc_nvmet.c | 138 +++++++++++++++++++++++++++++++++------ drivers/scsi/lpfc/lpfc_sli.c | 7 +- drivers/scsi/lpfc/lpfc_sli4.h | 6 +- 8 files changed, 144 insertions(+), 24 deletions(-)
drivers/scsi/lpfc/lpfc.h +1 -0
@@ -170,6 +170,7 @@ struct rqb_dmabuf {struct lpfc_dmabuf dbuf;uint16_t total_size;uint16_t bytes_recv; +uint16_t idx;struct lpfc_queue *hrq;/* ptr to associated Header RQ */ struct lpfc_queue *drq;/* ptr to associated Data RQ */ };
drivers/scsi/lpfc/lpfc_attr.c +6 -0
@@ -245,6 +245,12 @@ lpfc_nvme_info_show(struct device *dev, struct device_attribute *attr,atomic_read (&tgtp->xmt_abort_rsp), atomic_read(&tgtp->xmt_abort_rsp_error)); + len +=snprintf (buf + len,PAGE_SIZE - len, +"IO_CTX: %08x outstanding %08x total %x" , + phba->sli4_hba.nvmet_ctx_cnt, + phba->sli4_hba.nvmet_io_wait_cnt, + phba->sli4_hba.nvmet_io_wait_total); + len +=snprintf (buf+len,PAGE_SIZE -len,"\n" );return len; }
drivers/scsi/lpfc/lpfc_crtn.h +2 -0
@@ -77,6 +77,8 @@ void lpfc_retry_pport_discovery(struct lpfc_hba *);void lpfc_release_rpi (struct lpfc_hba *,struct lpfc_vport *,uint16_t );int lpfc_init_iocb_list (struct lpfc_hba *phba,int cnt);void lpfc_free_iocb_list (struct lpfc_hba *phba); +int lpfc_post_rq_buffer (struct lpfc_hba *phba,struct lpfc_queue *hrq, +struct lpfc_queue *drq,int count,int idx);void lpfc_mbx_cmpl_local_config_link (struct lpfc_hba *,LPFC_MBOXQ_t *);void lpfc_mbx_cmpl_reg_login (struct lpfc_hba *,LPFC_MBOXQ_t *);
drivers/scsi/lpfc/lpfc_debugfs.c +6 -0
@@ -842,6 +842,12 @@ lpfc_debugfs_nvmestat_data(struct lpfc_vport *vport, char *buf, int size) }spin_unlock (&phba->sli4_hba.abts_nvme_buf_list_lock); } + + len +=snprintf (buf + len, size - len, +"IO_CTX: %08x outstanding %08x total %08x\n" , + phba->sli4_hba.nvmet_ctx_cnt, + phba->sli4_hba.nvmet_io_wait_cnt, + phba->sli4_hba.nvmet_io_wait_total); }else {if (!(phba->cfg_enable_fc4_type &LPFC_ENABLE_NVME ))return len;
drivers/scsi/lpfc/lpfc_init.c +2 -0
@@ -5825,6 +5825,7 @@ lpfc_sli4_driver_resource_setup(struct lpfc_hba *phba)INIT_LIST_HEAD (&phba->sli4_hba.lpfc_abts_nvme_buf_list);INIT_LIST_HEAD (&phba->sli4_hba.lpfc_abts_nvmet_ctx_list);INIT_LIST_HEAD (&phba->sli4_hba.lpfc_nvmet_ctx_list); +INIT_LIST_HEAD (&phba->sli4_hba.lpfc_nvmet_io_wait_list);/* Fast-path XRI aborted CQ Event work queue list */ INIT_LIST_HEAD (&phba->sli4_hba.sp_nvme_xri_aborted_work_queue);
@@ -5833,6 +5834,7 @@ lpfc_sli4_driver_resource_setup(struct lpfc_hba *phba)/* This abort list used by worker thread */ spin_lock_init (&phba->sli4_hba.sgl_list_lock);spin_lock_init (&phba->sli4_hba.nvmet_io_lock); +spin_lock_init (&phba->sli4_hba.nvmet_io_wait_lock);/* * Initialize driver internal slow-path work queues
drivers/scsi/lpfc/lpfc_nvmet.c +118 -20
175 lines not on this page · raw message
drivers/scsi/lpfc/lpfc_sli.c +4 -3
@@ -6513,9 +6513,9 @@ lpfc_set_host_data(struct lpfc_hba *phba, LPFC_MBOXQ_t *mbox) (phba->hba_flag & HBA_FCOE_MODE) ? "FCoE" : "FC"); } -static int +int lpfc_post_rq_buffer (struct lpfc_hba *phba,struct lpfc_queue *hrq, -struct lpfc_queue *drq,int count) +struct lpfc_queue *drq,int count,int idx) {int rc, i;struct lpfc_rqe hrqe;
@@ -6534,6 +6534,7 @@ lpfc_post_rq_buffer(struct lpfc_hba *phba, struct lpfc_queue *hrq,break ; rqb_buffer->hrq = hrq; rqb_buffer->drq = drq; + rqb_buffer->idx = idx;list_add_tail (&rqb_buffer->hbuf.list, &rqb_buf_list); }while (!list_empty (&rqb_buf_list)) {
@@ -6980,7 +6981,7 @@ lpfc_sli4_hba_setup(struct lpfc_hba *phba)lpfc_post_rq_buffer ( phba, phba->sli4_hba.nvmet_mrq_hdr[i], phba->sli4_hba.nvmet_mrq_data[i], -LPFC_NVMET_RQE_DEF_COUNT ); +LPFC_NVMET_RQE_DEF_COUNT , i); } }
drivers/scsi/lpfc/lpfc_sli4.h +5 -1
@@ -619,13 +619,16 @@ struct lpfc_sli4_hba {uint16_t els_xri_cnt;uint16_t nvmet_xri_cnt;uint16_t nvmet_ctx_cnt; +uint16_t nvmet_io_wait_cnt; +uint16_t nvmet_io_wait_total;struct list_head lpfc_els_sgl_list;struct list_head lpfc_abts_els_sgl_list;struct list_head lpfc_nvmet_sgl_list;struct list_head lpfc_abts_nvmet_ctx_list; -struct list_head lpfc_nvmet_ctx_list;struct list_head lpfc_abts_scsi_buf_list;struct list_head lpfc_abts_nvme_buf_list; +struct list_head lpfc_nvmet_ctx_list; +struct list_head lpfc_nvmet_io_wait_list;struct lpfc_sglq **lpfc_sglq_active_list;struct list_head lpfc_rpi_hdr_list;unsigned long *rpi_bmask;
@@ -657,6 +660,7 @@ struct lpfc_sli4_hba {spinlock_t abts_scsi_buf_list_lock;/* list of aborted SCSI IOs */ spinlock_t sgl_list_lock;/* list of aborted els IOs */ spinlock_t nvmet_io_lock; +spinlock_t nvmet_io_wait_lock;/* IOs waiting for ctx resources */ uint32_t physical_port;/* CPU to vector mapping information */
Signature · 1 line
2.11.0
PATCH v2 10/15 lpfc: Fix NVME I+T not registering NVME as a supported FC4 type
In reply to James SmartWhen the driver send the RPA command, it does not send supported FC4 Type NVME to the management server.
Encode NVME (type x28) in the AttribEntry in the RPA command.
--- drivers/scsi/lpfc/lpfc_ct.c | 1 + 1 file changed, 1 insertion(+)
drivers/scsi/lpfc/lpfc_ct.c +1 -0
@@ -2092,6 +2092,7 @@ lpfc_fdmi_port_attr_fc4type(struct lpfc_vport *vport, ae->un.AttrTypes[3 ] =0x02 ;/* Type 1 - ELS */ ae->un.AttrTypes[2 ] =0x01 ;/* Type 8 - FCP */ + ae->un.AttrTypes[6 ] =0x01 ;/* Type 40 - NVME */ ae->un.AttrTypes[7 ] =0x01 ;/* Type 32 - CT */ size =FOURBYTES +32 ; ad->AttrLen =cpu_to_be16 (size);
Signature · 1 line
2.11.0
PATCH v2 11/15 lpfc: Fix debugfs root inode "lpfc" not getting deleted on driver unload.
In reply to James SmartWhen unloading and reloading the driver, the driver fails to recreate the lpfc root inode in the debugfs tree.
The driver is incorrectly removing the lpfc root inode in lpfc_debugfs_terminate in the first driver instance that unloads and then sets the lpfc_debugfs_root global parameter to NULL. When the final driver instance unloads, the debugfs calls quietly ignore the remove on a NULL pointer. The bug is that the debugfs_remove call returns void so the driver doesn't know to correctly set the global parameter to NULL.
Base the debugfs_remove of the lpfc_debugfs_root parameter on lpfc_debugfs_hba_count because this parameter tracks the fnX instance tracked per driver instance.
--- drivers/scsi/lpfc/lpfc_debugfs.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-)
drivers/scsi/lpfc/lpfc_debugfs.c +4 -2
@@ -5866,8 +5866,10 @@ lpfc_debugfs_terminate(struct lpfc_vport *vport) atomic_dec(&lpfc_debugfs_hba_count); } -debugfs_remove (lpfc_debugfs_root);/* lpfc */ - lpfc_debugfs_root =NULL ; +if (atomic_read (&lpfc_debugfs_hba_count) ==0 ) { +debugfs_remove (lpfc_debugfs_root);/* lpfc */ + lpfc_debugfs_root =NULL ; + } }#endif return ;
Signature · 1 line
2.11.0
PATCH v2 12/15 lpfc: Cleanup entry_repost settings on SLI4 queues
In reply to James SmartToo many work items being processed in IRQ context take a lot of CPU time and cause problems.
With a recent change, we get out of the ISR after hitting entry_repost work items on a queue. However, the actual values for entry repost are still high. EQ is 128 and CQ is 128, this could translate into processing 128 * 128 (16384) work items under IRQ context.
Set entry_repost in the actual queue creation routine now. Limit EQ repost to 8 and CQ repost to 64 to further limit the amount of time spent in the IRQ.
Fix fof IRQ routines as well.
---
v2: portions of the original patch reworked:
rq/mrq entry_repost values - moved to
0004-lpfc-Fix-nvmet-RQ-resource-needs-for-large-block-wri.patch
eq/cq processing breaks- moved to
0006-lpfc-Fix-NMI-watchdog-assertions-when-running-nvmet-.patch
drivers/scsi/lpfc/lpfc_sli.c | 19 ++++++++----------- drivers/scsi/lpfc/lpfc_sli4.h | 6 ++++-- 2 files changed, 12 insertions(+), 13 deletions(-)
drivers/scsi/lpfc/lpfc_sli.c +8 -11
@@ -13922,17 +13922,10 @@ lpfc_sli4_queue_alloc(struct lpfc_hba *phba, uint32_t entry_size, } queue->entry_size = entry_size; queue->entry_count = entry_count; - -/* -* entry_repost is calculated based on the number of entries in the -* queue. This works out except for RQs. If buffers are NOT initially -* posted for every RQE, entry_repost should be adjusted accordingly. -*/ - queue->entry_repost = (entry_count >>3 ); -if (queue->entry_repost <LPFC_QUEUE_MIN_REPOST ) - queue->entry_repost =LPFC_QUEUE_MIN_REPOST ; queue->phba = phba; +/* entry_repost will be set during q creation */ +return queue; out_fail:lpfc_sli4_queue_free (queue);
@@ -14163,6 +14156,7 @@ lpfc_eq_create(struct lpfc_hba *phba, struct lpfc_queue *eq, uint32_t imax) status = -ENXIO ; eq->host_index =0 ; eq->hba_index =0 ; + eq->entry_repost =LPFC_EQ_REPOST ;mempool_free (mbox, phba->mbox_mem_pool);return status;
@@ -14236,9 +14230,9 @@ lpfc_cq_create(struct lpfc_hba *phba, struct lpfc_queue *cq,default :lpfc_printf_log (phba,KERN_ERR ,LOG_SLI ,"0361 Unsupported CQ count: " -"entry cnt %d sz %d pg cnt %d repost %d\n" , +"entry cnt %d sz %d pg cnt %d\n" , cq->entry_count, cq->entry_size, - cq->page_count, cq->entry_repost); + cq->page_count);if (cq->entry_count <256 ) { status = -EINVAL ; goto out;
@@ -14291,6 +14285,7 @@ lpfc_cq_create(struct lpfc_hba *phba, struct lpfc_queue *cq, cq->assoc_qid = eq->queue_id; cq->host_index =0 ; cq->hba_index =0 ; + cq->entry_repost =LPFC_CQ_REPOST ; out:mempool_free (mbox, phba->mbox_mem_pool);
@@ -14482,6 +14477,7 @@ lpfc_cq_create_set(struct lpfc_hba *phba, struct lpfc_queue **cqp, cq->assoc_qid = eq->queue_id; cq->host_index =0 ; cq->hba_index =0 ; + cq->entry_repost =LPFC_CQ_REPOST ; rc =0 ;list_for_each_entry (dmabuf, &cq->page_list, list) {
@@ -14730,6 +14726,7 @@ lpfc_mq_create(struct lpfc_hba *phba, struct lpfc_queue *mq, mq->subtype = subtype; mq->host_index =0 ; mq->hba_index =0 ; + mq->entry_repost =LPFC_MQ_REPOST ;/* link the mq onto the parent cq child list */ list_add_tail (&mq->list, &cq->child_list);
drivers/scsi/lpfc/lpfc_sli4.h +4 -2
@@ -24,7 +24,6 @@#define LPFC_XRI_EXCH_BUSY_WAIT_TMO 10000#define LPFC_XRI_EXCH_BUSY_WAIT_T1 10#define LPFC_XRI_EXCH_BUSY_WAIT_T2 30000 -#define LPFC_RELEASE_NOTIFICATION_INTERVAL 32#define LPFC_RPI_LOW_WATER_MARK 10#define LPFC_UNREG_FCF 1
@@ -155,8 +154,11 @@ struct lpfc_queue {uint32_t entry_count;/* Number of entries to support on the queue */ uint32_t entry_size;/* Size of each queue entry. */ uint32_t entry_repost;/* Count of entries before doorbell is rung */ -#define LPFC_QUEUE_MIN_REPOST 8 +#define LPFC_EQ_REPOST 8 +#define LPFC_MQ_REPOST 8 +#define LPFC_CQ_REPOST 64#define LPFC_RQ_REPOST 64 +#define LPFC_RELEASE_NOTIFICATION_INTERVAL 32/* For WQs */ uint32_t queue_id;/* Queue ID assigned by the hardware */ uint32_t assoc_qid;/* Queue ID associated with, for CQ/WQ/MQ */ uint32_t page_count;/* Number of pages allocated for this queue */
Signature · 1 line
2.11.0
PATCH v2 12/15 lpfc: Cleanup entry_repost settings on SLI4 queues
Quoted 24 lines · James Smart
Too many work items being processed in IRQ context take a lot of CPU time and cause problems.
With a recent change, we get out of the ISR after hitting
entry_repostwork items on a queue. However, the actual values for entry repost are still high. EQ is 128 and CQ is 128, this could translate into processing 128 * 128 (16384) work items under IRQ context.Set
entry_repostin the actual queue creation routine now. Limit EQ repost to 8 and CQ repost to 64 to further limit the amount of time spent in the IRQ.Fix fof IRQ routines as well.
Signed-off-by Dick Kennedy <dick.kennedy at broadcom.com>Signed-off-by James Smart <james.smart at broadcom.com>---
v2: portions of the original patch reworked:
rq/mrqentry_repostvalues - moved to
0004-lpfc-Fix-nvmet-RQ-resource-needs-for-large-block-wri.patch
eq/cq processing breaks- moved to
0006-lpfc-Fix-NMI-watchdog-assertions-when-running-nvmet-.patch
Quoted 4 lines
drivers/scsi/lpfc/lpfc_sli.c | 19 ++++++++----------- drivers/scsi/lpfc/lpfc_sli4.h | 6 ++++-- 2 files changed, 12 insertions(+), 13 deletions(-)
Cheers,
Hannes
Signature · 5 lines
Dr. Hannes Reinecke Teamlead Storage & Networking hare at suse.de +49 911 74053 688 SUSE LINUX GmbH, Maxfeldstr. 5, 90409 N?rnberg GF: F. Imend?rffer, J. Smithard, J. Guild, D. Upmanyu, G. Norton HRB 21284 (AG N?rnberg)
PATCH v2 13/15 lpfc: Fix NVMEI's handling of NVMET's PRLI response attributes
In reply to James SmartCode review of NVMEI's FC_PORT_ROLE_NVME_DISCOVERY looked wrong.
Discussions with storage architecture team clarified NVMEI's audit of the PRLI response port roles. Following up discussion with code review showed a few minor corrections were required - especially in anticipation of NVME auto discovery.
During PRLI, NVMEI should sent prli_init - which it it does. NVMET should send prli_tgt and prli_disc - which it does. When NVMEI receives a PRLI Response now, it audits the incoming target bits and stores the attributes in the corresponding NDLP. Later, when NVMEI registers the NVME rport, it uses the stored ndlp attributes to set the rport port_roles correctly.
--- drivers/scsi/lpfc/lpfc_disc.h | 1 + drivers/scsi/lpfc/lpfc_nportdisc.c | 6 ++++++ 2 files changed, 7 insertions(+)
drivers/scsi/lpfc/lpfc_disc.h +1 -0
@@ -90,6 +90,7 @@ struct lpfc_nodelist {#define NLP_FCP_INITIATOR 0x10/* entry is an FCP Initiator */ #define NLP_NVME_TARGET 0x20/* entry is a NVME Target */ #define NLP_NVME_INITIATOR 0x40/* entry is a NVME Initiator */ +#define NLP_NVME_DISCOVERY 0x80/* entry has NVME disc srvc */ uint16_t nlp_fc4_type;/* FC types node supports. */ /* Assigned from GID_FF, only
drivers/scsi/lpfc/lpfc_nportdisc.c +6 -0
@@ -1944,7 +1944,13 @@ lpfc_cmpl_prli_prli_issue(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp,/* Target driver cannot solicit NVME FB. */ if (bf_get_be32 (prli_tgt, nvpr)) { +/* Complete the nvme target roles. The transport +* needs to know if the rport is capable of +* discovery in addition to its role. +*/ ndlp->nlp_type |=NLP_NVME_TARGET ; +if (bf_get_be32 (prli_disc, nvpr)) + ndlp->nlp_type |=NLP_NVME_DISCOVERY ; if ((bf_get_be32(prli_fba, nvpr) == 1) && (bf_get_be32(prli_fb_sz, nvpr) > 0) && (phba->cfg_nvme_enable_fb) &&
Signature · 1 line
2.11.0
PATCH v2 14/15 lpfc: Add MDS Diagnostic support.
In reply to James SmartAdded code to support Cisco MDS loopback diagnostic. The diagnostics run various loopbacks including one which loops-back frame through the driver.
--- drivers/scsi/lpfc/lpfc.h | 2 + drivers/scsi/lpfc/lpfc_els.c | 7 +++ drivers/scsi/lpfc/lpfc_hbadisc.c | 3 +- drivers/scsi/lpfc/lpfc_hw4.h | 15 ++++- drivers/scsi/lpfc/lpfc_init.c | 13 ++++ drivers/scsi/lpfc/lpfc_sli.c | 131 ++++++++++++++++++++++++++++++++++++--- 6 files changed, 161 insertions(+), 10 deletions(-)
drivers/scsi/lpfc/lpfc.h +2 -0
@@ -675,6 +675,8 @@ struct lpfc_hba {/* INIT_LINK mailbox command */ #define LS_NPIV_FAB_SUPPORTED 0x2/* Fabric supports NPIV */ #define LS_IGNORE_ERATT 0x4/* intr handler should ignore ERATT */ +#define LS_MDS_LINK_DOWN 0x8/* MDS Diagnostics Link Down */ +#define LS_MDS_LOOPBACK 0x16/* MDS Diagnostics Link Up (Loopback) */ uint32_t hba_flag;/* hba generic flags */ #define HBA_ERATT_HANDLED 0x1/* This flag is set when eratt handled */
drivers/scsi/lpfc/lpfc_els.c +7 -0
@@ -1047,6 +1047,13 @@ lpfc_cmpl_els_flogi(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb, irsp->ulpStatus, irsp->un.ulpWord[4 ], irsp->ulpTimeout); + +/* If this is not a loop open failure, bail out */ +if (!(irsp->ulpStatus ==IOSTAT_LOCAL_REJECT && + ((irsp->un.ulpWord[4 ] &IOERR_PARAM_MASK ) == +IOERR_LOOP_OPEN_FAILURE ))) + goto flogifail; +/* FLOGI failed, so there is no fabric */ spin_lock_irq (shost->host_lock); vport->fc_flag &= ~(FC_FABRIC |FC_PUBLIC_LOOP );
drivers/scsi/lpfc/lpfc_hbadisc.c +2 -1
@@ -701,7 +701,8 @@ lpfc_work_done(struct lpfc_hba *phba)/* Set the lpfc data pending flag */ set_bit (LPFC_DATA_READY , &phba->data_flags); }else { -if (phba->link_state >=LPFC_LINK_UP ) { +if (phba->link_state >=LPFC_LINK_UP || + phba->link_flag &LS_MDS_LOOPBACK ) { pring->flag &= ~LPFC_DEFERRED_RING_EVENT ; lpfc_sli_handle_slow_ring_event(phba, pring, (status &
drivers/scsi/lpfc/lpfc_hw4.h +14 -1
@@ -4421,6 +4421,19 @@ struct fcp_treceive64_wqe { };#define TXRDY_PAYLOAD_LEN 12 +#define CMD_SEND_FRAME 0xE1 + +struct send_frame_wqe { +struct ulp_bde64 bde;/* words 0-2 */ +uint32_t frame_len;/* word 3 */ +uint32_t fc_hdr_wd0;/* word 4 */ +uint32_t fc_hdr_wd1;/* word 5 */ +struct wqe_common wqe_com;/* words 6-11 */ +uint32_t fc_hdr_wd2;/* word 12 */ +uint32_t fc_hdr_wd3;/* word 13 */ +uint32_t fc_hdr_wd4;/* word 14 */ +uint32_t fc_hdr_wd5;/* word 15 */ +};union lpfc_wqe {uint32_t words[16 ];
@@ -4439,7 +4452,7 @@ union lpfc_wqe {struct fcp_trsp64_wqe fcp_trsp;struct fcp_tsend64_wqe fcp_tsend;struct fcp_treceive64_wqe fcp_treceive; - +struct send_frame_wqe send_frame; };union lpfc_wqe128 {
drivers/scsi/lpfc/lpfc_init.c +13 -0
@@ -4540,6 +4540,19 @@ lpfc_sli4_async_fc_evt(struct lpfc_hba *phba, struct lpfc_acqe_fc_la *acqe_fc) pmb->vport = phba->pport;if (phba->sli4_hba.link_state.status !=LPFC_FC_LA_TYPE_LINK_UP ) { + phba->link_flag &= ~(LS_MDS_LINK_DOWN |LS_MDS_LOOPBACK ); + +switch (phba->sli4_hba.link_state.status) { +case LPFC_FC_LA_TYPE_MDS_LINK_DOWN : + phba->link_flag |=LS_MDS_LINK_DOWN ; +break ; +case LPFC_FC_LA_TYPE_MDS_LOOPBACK : + phba->link_flag |=LS_MDS_LOOPBACK ; +break ; +default : +break ; + } +/* Parse and translate status field */ mb = &pmb->u.mb; mb->mbxStatus = lpfc_sli4_parse_latt_fault(phba,
drivers/scsi/lpfc/lpfc_sli.c +123 -8
198 lines not on this page · raw message
Signature · 1 line
2.11.0
PATCH v2 15/15 lpfc: update version to 11.2.0.14
In reply to James SmartChange driver version to 11.2.0.14
--- drivers/scsi/lpfc/lpfc_version.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
drivers/scsi/lpfc/lpfc_version.h +1 -1
@@ -20,7 +20,7 @@* included with this package. * *******************************************************************/ -#define LPFC_DRIVER_VERSION "11.2.0.12" +#define LPFC_DRIVER_VERSION "11.2.0.14"#define LPFC_DRIVER_NAME "lpfc"/* Used for SLI 2/3 */
Signature · 1 line
2.11.0
Martin K. Petersen
Appliedv2 cover lpfc updates for 11.2.0.14
In reply to James SmartJames,
Quoted 8 lines
This patch set provides a number of bug fixes, code cleanups, and error handling, mostly in the nvme area of lpfc.
There is one new feature in the series to add MDS diagnostics support
The patches were cut against the Martin's 4.12/scsi-fixes tree. There are no outside dependencies. The patches should merge via Martin's tree.
Applied to 4.12/scsi-fixes.
Please don't forget to send to linux-scsi. Otherwise the patches won't get captured in patchwork.
Signature · 1 line
Martin K. Petersen Oracle Linux Engineering