Skip to contentSkip to conversation

Patches for 5.5 rc

PATCH In review Dennis Dalessandro 2 participants 7 messages Dec 19, 2019 – Jan 3, 2020

Subsystems (3)
lore
Share & tools
lore
#0 lore raw

cover Patches for 5.5 rc

The first two patches I wouldn't ordinarily have sent for -rc but I noticed we did this in the previous -rc post, add an API because we use it in the next fix. That's what the first two patches here do. It's understandable if you are skeptical that those are OK for -rc and in which case you can drop and we can send for next.

---

Kaike Wan (1):
IB/hfi1: Don't cancel unused work item

Michael J. Ruhl (1):
IB/hfi1: List all receive contexts from debugfs

Mike Marciniszyn (2):
IB/hfi1: Add accessor API routines to access context members IB/rdmavt: Correct comments in rdmavt_qp.h header

 drivers/infiniband/hw/hfi1/chip.c        |   27 ++----
 drivers/infiniband/hw/hfi1/common.h      |    3 +
 drivers/infiniband/hw/hfi1/debugfs.c     |    2 
 drivers/infiniband/hw/hfi1/driver.c      |   86 +++++++++-----------
 drivers/infiniband/hw/hfi1/file_ops.c    |   12 +--
 drivers/infiniband/hw/hfi1/hfi.h         |  129 ++++++++++++++++++++++++++++++
 drivers/infiniband/hw/hfi1/init.c        |    6 +
 drivers/infiniband/hw/hfi1/iowait.c      |    4 +
 drivers/infiniband/hw/hfi1/trace_ctxts.h |    2 
 drivers/infiniband/hw/hfi1/trace_rx.h    |   13 +--
 drivers/infiniband/hw/hfi1/vnic_main.c   |    2 
 drivers/infiniband/sw/rdmavt/rc.c        |    9 ++
 include/rdma/rdmavt_qp.h                 |   22 -----
 13 files changed, 205 insertions(+), 112 deletions(-)

--
-Denny

#1 lore raw

PATCH 1/4 IB/hfi1: Add accessor API routines to access context members

From: Mike Marciniszyn <[email protected]>

This patch adds a set of accessor routines to access context members.

---
 drivers/infiniband/hw/hfi1/chip.c        |   27 ++----
 drivers/infiniband/hw/hfi1/common.h      |    3 +
 drivers/infiniband/hw/hfi1/driver.c      |   74 +++++++----------
 drivers/infiniband/hw/hfi1/file_ops.c    |   12 +--
 drivers/infiniband/hw/hfi1/hfi.h         |  129 ++++++++++++++++++++++++++++++
 drivers/infiniband/hw/hfi1/init.c        |    6 +
 drivers/infiniband/hw/hfi1/trace_ctxts.h |    2 
 drivers/infiniband/hw/hfi1/trace_rx.h    |   13 +--
 drivers/infiniband/hw/hfi1/vnic_main.c   |    2 
 9 files changed, 183 insertions(+), 85 deletions(-)
drivers/infiniband/hw/hfi1/chip.c +10 -17
index 9b1fb84..b4fe37a 100644
@@ -6862,7 +6862,7 @@ static void rxe_kernel_unfreeze(struct hfi1_devdata *dd)
 		}
 		rcvmask = HFI1_RCVCTRL_CTXT_ENB;
 		/* HFI1_RCVCTRL_TAILUPD_[ENB|DIS] needs to be set explicitly */
-		rcvmask |= rcd->rcvhdrtail_kvaddr ?
+		rcvmask |= hfi1_rcvhdrtail_kvaddr(rcd) ?
 			HFI1_RCVCTRL_TAILUPD_ENB : HFI1_RCVCTRL_TAILUPD_DIS;
 		hfi1_rcvctrl(dd, rcvmask, rcd);
 		hfi1_rcd_put(rcd);
@@ -8394,20 +8394,13 @@ void force_recv_intr(struct hfi1_ctxtdata *rcd)
 static inline int check_packet_present(struct hfi1_ctxtdata *rcd)
 {
 	u32 tail;
-	int present;
-
-	if (!rcd->rcvhdrtail_kvaddr)
-		present = (rcd->seq_cnt ==
-				rhf_rcv_seq(rhf_to_cpu(get_rhf_addr(rcd))));
-	else /* is RDMA rtail */
-		present = (rcd->head != get_rcvhdrtail(rcd));
 
-	if (present)
+	if (hfi1_packet_present(rcd))
 		return 1;
 
 	/* fall back to a CSR read, correct indpendent of DMA_RTAIL */
 	tail = (u32)read_uctxt_csr(rcd->dd, rcd->ctxt, RCV_HDR_TAIL);
-	return rcd->head != tail;
+	return hfi1_rcd_head(rcd) != tail;
 }
 
 /*
@@ -10049,7 +10042,7 @@ u32 lrh_max_header_bytes(struct hfi1_devdata *dd)
 	 * the first kernel context would have been allocated by now so
 	 * we are guaranteed a valid value.
 	 */
-	return (dd->rcd[0]->rcvhdrqentsize - 2/*PBC/RHF*/ + 1/*ICRC*/) << 2;
+	return (get_hdrqentsize(dd->rcd[0]) - 2/*PBC/RHF*/ + 1/*ICRC*/) << 2;
 }
 
 /*
@@ -10094,7 +10087,7 @@ static void set_send_length(struct hfi1_pportdata *ppd)
 		thres = min(sc_percent_to_threshold(dd->vld[i].sc, 50),
 			    sc_mtu_to_threshold(dd->vld[i].sc,
 						dd->vld[i].mtu,
-						dd->rcd[0]->rcvhdrqentsize));
+						get_hdrqentsize(dd->rcd[0])));
 		for (j = 0; j < INIT_SC_PER_VL; j++)
 			sc_set_cr_threshold(
 					pio_select_send_context_vl(dd, j, i),
@@ -11821,7 +11814,7 @@ u32 hdrqempty(struct hfi1_ctxtdata *rcd)
 	head = (read_uctxt_csr(rcd->dd, rcd->ctxt, RCV_HDR_HEAD)
 		& RCV_HDR_HEAD_HEAD_SMASK) >> RCV_HDR_HEAD_HEAD_SHIFT;
 
-	if (rcd->rcvhdrtail_kvaddr)
+	if (hfi1_rcvhdrtail_kvaddr(rcd))
 		tail = get_rcvhdrtail(rcd);
 	else
 		tail = read_uctxt_csr(rcd->dd, rcd->ctxt, RCV_HDR_TAIL);
@@ -11886,13 +11879,13 @@ void hfi1_rcvctrl(struct hfi1_devdata *dd, unsigned int op,
 		/* reset the tail and hdr addresses, and sequence count */
 		write_kctxt_csr(dd, ctxt, RCV_HDR_ADDR,
 				rcd->rcvhdrq_dma);
-		if (rcd->rcvhdrtail_kvaddr)
+		if (hfi1_rcvhdrtail_kvaddr(rcd))
 			write_kctxt_csr(dd, ctxt, RCV_HDR_TAIL_ADDR,
 					rcd->rcvhdrqtailaddr_dma);
-		rcd->seq_cnt = 1;
+		hfi1_set_seq_cnt(rcd, 1);
 
 		/* reset the cached receive header queue head value */
-		rcd->head = 0;
+		hfi1_set_rcd_head(rcd, 0);
 
 		/*
 		 * Zero the receive header queue so we don't get false
@@ -11972,7 +11965,7 @@ void hfi1_rcvctrl(struct hfi1_devdata *dd, unsigned int op,
 			      IS_RCVAVAIL_START + rcd->ctxt, false);
 		rcvctrl &= ~RCV_CTXT_CTRL_INTR_AVAIL_SMASK;
 	}
-	if ((op & HFI1_RCVCTRL_TAILUPD_ENB) && rcd->rcvhdrtail_kvaddr)
+	if ((op & HFI1_RCVCTRL_TAILUPD_ENB) && hfi1_rcvhdrtail_kvaddr(rcd))
 		rcvctrl |= RCV_CTXT_CTRL_TAIL_UPD_SMASK;
 	if (op & HFI1_RCVCTRL_TAILUPD_DIS) {
 		/* See comment on RcvCtxtCtrl.TailUpd above */
drivers/infiniband/hw/hfi1/common.h +3 -0
index d47da7b..40a1ff0 100644
@@ -323,6 +323,9 @@ struct diag_pkt {
 /* RHF receive type error - bypass packet errors */
 #define RHF_RTE_BYPASS_NO_ERR		0x0
 
+/* MAX RcvSEQ */
+#define RHF_MAX_SEQ 13
+
 /* IB - LRH header constants */
 #define HFI1_LRH_GRH 0x0003      /* 1. word of IB LRH - next header: GRH */
 #define HFI1_LRH_BTH 0x0002      /* 1. word of IB LRH - next header: BTH */
drivers/infiniband/hw/hfi1/driver.c +29 -45

185 lines not on this page · raw message

drivers/infiniband/hw/hfi1/file_ops.c +6 -6
index 7c5e3fb..bef6946 100644
@@ -505,12 +505,12 @@ static int hfi1_file_mmap(struct file *fp, struct vm_area_struct *vma)
 			ret = -EINVAL;
 			goto done;
 		}
-		if ((flags & VM_WRITE) || !uctxt->rcvhdrtail_kvaddr) {
+		if ((flags & VM_WRITE) || !hfi1_rcvhdrtail_kvaddr(uctxt)) {
 			ret = -EPERM;
 			goto done;
 		}
 		memlen = PAGE_SIZE;
-		memvirt = (void *)uctxt->rcvhdrtail_kvaddr;
+		memvirt = (void *)hfi1_rcvhdrtail_kvaddr(uctxt);
 		flags &= ~VM_MAYWRITE;
 		break;
 	case SUBCTXT_UREGS:
@@ -1090,7 +1090,7 @@ static void user_init(struct hfi1_ctxtdata *uctxt)
 	 * don't have to wait to be sure the DMA update has happened
 	 * (chip resets head/tail to 0 on transition to enable).
 	 */
-	if (uctxt->rcvhdrtail_kvaddr)
+	if (hfi1_rcvhdrtail_kvaddr(uctxt))
 		clear_rcvhdrtail(uctxt);
 
 	/* Setup J_KEY before enabling the context */
@@ -1154,8 +1154,8 @@ static int get_ctxt_info(struct hfi1_filedata *fd, unsigned long arg, u32 len)
 	cinfo.send_ctxt = uctxt->sc->hw_context;
 
 	cinfo.egrtids = uctxt->egrbufs.alloced;
-	cinfo.rcvhdrq_cnt = uctxt->rcvhdrq_cnt;
-	cinfo.rcvhdrq_entsize = uctxt->rcvhdrqentsize << 2;
+	cinfo.rcvhdrq_cnt = get_hdrq_cnt(uctxt);
+	cinfo.rcvhdrq_entsize = get_hdrqentsize(uctxt) << 2;
 	cinfo.sdma_ring_size = fd->cq->nentries;
 	cinfo.rcvegr_size = uctxt->egrbufs.rcvtid_size;
 
@@ -1543,7 +1543,7 @@ static int manage_rcvq(struct hfi1_ctxtdata *uctxt, u16 subctxt,
 		 * always resets it's tail register back to 0 on a
 		 * transition from disabled to enabled.
 		 */
-		if (uctxt->rcvhdrtail_kvaddr)
+		if (hfi1_rcvhdrtail_kvaddr(uctxt))
 			clear_rcvhdrtail(uctxt);
 		rcvctrl_op = HFI1_RCVCTRL_CTXT_ENB;
 	} else {
drivers/infiniband/hw/hfi1/hfi.h +127 -2

155 lines not on this page · raw message

drivers/infiniband/hw/hfi1/init.c +3 -3
index 26b792b..3e56ee3 100644
@@ -154,7 +154,7 @@ static int hfi1_create_kctxt(struct hfi1_devdata *dd,
 	/* Control context must use DMA_RTAIL */
 	if (rcd->ctxt == HFI1_CTRL_CTXT)
 		rcd->flags |= HFI1_CAP_DMA_RTAIL;
-	rcd->seq_cnt = 1;
+	hfi1_set_seq_cnt(rcd, 1);
 
 	rcd->sc = sc_alloc(dd, SC_ACK, rcd->rcvhdrqentsize, dd->node);
 	if (!rcd->sc) {
@@ -1149,9 +1149,9 @@ void hfi1_free_ctxtdata(struct hfi1_devdata *dd, struct hfi1_ctxtdata *rcd)
 		dma_free_coherent(&dd->pcidev->dev, rcvhdrq_size(rcd),
 				  rcd->rcvhdrq, rcd->rcvhdrq_dma);
 		rcd->rcvhdrq = NULL;
-		if (rcd->rcvhdrtail_kvaddr) {
+		if (hfi1_rcvhdrtail_kvaddr(rcd)) {
 			dma_free_coherent(&dd->pcidev->dev, PAGE_SIZE,
-					  (void *)rcd->rcvhdrtail_kvaddr,
+					  (void *)hfi1_rcvhdrtail_kvaddr(rcd),
 					  rcd->rcvhdrqtailaddr_dma);
 			rcd->rcvhdrtail_kvaddr = NULL;
 		}
drivers/infiniband/hw/hfi1/trace_ctxts.h +1 -1
index e00c8a7..b5fc5c6 100644
@@ -80,7 +80,7 @@
 			   __entry->credits = uctxt->sc->credits;
 			   __entry->hw_free = le64_to_cpu(*uctxt->sc->hw_free);
 			   __entry->piobase = uctxt->sc->base_addr;
-			   __entry->rcvhdrq_cnt = uctxt->rcvhdrq_cnt;
+			   __entry->rcvhdrq_cnt = get_hdrq_cnt(uctxt);
 			   __entry->rcvhdrq_dma = uctxt->rcvhdrq_dma;
 			   __entry->eager_cnt = uctxt->egrbufs.alloced;
 			   __entry->rcvegr_dma = uctxt->egrbufs.rcvtids[0].dma;
drivers/infiniband/hw/hfi1/trace_rx.h +3 -10
index 3cec960..ebce81c 100644
@@ -107,18 +107,11 @@
 	    TP_fast_assign(DD_DEV_ASSIGN(dd);
 			__entry->ctxt = rcd->ctxt;
 			if (rcd->do_interrupt ==
-			    &handle_receive_interrupt) {
+			    &handle_receive_interrupt)
 				__entry->slow_path = 1;
-				__entry->dma_rtail = 0xFF;
-			} else if (rcd->do_interrupt ==
-					&handle_receive_interrupt_dma_rtail){
-				__entry->dma_rtail = 1;
+			else
 				__entry->slow_path = 0;
-			} else if (rcd->do_interrupt ==
-					&handle_receive_interrupt_nodma_rtail) {
-				__entry->dma_rtail = 0;
-				__entry->slow_path = 0;
-			}
+			__entry->dma_rtail = get_dma_rtail_setting(rcd);
 			),
 	    TP_printk("[%s] ctxt %d SlowPath: %d DmaRtail: %d",
 		      __get_str(dev),
drivers/infiniband/hw/hfi1/vnic_main.c +1 -1
index b49e60e..6b14581 100644
@@ -78,7 +78,7 @@ static int setup_vnic_ctxt(struct hfi1_devdata *dd, struct hfi1_ctxtdata *uctxt)
 	if (ret)
 		goto done;
 
-	if (uctxt->rcvhdrtail_kvaddr)
+	if (hfi1_rcvhdrtail_kvaddr(uctxt))
 		clear_rcvhdrtail(uctxt);
 
 	rcvctrl_ops = HFI1_RCVCTRL_CTXT_ENB;
#2 lore raw

PATCH 2/4 IB/hfi1: List all receive contexts from debugfs

In reply to Dennis Dalessandro

From: Michael J. Ruhl <[email protected]>

The current debugfs output for receive contexts (rcds), stops after the kernel receive contexts have been displayed. This is not enough information to fully diagnose packet drops.

Display all of the receive contexts.

Augment the output with some more context information.

Limit the ring buffer header output to 5 entries to avoid overextending the sequential file output.

Fixes bf808b5039c ("IB/hfi1: Add kernel receive context info to debugfs")
---
 drivers/infiniband/hw/hfi1/debugfs.c |    2 +-
 drivers/infiniband/hw/hfi1/driver.c  |   12 +++++++++---
 2 files changed, 10 insertions(+), 4 deletions(-)
drivers/infiniband/hw/hfi1/debugfs.c +1 -1
index d268bf9..4633a0c 100644
@@ -379,7 +379,7 @@ static void *_rcds_seq_next(struct seq_file *s, void *v, loff_t *pos)
 	struct hfi1_devdata *dd = dd_from_dev(ibd);
 
 	++*pos;
-	if (!dd->rcd || *pos >= dd->n_krcv_queues)
+	if (!dd->rcd || *pos >= dd->num_rcv_contexts)
 		return NULL;
 	return pos;
 }
drivers/infiniband/hw/hfi1/driver.c +9 -3
index cbc5219..8374922 100644
@@ -1726,23 +1726,29 @@ static int process_receive_invalid(struct hfi1_packet *packet)
 	return RHF_RCV_CONTINUE;
 }
 
+#define HFI1_RCVHDR_DUMP_MAX	5
+
 void seqfile_dump_rcd(struct seq_file *s, struct hfi1_ctxtdata *rcd)
 {
 	struct hfi1_packet packet;
 	struct ps_mdata mdata;
+	int i;
 
-	seq_printf(s, "Rcd %u: RcvHdr cnt %u entsize %u %s head %llu tail %llu\n",
+	seq_printf(s, "Rcd %u: RcvHdr cnt %u entsize %u %s ctrl 0x%08llx status 0x%08llx, head %llu tail %llu  sw head %u\n",
 		   rcd->ctxt, get_hdrq_cnt(rcd), get_hdrqentsize(rcd),
 		   get_dma_rtail_setting(rcd) ?
 		   "dma_rtail" : "nodma_rtail",
+		   read_kctxt_csr(rcd->dd, rcd->ctxt, RCV_CTXT_CTRL),
+		   read_kctxt_csr(rcd->dd, rcd->ctxt, RCV_CTXT_STATUS),
 		   read_uctxt_csr(rcd->dd, rcd->ctxt, RCV_HDR_HEAD) &
 		   RCV_HDR_HEAD_HEAD_MASK,
-		   read_uctxt_csr(rcd->dd, rcd->ctxt, RCV_HDR_TAIL));
+		   read_uctxt_csr(rcd->dd, rcd->ctxt, RCV_HDR_TAIL),
+		   rcd->head);
 
 	init_packet(rcd, &packet);
 	init_ps_mdata(&mdata, &packet);
 
-	while (1) {
+	for (i = 0; i < HFI1_RCVHDR_DUMP_MAX; i++) {
 		__le32 *rhf_addr = (__le32 *)rcd->rcvhdrq + mdata.ps_head +
 					 rcd->rhf_offset;
 		struct ib_header *hdr;
#3 lore raw

PATCH 3/4 IB/rdmavt: Correct comments in rdmavt_qp.h header

In reply to Dennis Dalessandro

From: Mike Marciniszyn <[email protected]>

Comments need to be with the definition of rvt_restart_sge().

Other comments were duplicated in sw/rdmavt/rc.c and were removed.

Fixes 385156c5f2a6 ("IB/hfi: Move RC functions into a header file")
---
 drivers/infiniband/sw/rdmavt/rc.c |    9 ++++++++-
 include/rdma/rdmavt_qp.h          |   22 +---------------------
 2 files changed, 9 insertions(+), 22 deletions(-)
drivers/infiniband/sw/rdmavt/rc.c +8 -1
index 890d7b7..977906c 100644
@@ -195,7 +195,14 @@ void rvt_get_credit(struct rvt_qp *qp, u32 aeth)
 }
 EXPORT_SYMBOL(rvt_get_credit);
 
-/* rvt_restart_sge - rewind the sge state for a wqe */
+/**
+ * rvt_restart_sge - rewind the sge state for a wqe
+ * @ss: the sge state pointer
+ * @wqe: the wqe to rewind
+ * @len: the data length from the start of the wqe in bytes
+ *
+ * Returns the remaining data length.
+ */
 u32 rvt_restart_sge(struct rvt_sge_state *ss, struct rvt_swqe *wqe, u32 len)
 {
 	ss->sge = wqe->sg_list[0];
include/rdma/rdmavt_qp.h +1 -21
index b550ae8..0d5c70e 100644
@@ -640,34 +640,14 @@ static inline int rvt_cmp_msn(u32 a, u32 b)
 	return (((int)a) - ((int)b)) << 8;
 }
 
-/**
- * rvt_compute_aeth - compute the AETH (syndrome + MSN)
- * @qp: the queue pair to compute the AETH for
- *
- * Returns the AETH.
- */
 __be32 rvt_compute_aeth(struct rvt_qp *qp);
 
-/**
- * rvt_get_credit - flush the send work queue of a QP
- * @qp: the qp who's send work queue to flush
- * @aeth: the Acknowledge Extended Transport Header
- *
- * The QP s_lock should be held.
- */
 void rvt_get_credit(struct rvt_qp *qp, u32 aeth);
 
-/**
- * rvt_restart_sge - rewind the sge state for a wqe
- * @ss: the sge state pointer
- * @wqe: the wqe to rewind
- * @len: the data length from the start of the wqe in bytes
- *
- * Returns the remaining data length.
- */
 u32 rvt_restart_sge(struct rvt_sge_state *ss, struct rvt_swqe *wqe, u32 len);
 
 /**
+ * rvt_div_round_up_mtu - round up divide
  * @qp - the qp pair
  * @len - the length
  *
#4 lore raw

PATCH 4/4 IB/hfi1: Don't cancel unused work item

In reply to Dennis Dalessandro

From: Kaike Wan <[email protected]>

In iowait structure, two iowait_work entries were included to queue a given object: one for normal IB operations, and the other for TID RDMA operations. For non-TID RDMA operations, the iowait_work structure for TID RDMA is initialized to contain a NULL function (not used). When the QP is reset, the function iowait_cancel_work will be called to cancel any pending work. The problem is that this function will call cancel_work_sync for both iowait_work entries, even though the one for TID RDMA is not used at all. Eventually, the call cascades to __flush_work, wherein a WARN_ON will be triggered due to the fact that work->func is NULL.

The WARN_ON was introduced in the following commit:
commit 4d43d395fed1 ("workqueue: Try to catch flush_work() without INIT_WORK().")

This patch fixes the issue by making sure that a work function is present for TID RDMA before calling cancel_work_sync in iowait_cancel_work.

Fixes 5da0fc9dbf89 ("IB/hfi1: Prepare resource waits for dual leg")
---
 drivers/infiniband/hw/hfi1/iowait.c |    4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)
drivers/infiniband/hw/hfi1/iowait.c +3 -1
index adb4a1b..5836fe7 100644
@@ -81,7 +81,9 @@ void iowait_init(struct iowait *wait, u32 tx_limit,
 void iowait_cancel_work(struct iowait *w)
 {
 	cancel_work_sync(&iowait_get_ib_work(w)->iowork);
-	cancel_work_sync(&iowait_get_tid_work(w)->iowork);
+	/* Make sure that the iowork for TID RDMA is used */
+	if (iowait_get_tid_work(w)->iowork.func)
+		cancel_work_sync(&iowait_get_tid_work(w)->iowork);
 }
 
 /**
#5 lore raw
In reply to Dennis Dalessandro
Quoted 5 lines · Dennis Dalessandro

The first two patches I wouldn't ordinarily have sent for -rc but I noticed we did this in the previous -rc post, add an API because we use it in the next fix. That's what the first two patches here do. It's understandable if you are skeptical that those are OK for -rc and in which case you can drop and we can send for next.

The only one that looks RC worthy to me is:

Quoted 7 lines

Kaike Wan (1):
IB/hfi1: Don't cancel unused work item

Mike Marciniszyn (2):
IB/hfi1: Add accessor API routines to access context members IB/rdmavt: Correct comments in rdmavt_qp.h header

How is correcting comments a needed fix?

Jason

#6 lore raw
In reply to Dennis Dalessandro
Quoted 9 lines · Dennis Dalessandro

The first two patches I wouldn't ordinarily have sent for -rc but I noticed we did this in the previous -rc post, add an API because we use it in the next fix. That's what the first two patches here do. It's understandable if you are skeptical that those are OK for -rc and in which case you can drop and we can send for next.

Kaike Wan (1):
IB/hfi1: Don't cancel unused work item

As I said before, I took this one to -rc

Quoted 2 lines

Michael J. Ruhl (1):
IB/hfi1: List all receive contexts from debugfs

This version of the patch now applies cleanly, so please disregard my earlier email

Quoted 3 lines

Mike Marciniszyn (2):
IB/hfi1: Add accessor API routines to access context members IB/rdmavt: Correct comments in rdmavt_qp.h header

These three are applied to -next

Thanks,
Jason