Skip to contentSkip to conversation

Introduce mbox tracepoints for Octeontx2

PATCH Superseded [email protected] 2 participants 5 messages Sep 21, 2020 – Sep 22, 2020

Subsystems (4)

v2 of 3 Superseded Latest is v3 →

lore
Share & tools
lore
#0 lore raw

v2 cover Introduce mbox tracepoints for Octeontx2

From: Subbaraya Sundeep <[email protected]>

This patchset adds tracepoints support for mailbox. In Octeontx2, PFs and VFs need to communicate with AF for allocating and freeing resources. Once all the configuration is done by AF for a PF/VF then packet I/O can happen on PF/VF queues. When an interface
is brought up many mailbox messages are sent
to AF for initializing queues. Say a VF is brought up then each message is sent to PF and PF forwards to AF and response also traverses from AF to PF and then VF. To aid debugging, tracepoints are added at places where messages are allocated, sent and message interrupts. Below is the trace of one of the messages from VF to AF and AF response back to VF:

~ # echo 1 > /sys/kernel/tracing/events/rvu/enable ~ # ifconfig eth20 up

[  279.379559] eth20 NIC Link is UP 10000 Mbps Full duplex

~ # cat /sys/kernel/tracing/trace
# tracer: nop
#
# entries-in-buffer/entries-written: 880/880 #P:4
#
# _-----=> irqs-off
# / _----=> need-resched

#                            | / _---=> hardirq/softirq
#                            || / _--=> preempt-depth
#                            ||| /     delay
#           TASK-PID   CPU#  ||||    TIMESTAMP  FUNCTION
#              | |       |   ||||       |         |
        ifconfig-171   [000] ....   275.753345: otx2_msg_alloc: [0002:02:00.1] msg:(0x400) size:40

        ifconfig-171   [000] ...1   275.753347: otx2_msg_send: [0002:02:00.1] sent 1 msg(s) of size:48

          <idle>-0     [001] dNh1   275.753356: otx2_msg_interrupt: [0002:02:00.0] mbox interrupt VF(s) to PF (0x1)

    kworker/u9:1-90    [001] ...1   275.753364: otx2_msg_send: [0002:02:00.0] sent 1 msg(s) of size:48

    kworker/u9:1-90    [001] d.h.   275.753367: otx2_msg_interrupt: [0002:01:00.0] mbox interrupt PF(s) to AF (0x2)

    kworker/u9:2-167   [002] ....   275.753535: otx2_msg_process: [0002:01:00.0] msg:(0x400) error:0

    kworker/u9:2-167   [002] ...1   275.753537: otx2_msg_send: [0002:01:00.0] sent 1 msg(s) of size:32

          <idle>-0     [003] d.h1   275.753543: otx2_msg_interrupt: [0002:02:00.0] mbox interrupt AF to PF (0x1)

          <idle>-0     [001] d.h2   275.754376: otx2_msg_interrupt: [0002:02:00.1] mbox interrupt PF to VF (0x1)

v2 changes:
Removed otx2_msg_err tracepoint since it is similar to devlink_hwerr and it will be used instead when devlink supported is added.

Subbaraya Sundeep (2):
octeontx2-af: Introduce tracepoints for mailbox
octeontx2-pf: Add tracepoints for PF/VF mailbox

 drivers/net/ethernet/marvell/octeontx2/af/Makefile |   3 +-
 drivers/net/ethernet/marvell/octeontx2/af/mbox.c   |  11 ++-
 drivers/net/ethernet/marvell/octeontx2/af/rvu.c    |   7 ++
 .../net/ethernet/marvell/octeontx2/af/rvu_cgx.c    |   2 +
 .../net/ethernet/marvell/octeontx2/af/rvu_trace.c  |  14 +++
 .../net/ethernet/marvell/octeontx2/af/rvu_trace.h  | 103 +++++++++++++++++++++
 .../ethernet/marvell/octeontx2/nic/otx2_common.h   |   2 +
 .../net/ethernet/marvell/octeontx2/nic/otx2_pf.c   |   6 ++
 .../net/ethernet/marvell/octeontx2/nic/otx2_vf.c   |   2 +
 9 files changed, 148 insertions(+), 2 deletions(-)
 create mode 100644 drivers/net/ethernet/marvell/octeontx2/af/rvu_trace.c
 create mode 100644 drivers/net/ethernet/marvell/octeontx2/af/rvu_trace.h
Signature · 1 line
2.7.4
#1 lore raw

PATCH v2 1/2 octeontx2-af: Introduce tracepoints for mailbox

From: Subbaraya Sundeep <[email protected]>

Added tracepoints in mailbox code so that the mailbox operations like message allocation, sending message and message interrupts are traced. Also the mailbox errors occurred like timeout or wrong responses are traced.
These will help in debugging mailbox issues.

Here's an example output showing one of the mailbox messages sent by PF to AF and AF responding to it:

~# mount -t tracefs none /sys/kernel/tracing/
~# echo 1 > /sys/kernel/tracing/events/rvu/enable
~# ifconfig eth0 up
~# cat /sys/kernel/tracing/trace

~# cat /sys/kernel/tracing/trace
tracer: nop

		      _-----=> irqs-off
		     / _----=> need-resched
		    | / _---=> hardirq/softirq
		    || / _--=> preempt-depth
		    ||| /     delay
   TASK-PID   CPU#  ||||    TIMESTAMP  FUNCTION
      | |       |   ||||       |         |

ifconfig-2382 [002] .... 756.161892: otx2_msg_alloc: [0002:02:00.0] msg:(0x400) size:40

ifconfig-2382 [002] ...1 756.161895: otx2_msg_send: [0002:02:00.0] sent 1 msg(s) of size:48

<idle>-0 [000] d.h1 756.161902: otx2_msg_interrupt: [0002:01:00.0] mbox interrupt PF(s) to AF (0x2)

kworker/u49:0-1165 [000] .... 756.162049: otx2_msg_process: [0002:01:00.0] msg:(0x400) error:0

kworker/u49:0-1165 [000] ...1 756.162051: otx2_msg_send: [0002:01:00.0] sent 1 msg(s) of size:32

kworker/u49:0-1165 [000] d.h. 756.162056: otx2_msg_interrupt: [0002:02:00.0] mbox interrupt AF to PF (0x1)

---
 drivers/net/ethernet/marvell/octeontx2/af/Makefile |   3 +-
 drivers/net/ethernet/marvell/octeontx2/af/mbox.c   |  11 ++-
 drivers/net/ethernet/marvell/octeontx2/af/rvu.c    |   7 ++
 .../net/ethernet/marvell/octeontx2/af/rvu_cgx.c    |   2 +
 .../net/ethernet/marvell/octeontx2/af/rvu_trace.c  |  14 +++
 .../net/ethernet/marvell/octeontx2/af/rvu_trace.h  | 103 +++++++++++++++++++++
 6 files changed, 138 insertions(+), 2 deletions(-)
 create mode 100644 drivers/net/ethernet/marvell/octeontx2/af/rvu_trace.c
 create mode 100644 drivers/net/ethernet/marvell/octeontx2/af/rvu_trace.h
drivers/net/ethernet/marvell/octeontx2/af/Makefile +2 -1
index 0bc2410..2f7a861 100644
@@ -3,9 +3,10 @@
 # Makefile for Marvell's OcteonTX2 RVU Admin Function driver
 #
 
+ccflags-y += -I$(src)
 obj-$(CONFIG_OCTEONTX2_MBOX) += octeontx2_mbox.o
 obj-$(CONFIG_OCTEONTX2_AF) += octeontx2_af.o
 
-octeontx2_mbox-y := mbox.o
+octeontx2_mbox-y := mbox.o rvu_trace.o
 octeontx2_af-y := cgx.o rvu.o rvu_cgx.o rvu_npa.o rvu_nix.o \
 		  rvu_reg.o rvu_npc.o rvu_debugfs.o ptp.o
drivers/net/ethernet/marvell/octeontx2/af/mbox.c +10 -1
index 387e33f..4b4cf7d 100644
@@ -14,6 +14,7 @@
 
 #include "rvu_reg.h"
 #include "mbox.h"
+#include "rvu_trace.h"
 
 static const u16 msgs_offset = ALIGN(sizeof(struct mbox_hdr), MBOX_MSG_ALIGN);
 
@@ -199,6 +200,9 @@ void otx2_mbox_msg_send(struct otx2_mbox *mbox, int devid)
 	 */
 	tx_hdr->num_msgs = mdev->num_msgs;
 	rx_hdr->num_msgs = 0;
+
+	trace_otx2_msg_send(mbox->pdev, tx_hdr->num_msgs, tx_hdr->msg_size);
+
 	spin_unlock(&mdev->mbox_lock);
 
 	/* The interrupt should be fired after num_msgs is written
@@ -295,10 +299,15 @@ int otx2_mbox_check_rsp_msgs(struct otx2_mbox *mbox, int devid)
 		struct mbox_msghdr *preq = mdev->mbase + ireq;
 		struct mbox_msghdr *prsp = mdev->mbase + irsp;
 
-		if (preq->id != prsp->id)
+		if (preq->id != prsp->id) {
+			trace_otx2_msg_check(mbox->pdev, preq->id,
+					     prsp->id, prsp->rc);
 			goto exit;
+		}
 		if (prsp->rc) {
 			rc = prsp->rc;
+			trace_otx2_msg_check(mbox->pdev, preq->id,
+					     prsp->id, prsp->rc);
 			goto exit;
 		}
 
drivers/net/ethernet/marvell/octeontx2/af/rvu.c +7 -0
index c3ef73a..e1f9189 100644
@@ -20,6 +20,8 @@
 #include "rvu_reg.h"
 #include "ptp.h"
 
+#include "rvu_trace.h"
+
 #define DRV_NAME	"octeontx2-af"
 #define DRV_STRING      "Marvell OcteonTX2 RVU Admin Function Driver"
 
@@ -1549,6 +1551,7 @@ static int rvu_process_mbox_msg(struct otx2_mbox *mbox, int devid,
 		if (rsp && err)						\
 			rsp->hdr.rc = err;				\
 									\
+		trace_otx2_msg_process(mbox->pdev, _id, err);		\
 		return rsp ? err : -ENOMEM;				\
 	}
 MBOX_MESSAGES
@@ -1881,6 +1884,8 @@ static irqreturn_t rvu_mbox_intr_handler(int irq, void *rvu_irq)
 	intr = rvu_read64(rvu, BLKADDR_RVUM, RVU_AF_PFAF_MBOX_INT);
 	/* Clear interrupts */
 	rvu_write64(rvu, BLKADDR_RVUM, RVU_AF_PFAF_MBOX_INT, intr);
+	if (intr)
+		trace_otx2_msg_interrupt(rvu->pdev, "PF(s) to AF", intr);
 
 	/* Sync with mbox memory region */
 	rmb();
@@ -1898,6 +1903,8 @@ static irqreturn_t rvu_mbox_intr_handler(int irq, void *rvu_irq)
 
 	intr = rvupf_read64(rvu, RVU_PF_VFPF_MBOX_INTX(0));
 	rvupf_write64(rvu, RVU_PF_VFPF_MBOX_INTX(0), intr);
+	if (intr)
+		trace_otx2_msg_interrupt(rvu->pdev, "VF(s) to AF", intr);
 
 	rvu_queue_work(&rvu->afvf_wq_info, 0, vfs, intr);
 
drivers/net/ethernet/marvell/octeontx2/af/rvu_cgx.c +2 -0
index fe3389c..fa9152f 100644
@@ -15,6 +15,7 @@
 #include "rvu.h"
 #include "cgx.h"
 #include "rvu_reg.h"
+#include "rvu_trace.h"
 
 struct cgx_evq_entry {
 	struct list_head evq_node;
@@ -34,6 +35,7 @@ static struct _req_type __maybe_unused					\
 		return NULL;						\
 	req->hdr.sig = OTX2_MBOX_REQ_SIG;				\
 	req->hdr.id = _id;						\
+	trace_otx2_msg_alloc(rvu->pdev, _id, sizeof(*req));		\
 	return req;							\
 }
 
drivers/net/ethernet/marvell/octeontx2/af/rvu_trace.c new file +14 -0
new file mode 100644
index 0000000..f0b3f17
@@ -0,0 +1,14 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Marvell OcteonTx2 RVU Admin Function driver tracepoints
+ *
+ * Copyright (C) 2020 Marvell International Ltd.
+ */
+
+#define CREATE_TRACE_POINTS
+#include "rvu_trace.h"
+
+EXPORT_TRACEPOINT_SYMBOL(otx2_msg_alloc);
+EXPORT_TRACEPOINT_SYMBOL(otx2_msg_send);
+EXPORT_TRACEPOINT_SYMBOL(otx2_msg_check);
+EXPORT_TRACEPOINT_SYMBOL(otx2_msg_interrupt);
+EXPORT_TRACEPOINT_SYMBOL(otx2_msg_process);
drivers/net/ethernet/marvell/octeontx2/af/rvu_trace.h new file +103 -0

103 lines not on this page · raw message

Signature · 1 line
2.7.4
maintainer
#2 lore raw

Re: [net-next v2 PATCH 1/2] octeontx2-af: Introduce tracepoints for mailbox

Quoted 8 lines · On Mon, 21 Sep 2020 22:32:41 +0530 [email protected] wrote:
drivers/net/ethernet/marvell/octeontx2/af/rvu_trace.c new file +2 -0
new file mode 100644
index 0000000..f0b3f17
@@ -0,0 +1,14 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Marvell OcteonTx2 RVU Admin Function driver tracepoints

drivers/net/ethernet/marvell/octeontx2/af/rvu_trace.c line 12

Quoted 10 lines
drivers/net/ethernet/marvell/octeontx2/af/rvu_trace.c new file +10 -0
+ *
+ * Copyright (C) 2020 Marvell International Ltd.
+ */
+
+#define CREATE_TRACE_POINTS
+#include "rvu_trace.h"
+
+EXPORT_TRACEPOINT_SYMBOL(otx2_msg_alloc);
+EXPORT_TRACEPOINT_SYMBOL(otx2_msg_send);
+EXPORT_TRACEPOINT_SYMBOL(otx2_msg_check);

I don't think you need to export send and check.

They are only used in the mbox module where they are defined.

Otherwise looks acceptable to me.

Please make sure you CC everyone who gave you feedback.

#3 lore raw

Re: [net-next v2 PATCH 1/2] octeontx2-af: Introduce tracepoints for mailbox

Hi Jakub,

Quoted 23 lines · Jakub Kicinski
Quoted 8 lines · On Mon, 21 Sep 2020 22:32:41 +0530 [email protected] wrote:
drivers/net/ethernet/marvell/octeontx2/af/rvu_trace.c new file +2 -0
new file mode 100644
index 0000000..f0b3f17
@@ -0,0 +1,14 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Marvell OcteonTx2 RVU Admin Function driver tracepoints

drivers/net/ethernet/marvell/octeontx2/af/rvu_trace.c line 12

Quoted 10 lines
drivers/net/ethernet/marvell/octeontx2/af/rvu_trace.c new file +10 -0
+ *
+ * Copyright (C) 2020 Marvell International Ltd.
+ */
+
+#define CREATE_TRACE_POINTS
+#include "rvu_trace.h"
+
+EXPORT_TRACEPOINT_SYMBOL(otx2_msg_alloc);
+EXPORT_TRACEPOINT_SYMBOL(otx2_msg_send);
+EXPORT_TRACEPOINT_SYMBOL(otx2_msg_check);

I don't think you need to export send and check.

Quoted 2 lines

They are only used in the mbox module where they are defined.

Agreed. I will remove those and send the next spin.

Quoted 2 lines

Otherwise looks acceptable to me.

Thanks

Quoted 1 line

Please make sure you CC everyone who gave you feedback.

Sure.

Sundeep

#4 lore raw

PATCH v2 2/2 octeontx2-pf: Add tracepoints for PF/VF mailbox

In reply to [email protected]

From: Subbaraya Sundeep <[email protected]>

With tracepoints support present in the mailbox code this patch adds tracepoints in PF and VF drivers at places where mailbox messages are allocated, sent and at message interrupts.

---
 drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.h | 2 ++
 drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c     | 6 ++++++
 drivers/net/ethernet/marvell/octeontx2/nic/otx2_vf.c     | 2 ++
 3 files changed, 10 insertions(+)
drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.h +2 -0
index ac47762..d6253f2 100644
@@ -20,6 +20,7 @@
 #include <mbox.h>
 #include "otx2_reg.h"
 #include "otx2_txrx.h"
+#include <rvu_trace.h>
 
 /* PCI device IDs */
 #define PCI_DEVID_OCTEONTX2_RVU_PF              0xA063
@@ -523,6 +524,7 @@ static struct _req_type __maybe_unused					\
 		return NULL;						\
 	req->hdr.sig = OTX2_MBOX_REQ_SIG;				\
 	req->hdr.id = _id;						\
+	trace_otx2_msg_alloc(mbox->mbox.pdev, _id, sizeof(*req));	\
 	return req;							\
 }
 
drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c +6 -0
index aac2845..265e4d1 100644
@@ -22,6 +22,7 @@
 #include "otx2_txrx.h"
 #include "otx2_struct.h"
 #include "otx2_ptp.h"
+#include <rvu_trace.h>
 
 #define DRV_NAME	"octeontx2-nicpf"
 #define DRV_STRING	"Marvell OcteonTX2 NIC Physical Function Driver"
@@ -558,6 +559,8 @@ static irqreturn_t otx2_pfvf_mbox_intr_handler(int irq, void *pf_irq)
 
 	otx2_queue_work(mbox, pf->mbox_pfvf_wq, 0, vfs, intr, TYPE_PFVF);
 
+	trace_otx2_msg_interrupt(mbox->mbox.pdev, "VF(s) to PF", intr);
+
 	return IRQ_HANDLED;
 }
 
@@ -940,6 +943,9 @@ static irqreturn_t otx2_pfaf_mbox_intr_handler(int irq, void *pf_irq)
 	otx2_write64(pf, RVU_PF_INT, BIT_ULL(0));
 
 	mbox = &pf->mbox;
+
+	trace_otx2_msg_interrupt(mbox->mbox.pdev, "AF to PF", BIT_ULL(0));
+
 	otx2_queue_work(mbox, pf->mbox_wq, 0, 1, 1, TYPE_PFAF);
 
 	return IRQ_HANDLED;
drivers/net/ethernet/marvell/octeontx2/nic/otx2_vf.c +2 -0
index 70e0d4c..32daa3e 100644
@@ -187,6 +187,8 @@ static irqreturn_t otx2vf_vfaf_mbox_intr_handler(int irq, void *vf_irq)
 	mdev = &mbox->dev[0];
 	otx2_sync_mbox_bbuf(mbox, 0);
 
+	trace_otx2_msg_interrupt(mbox->pdev, "PF to VF", BIT_ULL(0));
+
 	hdr = (struct mbox_hdr *)(mdev->mbase + mbox->rx_start);
 	if (hdr->num_msgs) {
 		vf->mbox.num_msgs = hdr->num_msgs;
Signature · 1 line
2.7.4