Skip to contentSkip to conversation

Support PERF MGMT for RXE

PATCH Superseded zhenwei pi 2 participants 7 messages Mar 29

Subsystems (2)

v2 of 7 Superseded Latest is v7 →

lore
Share & tools
lore

zhenwei pi

author
#0 lore raw

v2 cover Support PERF MGMT for RXE

v2:
- Fix overflow for PMA counter *link_downed_counter*
- Use *rxe_counter_get* instead of *atomic64_read* for hw-counters

v1:
Support PERF MGMT for RXE, add sent/received bytes for RXE counters, also improve coding style.

zhenwei pi (4):
RDMA/rxe: use RXE_PORT instead of magic number 1 RDMA/rxe: add SENT/RCVD bytes
RDMA/rxe: support perf mgmt GET method RDMA/rxe: use rxe_counter_get

 drivers/infiniband/sw/rxe/Makefile          |  1 +
 drivers/infiniband/sw/rxe/rxe_hw_counters.c |  4 +-
 drivers/infiniband/sw/rxe/rxe_hw_counters.h |  2 +
 drivers/infiniband/sw/rxe/rxe_loc.h         |  6 ++
 drivers/infiniband/sw/rxe/rxe_mad.c         | 93 +++++++++++++++++++++
 drivers/infiniband/sw/rxe/rxe_net.c         |  7 +-
 drivers/infiniband/sw/rxe/rxe_recv.c        |  1 +
 drivers/infiniband/sw/rxe/rxe_verbs.c       |  9 +-
 drivers/infiniband/sw/rxe/rxe_verbs.h       | 11 +++
 9 files changed, 126 insertions(+), 8 deletions(-)
 create mode 100644 drivers/infiniband/sw/rxe/rxe_mad.c
Signature · 1 line
2.43.0
author
#1 lore raw

PATCH v2 1/4 RDMA/rxe: use RXE_PORT instead of magic number 1

Align with the existing code:
static ... rxe_ib_device_get_netdev(struct ib_device *dev)

{
        return ib_device_get_netdev(dev, RXE_PORT);
}

Use *RXE_PORT* instead of magic number 1 for all.

---
 drivers/infiniband/sw/rxe/rxe_net.c   | 6 +++---
 drivers/infiniband/sw/rxe/rxe_verbs.c | 8 ++++----
 2 files changed, 7 insertions(+), 7 deletions(-)
drivers/infiniband/sw/rxe/rxe_net.c +3 -3
index 0bd0902b11f7..20338cb8e3c2 100644
@@ -234,7 +234,7 @@ static int rxe_udp_encap_recv(struct sock *sk, struct sk_buff *skb)
 
 	udph = udp_hdr(skb);
 	pkt->rxe = rxe;
-	pkt->port_num = 1;
+	pkt->port_num = RXE_PORT;
 	pkt->hdr = (u8 *)(udph + 1);
 	pkt->mask = RXE_GRH_MASK;
 	pkt->paylen = be16_to_cpu(udph->len) - sizeof(*udph);
@@ -535,7 +535,7 @@ struct sk_buff *rxe_init_packet(struct rxe_dev *rxe, struct rxe_av *av,
 	struct sk_buff *skb = NULL;
 	struct net_device *ndev;
 	const struct ib_gid_attr *attr;
-	const int port_num = 1;
+	const int port_num = RXE_PORT;
 
 	attr = rdma_get_gid_attr(&rxe->ib_dev, port_num, av->grh.sgid_index);
 	if (IS_ERR(attr))
@@ -630,7 +630,7 @@ static void rxe_port_event(struct rxe_dev *rxe,
 	struct ib_event ev;
 
 	ev.device = &rxe->ib_dev;
-	ev.element.port_num = 1;
+	ev.element.port_num = RXE_PORT;
 	ev.event = event;
 
 	ib_dispatch_event(&ev);
drivers/infiniband/sw/rxe/rxe_verbs.c +4 -4
index fe41362c5144..bcd486e8668b 100644
@@ -44,7 +44,7 @@ static int rxe_query_port(struct ib_device *ibdev,
 	struct net_device *ndev;
 	int err, ret;
 
-	if (port_num != 1) {
+	if (port_num != RXE_PORT) {
 		err = -EINVAL;
 		rxe_dbg_dev(rxe, "bad port_num = %d\n", port_num);
 		goto err_out;
@@ -147,7 +147,7 @@ static int rxe_modify_port(struct ib_device *ibdev, u32 port_num,
 	struct rxe_port *port;
 	int err;
 
-	if (port_num != 1) {
+	if (port_num != RXE_PORT) {
 		err = -EINVAL;
 		rxe_dbg_dev(rxe, "bad port_num = %d\n", port_num);
 		goto err_out;
@@ -180,7 +180,7 @@ static enum rdma_link_layer rxe_get_link_layer(struct ib_device *ibdev,
 	struct rxe_dev *rxe = to_rdev(ibdev);
 	int err;
 
-	if (port_num != 1) {
+	if (port_num != RXE_PORT) {
 		err = -EINVAL;
 		rxe_dbg_dev(rxe, "bad port_num = %d\n", port_num);
 		goto err_out;
@@ -200,7 +200,7 @@ static int rxe_port_immutable(struct ib_device *ibdev, u32 port_num,
 	struct ib_port_attr attr = {};
 	int err;
 
-	if (port_num != 1) {
+	if (port_num != RXE_PORT) {
 		err = -EINVAL;
 		rxe_dbg_dev(rxe, "bad port_num = %d\n", port_num);
 		goto err_out;
Signature · 1 line
2.43.0
author
#2 lore raw

PATCH v2 2/4 RDMA/rxe: add SENT/RCVD bytes

In reply to zhenwei pi

There is a lack of sent/received counter in bytes.

---
 drivers/infiniband/sw/rxe/rxe_hw_counters.c | 2 ++
 drivers/infiniband/sw/rxe/rxe_hw_counters.h | 2 ++
 drivers/infiniband/sw/rxe/rxe_net.c         | 1 +
 drivers/infiniband/sw/rxe/rxe_recv.c        | 1 +
 drivers/infiniband/sw/rxe/rxe_verbs.h       | 6 ++++++
 5 files changed, 12 insertions(+)
drivers/infiniband/sw/rxe/rxe_hw_counters.c +2 -0
index 437917a7d8f2..17edaa9a9b9b 100644
@@ -22,6 +22,8 @@ static const struct rdma_stat_desc rxe_counter_descs[] = {
 	[RXE_CNT_LINK_DOWNED].name         =  "link_downed",
 	[RXE_CNT_RDMA_SEND].name           =  "rdma_sends",
 	[RXE_CNT_RDMA_RECV].name           =  "rdma_recvs",
+	[RXE_CNT_SENT_BYTES].name          =  "sent_bytes",
+	[RXE_CNT_RCVD_BYTES].name          =  "rcvd_bytes",
 };
 
 int rxe_ib_get_hw_stats(struct ib_device *ibdev,
drivers/infiniband/sw/rxe/rxe_hw_counters.h +2 -0
index 051f9e1c3852..01b355103cbc 100644
@@ -26,6 +26,8 @@ enum rxe_counters {
 	RXE_CNT_LINK_DOWNED,
 	RXE_CNT_RDMA_SEND,
 	RXE_CNT_RDMA_RECV,
+	RXE_CNT_SENT_BYTES,
+	RXE_CNT_RCVD_BYTES,
 	RXE_NUM_OF_COUNTERS
 };
 
drivers/infiniband/sw/rxe/rxe_net.c +1 -0
index 20338cb8e3c2..ec0ae7479fe7 100644
@@ -519,6 +519,7 @@ int rxe_xmit_packet(struct rxe_qp *qp, struct rxe_pkt_info *pkt,
 	}
 
 	rxe_counter_inc(rxe, RXE_CNT_SENT_PKTS);
+	rxe_counter_add(rxe, RXE_CNT_SENT_BYTES, skb->len);
 	goto done;
 
 drop:
drivers/infiniband/sw/rxe/rxe_recv.c +1 -0
index 5861e4244049..b5522017852d 100644
@@ -342,6 +342,7 @@ void rxe_rcv(struct sk_buff *skb)
 		goto drop;
 
 	rxe_counter_inc(rxe, RXE_CNT_RCVD_PKTS);
+	rxe_counter_add(rxe, RXE_CNT_RCVD_BYTES, skb->len);
 
 	if (unlikely(bth_qpn(pkt) == IB_MULTICAST_QPN))
 		rxe_rcv_mcast_pkt(rxe, skb);
drivers/infiniband/sw/rxe/rxe_verbs.h +6 -0
index fb149f37e91d..2bcfb919a40b 100644
@@ -460,6 +460,12 @@ static inline void rxe_counter_inc(struct rxe_dev *rxe, enum rxe_counters index)
 	atomic64_inc(&rxe->stats_counters[index]);
 }
 
+static inline void rxe_counter_add(struct rxe_dev *rxe, enum rxe_counters index,
+				   s64 val)
+{
+	atomic64_add(val, &rxe->stats_counters[index]);
+}
+
 static inline struct rxe_dev *to_rdev(struct ib_device *dev)
 {
 	return dev ? container_of(dev, struct rxe_dev, ib_dev) : NULL;
Signature · 1 line
2.43.0
author
#3 lore raw

PATCH v2 3/4 RDMA/rxe: support perf mgmt GET method

In reply to zhenwei pi

In RXE, hardware counters are already supported, but not in a standardized manner. For instance, user-space monitoring tools like atop only read from the *counters* directory. Therefore, it is necessary to add perf management support to RXE.

---
 drivers/infiniband/sw/rxe/Makefile    |  1 +
 drivers/infiniband/sw/rxe/rxe_loc.h   |  6 ++
 drivers/infiniband/sw/rxe/rxe_mad.c   | 93 +++++++++++++++++++++++++++
 drivers/infiniband/sw/rxe/rxe_verbs.c |  1 +
 drivers/infiniband/sw/rxe/rxe_verbs.h |  5 ++
 5 files changed, 106 insertions(+)
 create mode 100644 drivers/infiniband/sw/rxe/rxe_mad.c
drivers/infiniband/sw/rxe/Makefile +1 -0
index 93134f1d1d0c..3c47e5b982c2 100644
@@ -22,6 +22,7 @@ rdma_rxe-y := \
 	rxe_mcast.o \
 	rxe_task.o \
 	rxe_net.o \
+	rxe_mad.o \
 	rxe_hw_counters.o
 
 rdma_rxe-$(CONFIG_INFINIBAND_ON_DEMAND_PAGING) += rxe_odp.o
drivers/infiniband/sw/rxe/rxe_loc.h +6 -0
index 7992290886e1..a8ce85147c1f 100644
@@ -245,4 +245,10 @@ static inline int rxe_ib_advise_mr(struct ib_pd *pd,
 
 #endif /* CONFIG_INFINIBAND_ON_DEMAND_PAGING */
 
+/* rxe-mad.c */
+int rxe_process_mad(struct ib_device *ibdev, int mad_flags, u32 port_num,
+		    const struct ib_wc *in_wc, const struct ib_grh *in_grh,
+		    const struct ib_mad *in, struct ib_mad *out,
+		    size_t *out_mad_size, u16 *out_mad_pkey_index);
+
 #endif /* RXE_LOC_H */
drivers/infiniband/sw/rxe/rxe_mad.c new file +93 -0

93 lines not on this page · raw message

drivers/infiniband/sw/rxe/rxe_verbs.c +1 -0
index bcd486e8668b..7df0cb5a09a3 100644
@@ -1509,6 +1509,7 @@ static const struct ib_device_ops rxe_dev_ops = {
 	.post_recv = rxe_post_recv,
 	.post_send = rxe_post_send,
 	.post_srq_recv = rxe_post_srq_recv,
+	.process_mad = rxe_process_mad,
 	.query_ah = rxe_query_ah,
 	.query_device = rxe_query_device,
 	.query_pkey = rxe_query_pkey,
drivers/infiniband/sw/rxe/rxe_verbs.h +5 -0
index 2bcfb919a40b..1c4fa8eaa733 100644
@@ -466,6 +466,11 @@ static inline void rxe_counter_add(struct rxe_dev *rxe, enum rxe_counters index,
 	atomic64_add(val, &rxe->stats_counters[index]);
 }
 
+static inline s64 rxe_counter_get(struct rxe_dev *rxe, enum rxe_counters index)
+{
+	return atomic64_read(&rxe->stats_counters[index]);
+}
+
 static inline struct rxe_dev *to_rdev(struct ib_device *dev)
 {
 	return dev ? container_of(dev, struct rxe_dev, ib_dev) : NULL;
Signature · 1 line
2.43.0
author
#5 lore raw

PATCH v2 4/4 RDMA/rxe: use rxe_counter_get

In reply to zhenwei pi

There is *rxe_counter_get* helper in RXE, use it instead of raw atomic64_read.

---
 drivers/infiniband/sw/rxe/rxe_hw_counters.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
drivers/infiniband/sw/rxe/rxe_hw_counters.c +1 -1
index 17edaa9a9b9b..a612e96f7a88 100644
@@ -37,7 +37,7 @@ int rxe_ib_get_hw_stats(struct ib_device *ibdev,
 		return -EINVAL;
 
 	for (cnt = 0; cnt < ARRAY_SIZE(rxe_counter_descs); cnt++)
-		stats->value[cnt] = atomic64_read(&dev->stats_counters[cnt]);
+		stats->value[cnt] = rxe_counter_get(dev, cnt);
 
 	return ARRAY_SIZE(rxe_counter_descs);
 }
Signature · 1 line
2.43.0
#4 lore raw

Re: [PATCH v2 3/4] RDMA/rxe: support perf mgmt GET method

In reply to zhenwei pi
Quoted 105 lines · zhenwei pi

In RXE, hardware counters are already supported, but not in a standardized manner. For instance, user-space monitoring tools like atop only read from the *counters* directory. Therefore, it is necessary to add perf management support to RXE.

---
  drivers/infiniband/sw/rxe/Makefile    |  1 +
  drivers/infiniband/sw/rxe/rxe_loc.h   |  6 ++
  drivers/infiniband/sw/rxe/rxe_mad.c   | 93 +++++++++++++++++++++++++++
  drivers/infiniband/sw/rxe/rxe_verbs.c |  1 +
  drivers/infiniband/sw/rxe/rxe_verbs.h |  5 ++
  5 files changed, 106 insertions(+)

create mode 100644 drivers/infiniband/sw/rxe/rxe_mad.c

drivers/infiniband/sw/rxe/Makefile +1 -0
index 93134f1d1d0c..3c47e5b982c2 100644
@@ -22,6 +22,7 @@ rdma_rxe-y := \
  	rxe_mcast.o \
  	rxe_task.o \
  	rxe_net.o \
+	rxe_mad.o \
  	rxe_hw_counters.o
 
  rdma_rxe-$(CONFIG_INFINIBAND_ON_DEMAND_PAGING) += rxe_odp.o
drivers/infiniband/sw/rxe/rxe_loc.h +6 -0
index 7992290886e1..a8ce85147c1f 100644
@@ -245,4 +245,10 @@ static inline int rxe_ib_advise_mr(struct ib_pd *pd,
 
  #endif /* CONFIG_INFINIBAND_ON_DEMAND_PAGING */
 
+/* rxe-mad.c */
+int rxe_process_mad(struct ib_device *ibdev, int mad_flags, u32 port_num,
+		    const struct ib_wc *in_wc, const struct ib_grh *in_grh,
+		    const struct ib_mad *in, struct ib_mad *out,
+		    size_t *out_mad_size, u16 *out_mad_pkey_index);
+
  #endif /* RXE_LOC_H */
drivers/infiniband/sw/rxe/rxe_mad.c new file +57 -0
new file mode 100644
index 000000000000..5e0567806c02
@@ -0,0 +1,93 @@
+// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
+/*
+ * Copyright (c) 2026 zhenwei pi <[email protected]>
+ */
+
+#include <rdma/ib_pma.h>
+#include "rxe.h"
+#include "rxe_hw_counters.h"
+
+static int rxe_get_pma_info(struct ib_mad *out)
+{
+	struct ib_class_port_info cpi = {};
+
+	cpi.capability_mask = IB_PMA_CLASS_CAP_EXT_WIDTH;
+	memcpy((out->data + 40), &cpi, sizeof(cpi));
+
+	return IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_REPLY;
+}
+
+static int rxe_get_pma_counters(struct rxe_dev *rxe, struct ib_mad *out)
+{
+	struct ib_pma_portcounters *pma_cnt = (struct ib_pma_portcounters *)(out->data + 40);
+	s64 val;
+
+	/* IBA release 1.8, 16.1.3.5: During operation, instead of overflowing, they shall stop
+	 * at all ones.
+	 */
+	val = rxe_counter_get(rxe, RXE_CNT_LINK_DOWNED);
+	if (val > U8_MAX)
+		pma_cnt->link_downed_counter = U8_MAX;
+	else
+		pma_cnt->link_downed_counter = (u8)val;
+
+	return IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_REPLY;
+}
+
+static int rxe_get_pma_counters_ext(struct rxe_dev *rxe, struct ib_mad *out)
+{
+	struct ib_pma_portcounters_ext *pma_cnt_ext =
+		(struct ib_pma_portcounters_ext *)(out->data + 40);
+
+	pma_cnt_ext->port_xmit_data = cpu_to_be64(rxe_counter_get(rxe, RXE_CNT_SENT_BYTES) >> 2);
+	pma_cnt_ext->port_rcv_data = cpu_to_be64(rxe_counter_get(rxe, RXE_CNT_RCVD_BYTES) >> 2);
+	pma_cnt_ext->port_xmit_packets = cpu_to_be64(rxe_counter_get(rxe, RXE_CNT_SENT_PKTS));
+	pma_cnt_ext->port_rcv_packets = cpu_to_be64(rxe_counter_get(rxe, RXE_CNT_RCVD_PKTS));
+
+	return IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_REPLY;
+}
+
+static int rxe_get_perf_mgmt(struct rxe_dev *rxe, const struct ib_mad *in,
+				 struct ib_mad *out)
+{
+	switch (in->mad_hdr.attr_id) {
+	case IB_PMA_CLASS_PORT_INFO:
+		return rxe_get_pma_info(out);
+
+	case IB_PMA_PORT_COUNTERS:

drivers/infiniband/sw/rxe/rxe_mad.c line 67

Quoted 10 lines
drivers/infiniband/sw/rxe/rxe_mad.c new file +10 -0
+		return rxe_get_pma_counters(rxe, out);
+
+	case IB_PMA_PORT_COUNTERS_EXT:
+		return rxe_get_pma_counters_ext(rxe, out);
+
+	default:
+		break;
+	}
+
+	return IB_MAD_RESULT_FAILURE;

In rxe_get_perf_mgmt, the default case returns IB_MAD_RESULT_FAILURE.

Returning FAILURE in the MAD stack often results in the packet being dropped without a response. This causes management tools (like ibstat or perfquery) to hang until a timeout occurs.

According to IBA specs, the agent should return a reply with a proper error status.

Return IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_REPLY and set out->mad_hdr.status = cpu_to_be16(IB_MGMT_MAD_STATUS_BAD_ATTR);.

The source code in default branch should be:

"
default:
out->mad_hdr.status = cpu_to_be16(IB_MGMT_MAD_STATUS_BAD_ATTR); return IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_REPLY;
"

Quoted 4 lines
drivers/infiniband/sw/rxe/rxe_mad.c new file +4 -0
+}
+
+int rxe_process_mad(struct ib_device *ibdev, int mad_flags, u32 port_num,
+		    const struct ib_wc *in_wc, const struct ib_grh *in_grh,

drivers/infiniband/sw/rxe/rxe_mad.c line 81

Quoted 10 lines
drivers/infiniband/sw/rxe/rxe_mad.c new file +10 -0
+		    const struct ib_mad *in, struct ib_mad *out,
+		    size_t *out_mad_size, u16 *out_mad_pkey_index)
+{
+	struct rxe_dev *rxe = to_rdev(ibdev);
+	u8 mgmt_class = in->mad_hdr.mgmt_class;
+	u8 method = in->mad_hdr.method;
+
+	if (port_num != RXE_PORT)
+		return IB_MAD_RESULT_FAILURE;
+

In the IB Management Datagram (MAD) spec, many Reserved fields or unused counter fields are strictly required to be zero-filled.

By using memset(out, 0, sizeof(*out)), you ensure that all fields in the response packet—including those not currently supported by the driver—have a predefined initial value. This prevents the transmission of packets containing "random noise" (residual kernel data) to the querying entity.

Add "memset(out, 0, sizeof(*out));" here.

Zhu Yanjun

Quoted 40 lines
drivers/infiniband/sw/rxe/rxe_mad.c new file +12 -0
+	switch (mgmt_class) {
+	case IB_MGMT_CLASS_PERF_MGMT:
+		if (method == IB_MGMT_METHOD_GET)
+			return rxe_get_perf_mgmt(rxe, in, out);
+		break;
+
+	default:
+		break;
+	}
+
+	return IB_MAD_RESULT_FAILURE;
+}
drivers/infiniband/sw/rxe/rxe_verbs.c +1 -0
index bcd486e8668b..7df0cb5a09a3 100644
@@ -1509,6 +1509,7 @@ static const struct ib_device_ops rxe_dev_ops = {
  	.post_recv = rxe_post_recv,
  	.post_send = rxe_post_send,
  	.post_srq_recv = rxe_post_srq_recv,
+	.process_mad = rxe_process_mad,
  	.query_ah = rxe_query_ah,
  	.query_device = rxe_query_device,
  	.query_pkey = rxe_query_pkey,
drivers/infiniband/sw/rxe/rxe_verbs.h +5 -0
index 2bcfb919a40b..1c4fa8eaa733 100644
@@ -466,6 +466,11 @@ static inline void rxe_counter_add(struct rxe_dev *rxe, enum rxe_counters index,
  	atomic64_add(val, &rxe->stats_counters[index]);
  }
 
+static inline s64 rxe_counter_get(struct rxe_dev *rxe, enum rxe_counters index)
+{
+	return atomic64_read(&rxe->stats_counters[index]);
+}
+
  static inline struct rxe_dev *to_rdev(struct ib_device *dev)
  {
  	return dev ? container_of(dev, struct rxe_dev, ib_dev) : NULL;
#6 lore raw

Re: [PATCH v2 4/4] RDMA/rxe: use rxe_counter_get

In reply to zhenwei pi
Quoted 2 lines · zhenwei pi

There is *rxe_counter_get* helper in RXE, use it instead of raw atomic64_read.

This commit can be merged with the commit "[PATCH v2 3/4] RDMA/rxe: support perf mgmt GET method"

Zhu Yanjun

Quoted 19 lines
---
  drivers/infiniband/sw/rxe/rxe_hw_counters.c | 2 +-
  1 file changed, 1 insertion(+), 1 deletion(-)
drivers/infiniband/sw/rxe/rxe_hw_counters.c +1 -1
index 17edaa9a9b9b..a612e96f7a88 100644
@@ -37,7 +37,7 @@ int rxe_ib_get_hw_stats(struct ib_device *ibdev,
  		return -EINVAL;
 
  	for (cnt = 0; cnt < ARRAY_SIZE(rxe_counter_descs); cnt++)
-		stats->value[cnt] = atomic64_read(&dev->stats_counters[cnt]);
+		stats->value[cnt] = rxe_counter_get(dev, cnt);
 
  	return ARRAY_SIZE(rxe_counter_descs);
  }