Support PERF MGMT for RXE
Subsystems (2)
cover Support PERF MGMT for RXE
Support PERF MGMT for RXE, add sent/received bytes for RXE counters, also improve coding style.
zhenwei pi (3):
RDMA/rxe: use RXE_PORT instead of magic number 1 RDMA/rxe: add SENT/RCVD bytes
RDMA/rxe: support perf mgmt
drivers/infiniband/sw/rxe/Makefile | 1 + drivers/infiniband/sw/rxe/rxe_hw_counters.c | 2 + drivers/infiniband/sw/rxe/rxe_hw_counters.h | 2 + drivers/infiniband/sw/rxe/rxe_loc.h | 6 ++ drivers/infiniband/sw/rxe/rxe_mad.c | 86 +++++++++++++++++++++ drivers/infiniband/sw/rxe/rxe_net.c | 7 +- drivers/infiniband/sw/rxe/rxe_recv.c | 1 + drivers/infiniband/sw/rxe/rxe_verbs.c | 9 ++- drivers/infiniband/sw/rxe/rxe_verbs.h | 11 +++ 9 files changed, 118 insertions(+), 7 deletions(-) create mode 100644 drivers/infiniband/sw/rxe/rxe_mad.c
Signature · 1 line
2.43.0
PATCH 1/3 RDMA/rxe: use RXE_PORT instead of magic number 1
Align with the existing code:
static ... rxe_ib_device_get_netdev(struct ib_device *dev)
{
return ib_device_get_netdev(dev, RXE_PORT);
}Use *RXE_PORT* instead of magic number 1 for all.
--- drivers/infiniband/sw/rxe/rxe_net.c | 6 +++--- drivers/infiniband/sw/rxe/rxe_verbs.c | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-)
drivers/infiniband/sw/rxe/rxe_net.c +3 -3
@@ -234,7 +234,7 @@ static int rxe_udp_encap_recv(struct sock *sk, struct sk_buff *skb) udph =udp_hdr (skb); pkt->rxe = rxe; - pkt->port_num =1 ; + pkt->port_num =RXE_PORT ; pkt->hdr = (u8 *)(udph +1 ); pkt->mask =RXE_GRH_MASK ; pkt->paylen =be16_to_cpu (udph->len) -sizeof (*udph);
@@ -535,7 +535,7 @@ struct sk_buff *rxe_init_packet(struct rxe_dev *rxe, struct rxe_av *av,struct sk_buff *skb =NULL ;struct net_device *ndev;const struct ib_gid_attr *attr; -const int port_num =1 ; +const int port_num =RXE_PORT ; attr =rdma_get_gid_attr (&rxe->ib_dev, port_num, av->grh.sgid_index);if (IS_ERR (attr))
@@ -630,7 +630,7 @@ static void rxe_port_event(struct rxe_dev *rxe,struct ib_event ev; ev.device = &rxe->ib_dev; - ev.element.port_num =1 ; + ev.element.port_num =RXE_PORT ; ev.event = event;ib_dispatch_event (&ev);
drivers/infiniband/sw/rxe/rxe_verbs.c +4 -4
@@ -44,7 +44,7 @@ static int rxe_query_port(struct ib_device *ibdev,struct net_device *ndev;int err, ret; -if (port_num !=1 ) { +if (port_num !=RXE_PORT ) { err = -EINVAL ;rxe_dbg_dev (rxe,"bad port_num = %d\n" , port_num); goto err_out;
@@ -147,7 +147,7 @@ static int rxe_modify_port(struct ib_device *ibdev, u32 port_num,struct rxe_port *port;int err; -if (port_num !=1 ) { +if (port_num !=RXE_PORT ) { err = -EINVAL ;rxe_dbg_dev (rxe,"bad port_num = %d\n" , port_num); goto err_out;
@@ -180,7 +180,7 @@ static enum rdma_link_layer rxe_get_link_layer(struct ib_device *ibdev,struct rxe_dev *rxe =to_rdev (ibdev);int err; -if (port_num !=1 ) { +if (port_num !=RXE_PORT ) { err = -EINVAL ;rxe_dbg_dev (rxe,"bad port_num = %d\n" , port_num); goto err_out;
@@ -200,7 +200,7 @@ static int rxe_port_immutable(struct ib_device *ibdev, u32 port_num,struct ib_port_attr attr = {};int err; -if (port_num !=1 ) { +if (port_num !=RXE_PORT ) { err = -EINVAL ;rxe_dbg_dev (rxe,"bad port_num = %d\n" , port_num); goto err_out;
Signature · 1 line
2.43.0
PATCH 2/3 RDMA/rxe: add SENT/RCVD bytes
In reply to zhenwei piThere is a lack of sent/received counter in bytes.
--- drivers/infiniband/sw/rxe/rxe_hw_counters.c | 2 ++ drivers/infiniband/sw/rxe/rxe_hw_counters.h | 2 ++ drivers/infiniband/sw/rxe/rxe_net.c | 1 + drivers/infiniband/sw/rxe/rxe_recv.c | 1 + drivers/infiniband/sw/rxe/rxe_verbs.h | 6 ++++++ 5 files changed, 12 insertions(+)
drivers/infiniband/sw/rxe/rxe_hw_counters.c +2 -0
@@ -22,6 +22,8 @@ static const struct rdma_stat_desc rxe_counter_descs[] = { [RXE_CNT_LINK_DOWNED].name = "link_downed", [RXE_CNT_RDMA_SEND].name = "rdma_sends", [RXE_CNT_RDMA_RECV].name = "rdma_recvs", + [RXE_CNT_SENT_BYTES].name = "sent_bytes", + [RXE_CNT_RCVD_BYTES].name = "rcvd_bytes", }; int rxe_ib_get_hw_stats(struct ib_device *ibdev,
drivers/infiniband/sw/rxe/rxe_hw_counters.h +2 -0
@@ -26,6 +26,8 @@ enum rxe_counters {RXE_CNT_LINK_DOWNED ,RXE_CNT_RDMA_SEND ,RXE_CNT_RDMA_RECV , +RXE_CNT_SENT_BYTES , +RXE_CNT_RCVD_BYTES , RXE_NUM_OF_COUNTERS };
drivers/infiniband/sw/rxe/rxe_net.c +1 -0
@@ -519,6 +519,7 @@ int rxe_xmit_packet(struct rxe_qp *qp, struct rxe_pkt_info *pkt, }rxe_counter_inc (rxe,RXE_CNT_SENT_PKTS ); +rxe_counter_add (rxe,RXE_CNT_SENT_BYTES , skb->len); goto done; drop:
drivers/infiniband/sw/rxe/rxe_recv.c +1 -0
@@ -342,6 +342,7 @@ void rxe_rcv(struct sk_buff *skb) goto drop;rxe_counter_inc (rxe,RXE_CNT_RCVD_PKTS ); +rxe_counter_add (rxe,RXE_CNT_RCVD_BYTES , skb->len);if (unlikely (bth_qpn (pkt) ==IB_MULTICAST_QPN ))rxe_rcv_mcast_pkt (rxe, skb);
drivers/infiniband/sw/rxe/rxe_verbs.h +6 -0
@@ -460,6 +460,12 @@ static inline void rxe_counter_inc(struct rxe_dev *rxe, enum rxe_counters index)atomic64_inc (&rxe->stats_counters[index]); } +static inline void rxe_counter_add (struct rxe_dev *rxe,enum rxe_counters index, +s64 val) +{ +atomic64_add (val, &rxe->stats_counters[index]); +} +static inline struct rxe_dev *to_rdev (struct ib_device *dev) { return dev ? container_of(dev, struct rxe_dev, ib_dev) : NULL;
Signature · 1 line
2.43.0
Re: [PATCH 2/3] RDMA/rxe: add SENT/RCVD bytes
Quoted 66 lines · zhenwei pi
There is a lack of sent/received counter in bytes.
Signed-off-by zhenwei pi <[email protected]>--- drivers/infiniband/sw/rxe/rxe_hw_counters.c | 2 ++ drivers/infiniband/sw/rxe/rxe_hw_counters.h | 2 ++ drivers/infiniband/sw/rxe/rxe_net.c | 1 + drivers/infiniband/sw/rxe/rxe_recv.c | 1 + drivers/infiniband/sw/rxe/rxe_verbs.h | 6 ++++++ 5 files changed, 12 insertions(+)drivers/infiniband/sw/rxe/rxe_hw_counters.c +2
-0@@ -22,6 +22,8 @@ static const struct rdma_stat_desc rxe_counter_descs[] = { [RXE_CNT_LINK_DOWNED].name = "link_downed", [RXE_CNT_RDMA_SEND].name = "rdma_sends", [RXE_CNT_RDMA_RECV].name = "rdma_recvs", + [RXE_CNT_SENT_BYTES].name = "sent_bytes", + [RXE_CNT_RCVD_BYTES].name = "rcvd_bytes", }; int rxe_ib_get_hw_stats(struct ib_device *ibdev,drivers/infiniband/sw/rxe/rxe_hw_counters.h +2
-0@@ -26,6 +26,8 @@ enum rxe_counters {RXE_CNT_LINK_DOWNED ,RXE_CNT_RDMA_SEND ,RXE_CNT_RDMA_RECV , +RXE_CNT_SENT_BYTES , +RXE_CNT_RCVD_BYTES , RXE_NUM_OF_COUNTERS };drivers/infiniband/sw/rxe/rxe_net.c +1
-0@@ -519,6 +519,7 @@ int rxe_xmit_packet(struct rxe_qp *qp, struct rxe_pkt_info *pkt, }rxe_counter_inc (rxe,RXE_CNT_SENT_PKTS ); +rxe_counter_add (rxe,RXE_CNT_SENT_BYTES , skb->len); goto done; drop:drivers/infiniband/sw/rxe/rxe_recv.c +1
-0@@ -342,6 +342,7 @@ void rxe_rcv(struct sk_buff *skb) goto drop;rxe_counter_inc (rxe,RXE_CNT_RCVD_PKTS ); +rxe_counter_add (rxe,RXE_CNT_RCVD_BYTES , skb->len);if (unlikely (bth_qpn (pkt) ==IB_MULTICAST_QPN ))rxe_rcv_mcast_pkt (rxe, skb);drivers/infiniband/sw/rxe/rxe_verbs.h +0
-0
drivers/infiniband/sw/rxe/rxe_verbs.h static inline void rxe_counter_inc(struct rxe_dev *rxe, enum rxe_counters index) · line 466
Quoted 7 lines
drivers/infiniband/sw/rxe/rxe_verbs.h +4
-0@@ -460,6 +460,12 @@ static inline void rxe_counter_inc(struct rxe_dev *rxe, enum rxe_counters index)atomic64_inc (&rxe->stats_counters[index]); } +static inline void rxe_counter_add (struct rxe_dev *rxe,enum rxe_counters index, +s64 val) +{ +atomic64_add (val, &rxe->stats_counters[index]);
Currently atomic64 variable is used to calculate the statistics. But to get better performance, per cpu variable is preferred. Since RXE is a software emulation rdma driver, the atomic64 variable is fine in RXE.
To mlx5, broadcom and efa driver, to get better performance, the per cpu variable is preferred.
Zhu Yanjun
Quoted 5 lines
drivers/infiniband/sw/rxe/rxe_verbs.h +2
-0+} +static inline struct rxe_dev *to_rdev (struct ib_device *dev) { return dev ? container_of(dev, struct rxe_dev, ib_dev) : NULL;
PATCH 3/3 RDMA/rxe: support perf mgmt
In reply to zhenwei piIn RXE, hardware counters are already supported, but not in a standardized manner. For instance, user-space monitoring tools like atop only read from the *counters* directory. Therefore, it is necessary to add perf management support to RXE.
--- drivers/infiniband/sw/rxe/Makefile | 1 + drivers/infiniband/sw/rxe/rxe_loc.h | 6 ++ drivers/infiniband/sw/rxe/rxe_mad.c | 86 +++++++++++++++++++++++++++ drivers/infiniband/sw/rxe/rxe_verbs.c | 1 + drivers/infiniband/sw/rxe/rxe_verbs.h | 5 ++ 5 files changed, 99 insertions(+) create mode 100644 drivers/infiniband/sw/rxe/rxe_mad.c
drivers/infiniband/sw/rxe/Makefile +1 -0
@@ -22,6 +22,7 @@ rdma_rxe-y := \ rxe_mcast.o \ rxe_task.o \ rxe_net.o \ + rxe_mad.o \ rxe_hw_counters.o rdma_rxe-$(CONFIG_INFINIBAND_ON_DEMAND_PAGING) += rxe_odp.o
drivers/infiniband/sw/rxe/rxe_loc.h +6 -0
@@ -245,4 +245,10 @@ static inline int rxe_ib_advise_mr(struct ib_pd *pd,#endif /* CONFIG_INFINIBAND_ON_DEMAND_PAGING */ +/* rxe-mad.c */ +int rxe_process_mad (struct ib_device *ibdev,int mad_flags,u32 port_num, +const struct ib_wc *in_wc,const struct ib_grh *in_grh, +const struct ib_mad *in,struct ib_mad *out, +size_t *out_mad_size,u16 *out_mad_pkey_index); +#endif /* RXE_LOC_H */
drivers/infiniband/sw/rxe/rxe_mad.c new file +86 -0
86 lines not on this page · raw message
drivers/infiniband/sw/rxe/rxe_verbs.c +1 -0
@@ -1509,6 +1509,7 @@ static const struct ib_device_ops rxe_dev_ops = { .post_recv = rxe_post_recv, .post_send = rxe_post_send, .post_srq_recv = rxe_post_srq_recv, + .process_mad = rxe_process_mad, .query_ah = rxe_query_ah, .query_device = rxe_query_device, .query_pkey = rxe_query_pkey,
drivers/infiniband/sw/rxe/rxe_verbs.h +5 -0
@@ -466,6 +466,11 @@ static inline void rxe_counter_add(struct rxe_dev *rxe, enum rxe_counters index,atomic64_add (val, &rxe->stats_counters[index]); } +static inline s64 rxe_counter_get (struct rxe_dev *rxe,enum rxe_counters index) +{ +return atomic64_read (&rxe->stats_counters[index]); +} +static inline struct rxe_dev *to_rdev (struct ib_device *dev) { return dev ? container_of(dev, struct rxe_dev, ib_dev) : NULL;
Signature · 1 line
2.43.0
Re: [PATCH 3/3] RDMA/rxe: support perf mgmt
Quoted 76 lines · zhenwei pi
In RXE, hardware counters are already supported, but not in a standardized manner. For instance, user-space monitoring tools like atop only read from the *counters* directory. Therefore, it is necessary to add perf management support to RXE.
Signed-off-by zhenwei pi <[email protected]>--- drivers/infiniband/sw/rxe/Makefile | 1 + drivers/infiniband/sw/rxe/rxe_loc.h | 6 ++ drivers/infiniband/sw/rxe/rxe_mad.c | 86 +++++++++++++++++++++++++++ drivers/infiniband/sw/rxe/rxe_verbs.c | 1 + drivers/infiniband/sw/rxe/rxe_verbs.h | 5 ++ 5 files changed, 99 insertions(+)create mode 100644
drivers/infiniband/sw/rxe/rxe_mad.cdrivers/infiniband/sw/rxe/Makefile +1
-0@@ -22,6 +22,7 @@ rdma_rxe-y := \ rxe_mcast.o \ rxe_task.o \ rxe_net.o \ + rxe_mad.o \ rxe_hw_counters.o rdma_rxe-$(CONFIG_INFINIBAND_ON_DEMAND_PAGING) += rxe_odp.odrivers/infiniband/sw/rxe/rxe_loc.h +6
-0@@ -245,4 +245,10 @@ static inline int rxe_ib_advise_mr(struct ib_pd *pd,#endif /* CONFIG_INFINIBAND_ON_DEMAND_PAGING */ +/* rxe-mad.c */ +int rxe_process_mad (struct ib_device *ibdev,int mad_flags,u32 port_num, +const struct ib_wc *in_wc,const struct ib_grh *in_grh, +const struct ib_mad *in,struct ib_mad *out, +size_t *out_mad_size,u16 *out_mad_pkey_index); +#endif /* RXE_LOC_H */ drivers/infiniband/sw/rxe/rxe_mad.c new file +28
-0@@ -0,0 +1,86 @@ +// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB +/* +* Copyright (c) 2026 zhenwei pi <[email protected]> +*/ + +#include <rdma/ib_pma.h> +#include "rxe.h" +#include "rxe_hw_counters.h" + +static int rxe_process_pma_info (struct ib_mad *out) +{ +struct ib_class_port_info cpi = {}; + + cpi.capability_mask =IB_PMA_CLASS_CAP_EXT_WIDTH ; +memcpy ((out->data +40 ), &cpi,sizeof (cpi)); + +return IB_MAD_RESULT_SUCCESS |IB_MAD_RESULT_REPLY ; +} + +static int rxe_process_pma_counters (struct rxe_dev *rxe,struct ib_mad *out) +{ +struct ib_pma_portcounters *pma_cnt = + (struct ib_pma_portcounters *)(out->data +40 ); + + pma_cnt->link_downed_counter =rxe_counter_get (rxe,RXE_CNT_LINK_DOWNED ); + +return IB_MAD_RESULT_SUCCESS |IB_MAD_RESULT_REPLY ; +}
drivers/infiniband/sw/rxe/rxe_mad.c line 38
Quoted 10 lines
drivers/infiniband/sw/rxe/rxe_mad.c new file +10
-0+ +static int rxe_process_pma_counters_ext (struct rxe_dev *rxe,struct ib_mad *out) +{ +struct ib_pma_portcounters_ext *pma_cnt_ext = + (struct ib_pma_portcounters_ext *)(out->data +40 ); + + pma_cnt_ext->port_xmit_data =cpu_to_be64 (rxe_counter_get (rxe,RXE_CNT_SENT_BYTES ) >>2 ); + pma_cnt_ext->port_rcv_data =cpu_to_be64 (rxe_counter_get (rxe,RXE_CNT_RCVD_BYTES ) >>2 ); + pma_cnt_ext->port_xmit_packets =cpu_to_be64 (rxe_counter_get (rxe,RXE_CNT_SENT_PKTS )); + pma_cnt_ext->port_rcv_packets =cpu_to_be64 (rxe_counter_get (rxe,RXE_CNT_RCVD_PKTS ));
In rxe_process_pma_counters_ext, cpu_to_be64() is used to calculate the 64-bit counters.
But in rxe_process_pma_counters, pma_cnt->link_downed_counter = rxe_counter_get(), this misses the endianness conversion. The IB spec requires MAD data to be in Big Endian.
struct ib_pma_portcounters {
u8 reserved;
u8 port_select;
__be16 counter_select;
__be16 symbol_error_counter;
u8 link_error_recovery_counter;
u8 link_downed_counter; <--- It is u8.
__be16 port_rcv_errors;
__be16 port_rcv_remphys_errors;
__be16 port_rcv_switch_relay_errors;
__be16 port_xmit_discards;
u8 port_xmit_constraint_errors;
u8 port_rcv_constraint_errors;
u8 reserved1;
u8 link_overrun_errors; /* LocalLink: 7:4, BufferOverrun: 3:0 */
__be16 reserved2;
__be16 vl15_dropped;
__be32 port_xmit_data;
__be32 port_rcv_data;
__be32 port_xmit_packets;
__be32 port_rcv_packets;
__be32 port_xmit_wait;
} __packed;Please fix it.
Quoted 29 lines
drivers/infiniband/sw/rxe/rxe_mad.c new file +29
-0+ +return IB_MAD_RESULT_SUCCESS |IB_MAD_RESULT_REPLY ; +} + +static int rxe_process_perf_mgmt (struct rxe_dev *rxe,const struct ib_mad *in, +struct ib_mad *out) +{ +switch (in->mad_hdr.attr_id) { +case IB_PMA_CLASS_PORT_INFO : +return rxe_process_pma_info (out); + +case IB_PMA_PORT_COUNTERS : +return rxe_process_pma_counters (rxe, out); + +case IB_PMA_PORT_COUNTERS_EXT : +return rxe_process_pma_counters_ext (rxe, out); + +default : +break ; + } + +return IB_MAD_RESULT_FAILURE ; +} + +int rxe_process_mad (struct ib_device *ibdev,int mad_flags,u32 port_num, +const struct ib_wc *in_wc,const struct ib_grh *in_grh, +const struct ib_mad *in,struct ib_mad *out, +size_t *out_mad_size,u16 *out_mad_pkey_index) +{
drivers/infiniband/sw/rxe/rxe_mad.c line 77
Quoted 10 lines
drivers/infiniband/sw/rxe/rxe_mad.c new file +10
-0+struct rxe_dev *rxe =to_rdev (ibdev); +u8 mgmt_class = in->mad_hdr.mgmt_class; +u8 method = in->mad_hdr.method; + +if (port_num !=RXE_PORT ) +return IB_MAD_RESULT_FAILURE ; + +switch (mgmt_class) { +case IB_MGMT_CLASS_PERF_MGMT : +if (method ==IB_MGMT_METHOD_GET )
The function rxe_process_mad receives *in* MAD and populates the *out* MAD. Since the out buffer is eventually transmitted back to the caller—which could be a user-space application or a remote node on the network—it is important to ensure that any unassigned bytes (padding) within the out packet are initialized to zero. If the underlying framework does not pre-zero the out memory, areas outside of the memcpy((out->data + 40), &cpi, sizeof(cpi)) call could inadvertently leak sensitive residual kernel data.
Thus add this line in the function before filling in data:
memset(out->data, 0, 256);
256 is the length of out->data, please adjust this number based on the actual data length.
Please also consider adding support for IB_MGMT_METHOD_SET. This would allow the user space application to clear or reset specific metrics by calling atomic64_set(..., 0) on the underlying RXE counters.
Thanks a lot.
Zhu Yanjun
Quoted 37 lines
drivers/infiniband/sw/rxe/rxe_mad.c new file +9
-0+return rxe_process_perf_mgmt (rxe, in, out); +break ; + +default : +break ; + } + +return IB_MAD_RESULT_FAILURE ; +}drivers/infiniband/sw/rxe/rxe_verbs.c +1
-0@@ -1509,6 +1509,7 @@ static const struct ib_device_ops rxe_dev_ops = { .post_recv = rxe_post_recv, .post_send = rxe_post_send, .post_srq_recv = rxe_post_srq_recv, + .process_mad = rxe_process_mad, .query_ah = rxe_query_ah, .query_device = rxe_query_device, .query_pkey = rxe_query_pkey,drivers/infiniband/sw/rxe/rxe_verbs.h +5
-0@@ -466,6 +466,11 @@ static inline void rxe_counter_add(struct rxe_dev *rxe, enum rxe_counters index,atomic64_add (val, &rxe->stats_counters[index]); } +static inline s64 rxe_counter_get (struct rxe_dev *rxe,enum rxe_counters index) +{ +return atomic64_read (&rxe->stats_counters[index]); +} +static inline struct rxe_dev *to_rdev (struct ib_device *dev) { return dev ? container_of(dev, struct rxe_dev, ib_dev) : NULL;
Re: [PATCH 3/3] RDMA/rxe: support perf mgmt
Quoted 121 lines · Zhu Yanjun
Quoted 84 lines · zhenwei pi
In RXE, hardware counters are already supported, but not in a standardized manner. For instance, user-space monitoring tools like atop only read from the *counters* directory. Therefore, it is necessary to add perf management support to RXE.
Signed-off-by zhenwei pi <[email protected]>---
drivers/infiniband/sw/rxe/Makefile | 1 + drivers/infiniband/sw/rxe/rxe_loc.h | 6 ++ drivers/infiniband/sw/rxe/rxe_mad.c | 86 +++++++++++++++++++++++++++drivers/infiniband/sw/rxe/rxe_verbs.c| 1 +drivers/infiniband/sw/rxe/rxe_verbs.h| 5 ++5 files changed, 99 insertions(+)create mode 100644
drivers/infiniband/sw/rxe/rxe_mad.cdrivers/infiniband/sw/rxe/Makefile +1
-0@@ -22,6 +22,7 @@ rdma_rxe-y := \ rxe_mcast.o \ rxe_task.o \ rxe_net.o \ + rxe_mad.o \rxe_hw_counters.o rdma_rxe-$(CONFIG_INFINIBAND_ON_DEMAND_PAGING) += rxe_odp.o
drivers/infiniband/sw/rxe/rxe_loc.h +6
-0@@ -245,4 +245,10 @@ static inline int rxe_ib_advise_mr(struct ib_pd *pd, #endif /* CONFIG_INFINIBAND_ON_DEMAND_PAGING */ +/* rxe-mad.c */ +int rxe_process_mad (struct ib_device *ibdev,int mad_flags,u32 port_num, + const struct ib_wc *in_wc, const struct ib_grh *in_grh, + const struct ib_mad *in, struct ib_mad *out, + size_t *out_mad_size, u16 *out_mad_pkey_index); + #endif /* RXE_LOC_H */drivers/infiniband/sw/rxe/rxe_mad.c new file +20
-0@@ -0,0 +1,86 @@ +// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB +/* +* Copyright (c) 2026 zhenwei pi <[email protected]> +*/ + +#include <rdma/ib_pma.h> +#include "rxe.h" +#include "rxe_hw_counters.h" + +static int rxe_process_pma_info (struct ib_mad *out) +{ + struct ib_class_port_info cpi = {}; + + cpi.capability_mask = IB_PMA_CLASS_CAP_EXT_WIDTH; + memcpy((out->data + 40), &cpi, sizeof(cpi)); + + return IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_REPLY; +} + +static int rxe_process_pma_counters(struct rxe_dev *rxe, struct
ib_mad*out)drivers/infiniband/sw/rxe/rxe_mad.c new file +5
-0+{ + struct ib_pma_portcounters *pma_cnt = + (struct ib_pma_portcounters *)(out->data + 40); + + pma_cnt->link_downed_counter = rxe_counter_get(rxe,
RXE_CNT_LINK_DOWNED);drivers/infiniband/sw/rxe/rxe_mad.c new file +5
-0+ + return IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_REPLY; +} + +static int rxe_process_pma_counters_ext(struct rxe_dev *rxe, struct
ib_mad*out)drivers/infiniband/sw/rxe/rxe_mad.c new file +5
-0+{ + struct ib_pma_portcounters_ext *pma_cnt_ext = + (struct ib_pma_portcounters_ext *)(out->data + 40); + + pma_cnt_ext->port_xmit_data = cpu_to_be64(rxe_counter_get(rxe,Quoted 7 lines
RXE_CNT_SENT_BYTES) >> 2);
+pma_cnt_ext->port_rcv_data=cpu_to_be64(rxe_counter_get(rxe, RXE_CNT_RCVD_BYTES) >> 2);
+pma_cnt_ext->port_xmit_packets=cpu_to_be64(rxe_counter_get(rxe, RXE_CNT_SENT_PKTS));
+pma_cnt_ext->port_rcv_packets=cpu_to_be64(rxe_counter_get(rxe, RXE_CNT_RCVD_PKTS));In
rxe_process_pma_counters_ext,cpu_to_be64()is used to calculate the 64-bit counters.But in
rxe_process_pma_counters,pma_cnt->link_downed_counter=rxe_counter_get(), this misses the endianness conversion. The IB spec requires MAD data to be in Big Endian.struct
ib_pma_portcounters{
u8 reserved;
u8port_select;
__be16counter_select;
__be16symbol_error_counter;
u8link_error_recovery_counter;
u8 link_downed_counter; <--- It is u8.__be16port_rcv_errors;
__be16port_rcv_remphys_errors;
__be16port_rcv_switch_relay_errors;
__be16port_xmit_discards;
u8port_xmit_constraint_errors;
u8port_rcv_constraint_errors;
u8 reserved1;
u8link_overrun_errors; /* LocalLink: 7:4, BufferOverrun: 3:0 */__be16reserved2;
__be16vl15_dropped;
__be32port_xmit_data;
__be32port_rcv_data;
__be32port_xmit_packets;
__be32port_rcv_packets;
__be32port_xmit_wait;
}__packed;
Quoted 1 line
Please fix it.
OK.
Quoted 45 lines
Quoted 32 lines
drivers/infiniband/sw/rxe/rxe_mad.c new file +5
-0+ + return IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_REPLY; +} + +static int rxe_process_perf_mgmt(struct rxe_dev *rxe, const struct
ib_mad*in,
+ structib_mad*out)drivers/infiniband/sw/rxe/rxe_mad.c new file +19
-0+{ + switch (in->mad_hdr.attr_id) { + case IB_PMA_CLASS_PORT_INFO: + return rxe_process_pma_info(out); + + case IB_PMA_PORT_COUNTERS: + return rxe_process_pma_counters(rxe, out); + + case IB_PMA_PORT_COUNTERS_EXT: + return rxe_process_pma_counters_ext(rxe, out); + + default: + break; + } + + return IB_MAD_RESULT_FAILURE; +} + +int rxe_process_mad(struct ib_device *ibdev, int mad_flags, u32
port_num,
+ const structib_wc*in_wc, const structib_grh*in_grh, + const structib_mad*in, structib_mad*out, +size_t*out_mad_size, u16*out_mad_pkey_index)drivers/infiniband/sw/rxe/rxe_mad.c new file +1
-0+{drivers/infiniband/sw/rxe/rxe_mad.c line 77
Quoted 10 lines
drivers/infiniband/sw/rxe/rxe_mad.c new file +10
-0+ struct rxe_dev *rxe = to_rdev(ibdev); + u8 mgmt_class = in->mad_hdr.mgmt_class; + u8 method = in->mad_hdr.method; + + if (port_num != RXE_PORT) + return IB_MAD_RESULT_FAILURE; + + switch (mgmt_class) { + case IB_MGMT_CLASS_PERF_MGMT: + if (method == IB_MGMT_METHOD_GET)The function
rxe_process_madreceives *in* MAD and populates the *out* MAD. Since the out buffer is eventually transmitted back to the caller— which could be a user-space application or a remote node on the network— it is important to ensure that any unassigned bytes (padding) within the out packet are initialized to zero. If the underlying framework does not pre-zero the out memory, areas outside of thememcpy((out->data + 40), &cpi, sizeof(cpi))call could inadvertently leak sensitive residual kernel data.
Quoted 6 lines
Thus add this line in the function before filling in data:
memset(out->data, 0, 256);256 is the length of
out->data, please adjust this number based on the actual data length.
I notice that all of the callers use kzalloc to allocate out MAD data, so there is no need to clear it again. In addition, other drivers like mlx5 does not clear memory too. I also have no objection to clear buffer in RXE, please let me know if it's necessary.
Quoted 2 lines
Please also consider adding support for
IB_MGMT_METHOD_SET. This would allow the user space application to clear or reset specific metrics by callingatomic64_set(..., 0)on the underlying RXE counters.
Sure, I'll rename functions, for example, rename 'rxe_process_perf_mgmt' to 'rxe_get_perf_mgmt'. 'rxe_set_perf_mgmt' will be supported in the future easily. I'm not going to support SET method in this serries because of the lack of *store* opeation from these sysfs attributes.
Yanjun, thanks!
Quoted 41 lines
Thanks a lot.
Zhu YanjunQuoted 38 lines
+ return
rxe_process_perf_mgmt(rxe, in, out); + break;drivers/infiniband/sw/rxe/rxe_mad.c new file +7
-0+ + default: + break; + } + + return IB_MAD_RESULT_FAILURE; +}drivers/infiniband/sw/rxe/rxe_verbs.c +1
-0@@ -1509,6 +1509,7 @@ static const struct ib_device_ops rxe_dev_ops = { .post_recv = rxe_post_recv, .post_send = rxe_post_send, .post_srq_recv = rxe_post_srq_recv, + .process_mad = rxe_process_mad, .query_ah = rxe_query_ah, .query_device = rxe_query_device, .query_pkey = rxe_query_pkey,drivers/infiniband/sw/rxe/rxe_verbs.h +5
-0@@ -466,6 +466,11 @@ static inline void rxe_counter_add(struct rxe_dev *rxe, enum rxe_counters index, atomic64_add(val, &rxe->stats_counters[index]); } +static inline s64 rxe_counter_get (struct rxe_dev *rxe,enum rxe_counters index) +{ + return atomic64_read(&rxe->stats_counters[index]); +} + static inline struct rxe_dev *to_rdev(struct ib_device *dev) {return dev ?
container_of(dev, struct rxe_dev, ib_dev): NULL;