Support PERF MGMT for RXE
Subsystems (2)
v3 cover Support PERF MGMT for RXE
v3:
- merge 'RDMA/rxe: use rxe_counter_get' into previous commit
- zero *out* MAD memory
- return success with error status rather than failure to avoid uplayer hang
v2:
- Fix overflow for PMA counter *link_downed_counter*
- Use *rxe_counter_get* instead of *atomic64_read* for hw-counters
v1:
Support PERF MGMT for RXE, add sent/received bytes for RXE counters, also improve coding style.
zhenwei pi (3):
RDMA/rxe: use RXE_PORT instead of magic number 1 RDMA/rxe: add SENT/RCVD bytes
RDMA/rxe: support perf mgmt GET method
drivers/infiniband/sw/rxe/Makefile | 1 + drivers/infiniband/sw/rxe/rxe_hw_counters.c | 4 +- drivers/infiniband/sw/rxe/rxe_hw_counters.h | 2 + drivers/infiniband/sw/rxe/rxe_loc.h | 6 ++ drivers/infiniband/sw/rxe/rxe_mad.c | 93 +++++++++++++++++++++ drivers/infiniband/sw/rxe/rxe_net.c | 7 +- drivers/infiniband/sw/rxe/rxe_recv.c | 1 + drivers/infiniband/sw/rxe/rxe_verbs.c | 9 +- drivers/infiniband/sw/rxe/rxe_verbs.h | 11 +++ 9 files changed, 126 insertions(+), 8 deletions(-) create mode 100644 drivers/infiniband/sw/rxe/rxe_mad.c
Signature · 1 line
2.43.0
PATCH v3 1/3 RDMA/rxe: use RXE_PORT instead of magic number 1
Align with the existing code:
static ... rxe_ib_device_get_netdev(struct ib_device *dev)
{
return ib_device_get_netdev(dev, RXE_PORT);
}Use *RXE_PORT* instead of magic number 1 for all.
--- drivers/infiniband/sw/rxe/rxe_net.c | 6 +++--- drivers/infiniband/sw/rxe/rxe_verbs.c | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-)
drivers/infiniband/sw/rxe/rxe_net.c +3 -3
@@ -234,7 +234,7 @@ static int rxe_udp_encap_recv(struct sock *sk, struct sk_buff *skb) udph =udp_hdr (skb); pkt->rxe = rxe; - pkt->port_num =1 ; + pkt->port_num =RXE_PORT ; pkt->hdr = (u8 *)(udph +1 ); pkt->mask =RXE_GRH_MASK ; pkt->paylen =be16_to_cpu (udph->len) -sizeof (*udph);
@@ -535,7 +535,7 @@ struct sk_buff *rxe_init_packet(struct rxe_dev *rxe, struct rxe_av *av,struct sk_buff *skb =NULL ;struct net_device *ndev;const struct ib_gid_attr *attr; -const int port_num =1 ; +const int port_num =RXE_PORT ; attr =rdma_get_gid_attr (&rxe->ib_dev, port_num, av->grh.sgid_index);if (IS_ERR (attr))
@@ -630,7 +630,7 @@ static void rxe_port_event(struct rxe_dev *rxe,struct ib_event ev; ev.device = &rxe->ib_dev; - ev.element.port_num =1 ; + ev.element.port_num =RXE_PORT ; ev.event = event;ib_dispatch_event (&ev);
drivers/infiniband/sw/rxe/rxe_verbs.c +4 -4
@@ -44,7 +44,7 @@ static int rxe_query_port(struct ib_device *ibdev,struct net_device *ndev;int err, ret; -if (port_num !=1 ) { +if (port_num !=RXE_PORT ) { err = -EINVAL ;rxe_dbg_dev (rxe,"bad port_num = %d\n" , port_num); goto err_out;
@@ -147,7 +147,7 @@ static int rxe_modify_port(struct ib_device *ibdev, u32 port_num,struct rxe_port *port;int err; -if (port_num !=1 ) { +if (port_num !=RXE_PORT ) { err = -EINVAL ;rxe_dbg_dev (rxe,"bad port_num = %d\n" , port_num); goto err_out;
@@ -180,7 +180,7 @@ static enum rdma_link_layer rxe_get_link_layer(struct ib_device *ibdev,struct rxe_dev *rxe =to_rdev (ibdev);int err; -if (port_num !=1 ) { +if (port_num !=RXE_PORT ) { err = -EINVAL ;rxe_dbg_dev (rxe,"bad port_num = %d\n" , port_num); goto err_out;
@@ -200,7 +200,7 @@ static int rxe_port_immutable(struct ib_device *ibdev, u32 port_num,struct ib_port_attr attr = {};int err; -if (port_num !=1 ) { +if (port_num !=RXE_PORT ) { err = -EINVAL ;rxe_dbg_dev (rxe,"bad port_num = %d\n" , port_num); goto err_out;
Signature · 1 line
2.43.0
PATCH v3 2/3 RDMA/rxe: add SENT/RCVD bytes
In reply to zhenwei piThere is a lack of sent/received counter in bytes.
--- drivers/infiniband/sw/rxe/rxe_hw_counters.c | 2 ++ drivers/infiniband/sw/rxe/rxe_hw_counters.h | 2 ++ drivers/infiniband/sw/rxe/rxe_net.c | 1 + drivers/infiniband/sw/rxe/rxe_recv.c | 1 + drivers/infiniband/sw/rxe/rxe_verbs.h | 6 ++++++ 5 files changed, 12 insertions(+)
drivers/infiniband/sw/rxe/rxe_hw_counters.c +2 -0
@@ -22,6 +22,8 @@ static const struct rdma_stat_desc rxe_counter_descs[] = { [RXE_CNT_LINK_DOWNED].name = "link_downed", [RXE_CNT_RDMA_SEND].name = "rdma_sends", [RXE_CNT_RDMA_RECV].name = "rdma_recvs", + [RXE_CNT_SENT_BYTES].name = "sent_bytes", + [RXE_CNT_RCVD_BYTES].name = "rcvd_bytes", }; int rxe_ib_get_hw_stats(struct ib_device *ibdev,
drivers/infiniband/sw/rxe/rxe_hw_counters.h +2 -0
@@ -26,6 +26,8 @@ enum rxe_counters {RXE_CNT_LINK_DOWNED ,RXE_CNT_RDMA_SEND ,RXE_CNT_RDMA_RECV , +RXE_CNT_SENT_BYTES , +RXE_CNT_RCVD_BYTES , RXE_NUM_OF_COUNTERS };
drivers/infiniband/sw/rxe/rxe_net.c +1 -0
@@ -519,6 +519,7 @@ int rxe_xmit_packet(struct rxe_qp *qp, struct rxe_pkt_info *pkt, }rxe_counter_inc (rxe,RXE_CNT_SENT_PKTS ); +rxe_counter_add (rxe,RXE_CNT_SENT_BYTES , skb->len); goto done; drop:
drivers/infiniband/sw/rxe/rxe_recv.c +1 -0
@@ -342,6 +342,7 @@ void rxe_rcv(struct sk_buff *skb) goto drop;rxe_counter_inc (rxe,RXE_CNT_RCVD_PKTS ); +rxe_counter_add (rxe,RXE_CNT_RCVD_BYTES , skb->len);if (unlikely (bth_qpn (pkt) ==IB_MULTICAST_QPN ))rxe_rcv_mcast_pkt (rxe, skb);
drivers/infiniband/sw/rxe/rxe_verbs.h +6 -0
@@ -460,6 +460,12 @@ static inline void rxe_counter_inc(struct rxe_dev *rxe, enum rxe_counters index)atomic64_inc (&rxe->stats_counters[index]); } +static inline void rxe_counter_add (struct rxe_dev *rxe,enum rxe_counters index, +s64 val) +{ +atomic64_add (val, &rxe->stats_counters[index]); +} +static inline struct rxe_dev *to_rdev (struct ib_device *dev) { return dev ? container_of(dev, struct rxe_dev, ib_dev) : NULL;
Signature · 1 line
2.43.0
PATCH v3 3/3 RDMA/rxe: support perf mgmt GET method
In reply to zhenwei piIn RXE, hardware counters are already supported, but not in a standardized manner. For instance, user-space monitoring tools like atop only read from the *counters* directory. Therefore, it is necessary to add perf management support to RXE.
Also use rxe_counter_get instead of raw atomic64_read in hw-counters.
--- drivers/infiniband/sw/rxe/Makefile | 1 + drivers/infiniband/sw/rxe/rxe_hw_counters.c | 2 +- drivers/infiniband/sw/rxe/rxe_loc.h | 6 ++ drivers/infiniband/sw/rxe/rxe_mad.c | 93 +++++++++++++++++++++ drivers/infiniband/sw/rxe/rxe_verbs.c | 1 + drivers/infiniband/sw/rxe/rxe_verbs.h | 5 ++ 6 files changed, 107 insertions(+), 1 deletion(-) create mode 100644 drivers/infiniband/sw/rxe/rxe_mad.c
drivers/infiniband/sw/rxe/Makefile +1 -0
@@ -22,6 +22,7 @@ rdma_rxe-y := \ rxe_mcast.o \ rxe_task.o \ rxe_net.o \ + rxe_mad.o \ rxe_hw_counters.o rdma_rxe-$(CONFIG_INFINIBAND_ON_DEMAND_PAGING) += rxe_odp.o
drivers/infiniband/sw/rxe/rxe_hw_counters.c +1 -1
@@ -37,7 +37,7 @@ int rxe_ib_get_hw_stats(struct ib_device *ibdev,return -EINVAL ;for (cnt =0 ; cnt <ARRAY_SIZE (rxe_counter_descs); cnt++) - stats->value[cnt] =atomic64_read (&dev->stats_counters[cnt]); + stats->value[cnt] =rxe_counter_get (dev, cnt);return ARRAY_SIZE (rxe_counter_descs); }
drivers/infiniband/sw/rxe/rxe_loc.h +6 -0
@@ -245,4 +245,10 @@ static inline int rxe_ib_advise_mr(struct ib_pd *pd,#endif /* CONFIG_INFINIBAND_ON_DEMAND_PAGING */ +/* rxe-mad.c */ +int rxe_process_mad (struct ib_device *ibdev,int mad_flags,u32 port_num, +const struct ib_wc *in_wc,const struct ib_grh *in_grh, +const struct ib_mad *in,struct ib_mad *out, +size_t *out_mad_size,u16 *out_mad_pkey_index); +#endif /* RXE_LOC_H */
drivers/infiniband/sw/rxe/rxe_mad.c new file +93 -0
93 lines not on this page · raw message
drivers/infiniband/sw/rxe/rxe_verbs.c +1 -0
@@ -1509,6 +1509,7 @@ static const struct ib_device_ops rxe_dev_ops = { .post_recv = rxe_post_recv, .post_send = rxe_post_send, .post_srq_recv = rxe_post_srq_recv, + .process_mad = rxe_process_mad, .query_ah = rxe_query_ah, .query_device = rxe_query_device, .query_pkey = rxe_query_pkey,
drivers/infiniband/sw/rxe/rxe_verbs.h +5 -0
@@ -466,6 +466,11 @@ static inline void rxe_counter_add(struct rxe_dev *rxe, enum rxe_counters index,atomic64_add (val, &rxe->stats_counters[index]); } +static inline s64 rxe_counter_get (struct rxe_dev *rxe,enum rxe_counters index) +{ +return atomic64_read (&rxe->stats_counters[index]); +} +static inline struct rxe_dev *to_rdev (struct ib_device *dev) { return dev ? container_of(dev, struct rxe_dev, ib_dev) : NULL;
Signature · 1 line
2.43.0
Re: [PATCH v3 1/3] RDMA/rxe: use RXE_PORT instead of magic number 1
In reply to zhenwei piQuoted 8 lines · zhenwei pi
Align with the existing code:
static ...rxe_ib_device_get_netdev(struct ib_device *dev){ return ib_device_get_netdev(dev, RXE_PORT); }Use *RXE_PORT* instead of magic number 1 for all.
Thanks a lot.
Zhu Yanjun
Quoted 77 lines
Signed-off-by zhenwei pi <[email protected]>--- drivers/infiniband/sw/rxe/rxe_net.c | 6 +++--- drivers/infiniband/sw/rxe/rxe_verbs.c | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-)drivers/infiniband/sw/rxe/rxe_net.c +3
-3@@ -234,7 +234,7 @@ static int rxe_udp_encap_recv(struct sock *sk, struct sk_buff *skb) udph =udp_hdr (skb); pkt->rxe = rxe; - pkt->port_num =1 ; + pkt->port_num =RXE_PORT ; pkt->hdr = (u8 *)(udph +1 ); pkt->mask =RXE_GRH_MASK ; pkt->paylen =be16_to_cpu (udph->len) -sizeof (*udph);@@ -535,7 +535,7 @@ struct sk_buff *rxe_init_packet(struct rxe_dev *rxe, struct rxe_av *av,struct sk_buff *skb =NULL ;struct net_device *ndev;const struct ib_gid_attr *attr; -const int port_num =1 ; +const int port_num =RXE_PORT ; attr =rdma_get_gid_attr (&rxe->ib_dev, port_num, av->grh.sgid_index);if (IS_ERR (attr))@@ -630,7 +630,7 @@ static void rxe_port_event(struct rxe_dev *rxe,struct ib_event ev; ev.device = &rxe->ib_dev; - ev.element.port_num =1 ; + ev.element.port_num =RXE_PORT ; ev.event = event;ib_dispatch_event (&ev);drivers/infiniband/sw/rxe/rxe_verbs.c +4
-4@@ -44,7 +44,7 @@ static int rxe_query_port(struct ib_device *ibdev,struct net_device *ndev;int err, ret; -if (port_num !=1 ) { +if (port_num !=RXE_PORT ) { err = -EINVAL ;rxe_dbg_dev (rxe,"bad port_num = %d\n" , port_num); goto err_out;@@ -147,7 +147,7 @@ static int rxe_modify_port(struct ib_device *ibdev, u32 port_num,struct rxe_port *port;int err; -if (port_num !=1 ) { +if (port_num !=RXE_PORT ) { err = -EINVAL ;rxe_dbg_dev (rxe,"bad port_num = %d\n" , port_num); goto err_out;@@ -180,7 +180,7 @@ static enum rdma_link_layer rxe_get_link_layer(struct ib_device *ibdev,struct rxe_dev *rxe =to_rdev (ibdev);int err; -if (port_num !=1 ) { +if (port_num !=RXE_PORT ) { err = -EINVAL ;rxe_dbg_dev (rxe,"bad port_num = %d\n" , port_num); goto err_out;@@ -200,7 +200,7 @@ static int rxe_port_immutable(struct ib_device *ibdev, u32 port_num,struct ib_port_attr attr = {};int err; -if (port_num !=1 ) { +if (port_num !=RXE_PORT ) { err = -EINVAL ;rxe_dbg_dev (rxe,"bad port_num = %d\n" , port_num); goto err_out;
Re: [PATCH v3 2/3] RDMA/rxe: add SENT/RCVD bytes
In reply to zhenwei piQuoted 2 lines · zhenwei pi
There is a lack of sent/received counter in bytes.
Thanks a lot.
Zhu Yanjun
Quoted 76 lines
Signed-off-by zhenwei pi <[email protected]>--- drivers/infiniband/sw/rxe/rxe_hw_counters.c | 2 ++ drivers/infiniband/sw/rxe/rxe_hw_counters.h | 2 ++ drivers/infiniband/sw/rxe/rxe_net.c | 1 + drivers/infiniband/sw/rxe/rxe_recv.c | 1 + drivers/infiniband/sw/rxe/rxe_verbs.h | 6 ++++++ 5 files changed, 12 insertions(+)drivers/infiniband/sw/rxe/rxe_hw_counters.c +2
-0@@ -22,6 +22,8 @@ static const struct rdma_stat_desc rxe_counter_descs[] = { [RXE_CNT_LINK_DOWNED].name = "link_downed", [RXE_CNT_RDMA_SEND].name = "rdma_sends", [RXE_CNT_RDMA_RECV].name = "rdma_recvs", + [RXE_CNT_SENT_BYTES].name = "sent_bytes", + [RXE_CNT_RCVD_BYTES].name = "rcvd_bytes", }; int rxe_ib_get_hw_stats(struct ib_device *ibdev,drivers/infiniband/sw/rxe/rxe_hw_counters.h +2
-0@@ -26,6 +26,8 @@ enum rxe_counters {RXE_CNT_LINK_DOWNED ,RXE_CNT_RDMA_SEND ,RXE_CNT_RDMA_RECV , +RXE_CNT_SENT_BYTES , +RXE_CNT_RCVD_BYTES , RXE_NUM_OF_COUNTERS };drivers/infiniband/sw/rxe/rxe_net.c +1
-0@@ -519,6 +519,7 @@ int rxe_xmit_packet(struct rxe_qp *qp, struct rxe_pkt_info *pkt, }rxe_counter_inc (rxe,RXE_CNT_SENT_PKTS ); +rxe_counter_add (rxe,RXE_CNT_SENT_BYTES , skb->len); goto done; drop:drivers/infiniband/sw/rxe/rxe_recv.c +1
-0@@ -342,6 +342,7 @@ void rxe_rcv(struct sk_buff *skb) goto drop;rxe_counter_inc (rxe,RXE_CNT_RCVD_PKTS ); +rxe_counter_add (rxe,RXE_CNT_RCVD_BYTES , skb->len);if (unlikely (bth_qpn (pkt) ==IB_MULTICAST_QPN ))rxe_rcv_mcast_pkt (rxe, skb);drivers/infiniband/sw/rxe/rxe_verbs.h +6
-0@@ -460,6 +460,12 @@ static inline void rxe_counter_inc(struct rxe_dev *rxe, enum rxe_counters index)atomic64_inc (&rxe->stats_counters[index]); } +static inline void rxe_counter_add (struct rxe_dev *rxe,enum rxe_counters index, +s64 val) +{ +atomic64_add (val, &rxe->stats_counters[index]); +} +static inline struct rxe_dev *to_rdev (struct ib_device *dev) { return dev ? container_of(dev, struct rxe_dev, ib_dev) : NULL;
Re: [PATCH v3 3/3] RDMA/rxe: support perf mgmt GET method
In reply to zhenwei piQuoted 7 lines · zhenwei pi
In RXE, hardware counters are already supported, but not in a standardized manner. For instance, user-space monitoring tools like atop only read from the *counters* directory. Therefore, it is necessary to add perf management support to RXE.
Also use
rxe_counter_getinstead of rawatomic64_readin hw-counters.
Thanks a lot.
Zhu Yanjun
Quoted 178 lines
Signed-off-by zhenwei pi <[email protected]>--- drivers/infiniband/sw/rxe/Makefile | 1 + drivers/infiniband/sw/rxe/rxe_hw_counters.c | 2 +- drivers/infiniband/sw/rxe/rxe_loc.h | 6 ++ drivers/infiniband/sw/rxe/rxe_mad.c | 93 +++++++++++++++++++++ drivers/infiniband/sw/rxe/rxe_verbs.c | 1 + drivers/infiniband/sw/rxe/rxe_verbs.h | 5 ++ 6 files changed, 107 insertions(+), 1 deletion(-)create mode 100644
drivers/infiniband/sw/rxe/rxe_mad.cdrivers/infiniband/sw/rxe/Makefile +1
-0@@ -22,6 +22,7 @@ rdma_rxe-y := \ rxe_mcast.o \ rxe_task.o \ rxe_net.o \ + rxe_mad.o \ rxe_hw_counters.o rdma_rxe-$(CONFIG_INFINIBAND_ON_DEMAND_PAGING) += rxe_odp.odrivers/infiniband/sw/rxe/rxe_hw_counters.c +1
-1@@ -37,7 +37,7 @@ int rxe_ib_get_hw_stats(struct ib_device *ibdev,return -EINVAL ;for (cnt =0 ; cnt <ARRAY_SIZE (rxe_counter_descs); cnt++) - stats->value[cnt] =atomic64_read (&dev->stats_counters[cnt]); + stats->value[cnt] =rxe_counter_get (dev, cnt);return ARRAY_SIZE (rxe_counter_descs); }drivers/infiniband/sw/rxe/rxe_loc.h +6
-0@@ -245,4 +245,10 @@ static inline int rxe_ib_advise_mr(struct ib_pd *pd,#endif /* CONFIG_INFINIBAND_ON_DEMAND_PAGING */ +/* rxe-mad.c */ +int rxe_process_mad (struct ib_device *ibdev,int mad_flags,u32 port_num, +const struct ib_wc *in_wc,const struct ib_grh *in_grh, +const struct ib_mad *in,struct ib_mad *out, +size_t *out_mad_size,u16 *out_mad_pkey_index); +#endif /* RXE_LOC_H */ drivers/infiniband/sw/rxe/rxe_mad.c new file +93
-0@@ -0,0 +1,93 @@ +// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB +/* +* Copyright (c) 2026 zhenwei pi <[email protected]> +*/ + +#include <rdma/ib_pma.h> +#include "rxe.h" +#include "rxe_hw_counters.h" + +static int rxe_get_pma_info (struct ib_mad *out) +{ +struct ib_class_port_info cpi = {}; + + cpi.capability_mask =IB_PMA_CLASS_CAP_EXT_WIDTH ; +memcpy ((out->data +40 ), &cpi,sizeof (cpi)); + +return IB_MAD_RESULT_SUCCESS |IB_MAD_RESULT_REPLY ; +} + +static int rxe_get_pma_counters (struct rxe_dev *rxe,struct ib_mad *out) +{ +struct ib_pma_portcounters *pma_cnt = (struct ib_pma_portcounters *)(out->data +40 ); +s64 val; + +/* IBA release 1.8, 16.1.3.5: During operation, instead of overflowing, they shall stop +* at all ones. +*/ + val =rxe_counter_get (rxe,RXE_CNT_LINK_DOWNED ); +if (val >U8_MAX ) + pma_cnt->link_downed_counter =U8_MAX ; +else + pma_cnt->link_downed_counter = (u8 )val; + +return IB_MAD_RESULT_SUCCESS |IB_MAD_RESULT_REPLY ; +} + +static int rxe_get_pma_counters_ext (struct rxe_dev *rxe,struct ib_mad *out) +{ +struct ib_pma_portcounters_ext *pma_cnt_ext = + (struct ib_pma_portcounters_ext *)(out->data +40 ); + + pma_cnt_ext->port_xmit_data =cpu_to_be64 (rxe_counter_get (rxe,RXE_CNT_SENT_BYTES ) >>2 ); + pma_cnt_ext->port_rcv_data =cpu_to_be64 (rxe_counter_get (rxe,RXE_CNT_RCVD_BYTES ) >>2 ); + pma_cnt_ext->port_xmit_packets =cpu_to_be64 (rxe_counter_get (rxe,RXE_CNT_SENT_PKTS )); + pma_cnt_ext->port_rcv_packets =cpu_to_be64 (rxe_counter_get (rxe,RXE_CNT_RCVD_PKTS )); + +return IB_MAD_RESULT_SUCCESS |IB_MAD_RESULT_REPLY ; +} + +static int rxe_get_perf_mgmt (struct rxe_dev *rxe,const struct ib_mad *in,struct ib_mad *out) +{ +switch (in->mad_hdr.attr_id) { +case IB_PMA_CLASS_PORT_INFO : +return rxe_get_pma_info (out); + +case IB_PMA_PORT_COUNTERS : +return rxe_get_pma_counters (rxe, out); + +case IB_PMA_PORT_COUNTERS_EXT : +return rxe_get_pma_counters_ext (rxe, out); + +default : + out->mad_hdr.status =cpu_to_be16 (IB_MGMT_MAD_STATUS_UNSUPPORTED_METHOD_ATTRIB ); +return IB_MAD_RESULT_SUCCESS |IB_MAD_RESULT_REPLY ; + } +} + +int rxe_process_mad (struct ib_device *ibdev,int mad_flags,u32 port_num, +const struct ib_wc *in_wc,const struct ib_grh *in_grh, +const struct ib_mad *in,struct ib_mad *out, +size_t *out_mad_size,u16 *out_mad_pkey_index) +{ +struct rxe_dev *rxe =to_rdev (ibdev); +u8 mgmt_class = in->mad_hdr.mgmt_class; +u8 method = in->mad_hdr.method; + +if (port_num !=RXE_PORT ) +return IB_MAD_RESULT_FAILURE ; + +memset (out,0 ,sizeof (*out)); +switch (mgmt_class) { +case IB_MGMT_CLASS_PERF_MGMT : +if (method ==IB_MGMT_METHOD_GET ) +return rxe_get_perf_mgmt (rxe, in, out); +break ; + +default : +break ; + } + + out->mad_hdr.status =cpu_to_be16 (IB_MGMT_MAD_STATUS_UNSUPPORTED_METHOD ); +return IB_MAD_RESULT_SUCCESS |IB_MAD_RESULT_REPLY ; +}drivers/infiniband/sw/rxe/rxe_verbs.c +1
-0@@ -1509,6 +1509,7 @@ static const struct ib_device_ops rxe_dev_ops = { .post_recv = rxe_post_recv, .post_send = rxe_post_send, .post_srq_recv = rxe_post_srq_recv, + .process_mad = rxe_process_mad, .query_ah = rxe_query_ah, .query_device = rxe_query_device, .query_pkey = rxe_query_pkey,drivers/infiniband/sw/rxe/rxe_verbs.h +5
-0@@ -466,6 +466,11 @@ static inline void rxe_counter_add(struct rxe_dev *rxe, enum rxe_counters index,atomic64_add (val, &rxe->stats_counters[index]); } +static inline s64 rxe_counter_get (struct rxe_dev *rxe,enum rxe_counters index) +{ +return atomic64_read (&rxe->stats_counters[index]); +} +static inline struct rxe_dev *to_rdev (struct ib_device *dev) { return dev ? container_of(dev, struct rxe_dev, ib_dev) : NULL;
Re: [PATCH v3 1/3] RDMA/rxe: use RXE_PORT instead of magic number 1
In reply to zhenwei piQuoted 5 lines · zhenwei pi
Align with the existing code:
static ...rxe_ib_device_get_netdev(struct ib_device *dev){ return ib_device_get_netdev(dev, RXE_PORT); }
Please submit the inverse patch that removes the rxe_ib_device_get_netdev() wrapper and the RXE_PORT definition. These additions do not improve readability, and RXE has always had only a single port.
Thanks
Quoted 84 lines
Use *RXE_PORT* instead of magic number 1 for all.
Signed-off-by zhenwei pi <[email protected]>--- drivers/infiniband/sw/rxe/rxe_net.c | 6 +++--- drivers/infiniband/sw/rxe/rxe_verbs.c | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-)drivers/infiniband/sw/rxe/rxe_net.c +3
-3@@ -234,7 +234,7 @@ static int rxe_udp_encap_recv(struct sock *sk, struct sk_buff *skb) udph =udp_hdr (skb); pkt->rxe = rxe; - pkt->port_num =1 ; + pkt->port_num =RXE_PORT ; pkt->hdr = (u8 *)(udph +1 ); pkt->mask =RXE_GRH_MASK ; pkt->paylen =be16_to_cpu (udph->len) -sizeof (*udph);@@ -535,7 +535,7 @@ struct sk_buff *rxe_init_packet(struct rxe_dev *rxe, struct rxe_av *av,struct sk_buff *skb =NULL ;struct net_device *ndev;const struct ib_gid_attr *attr; -const int port_num =1 ; +const int port_num =RXE_PORT ; attr =rdma_get_gid_attr (&rxe->ib_dev, port_num, av->grh.sgid_index);if (IS_ERR (attr))@@ -630,7 +630,7 @@ static void rxe_port_event(struct rxe_dev *rxe,struct ib_event ev; ev.device = &rxe->ib_dev; - ev.element.port_num =1 ; + ev.element.port_num =RXE_PORT ; ev.event = event;ib_dispatch_event (&ev);drivers/infiniband/sw/rxe/rxe_verbs.c +4
-4@@ -44,7 +44,7 @@ static int rxe_query_port(struct ib_device *ibdev,struct net_device *ndev;int err, ret; -if (port_num !=1 ) { +if (port_num !=RXE_PORT ) { err = -EINVAL ;rxe_dbg_dev (rxe,"bad port_num = %d\n" , port_num); goto err_out;@@ -147,7 +147,7 @@ static int rxe_modify_port(struct ib_device *ibdev, u32 port_num,struct rxe_port *port;int err; -if (port_num !=1 ) { +if (port_num !=RXE_PORT ) { err = -EINVAL ;rxe_dbg_dev (rxe,"bad port_num = %d\n" , port_num); goto err_out;@@ -180,7 +180,7 @@ static enum rdma_link_layer rxe_get_link_layer(struct ib_device *ibdev,struct rxe_dev *rxe =to_rdev (ibdev);int err; -if (port_num !=1 ) { +if (port_num !=RXE_PORT ) { err = -EINVAL ;rxe_dbg_dev (rxe,"bad port_num = %d\n" , port_num); goto err_out;@@ -200,7 +200,7 @@ static int rxe_port_immutable(struct ib_device *ibdev, u32 port_num,struct ib_port_attr attr = {};int err; -if (port_num !=1 ) { +if (port_num !=RXE_PORT ) { err = -EINVAL ;rxe_dbg_dev (rxe,"bad port_num = %d\n" , port_num); goto err_out;Signature · 1 line
2.43.0
Re: [PATCH v3 2/3] RDMA/rxe: add SENT/RCVD bytes
In reply to zhenwei piQuoted 42 lines · zhenwei pi
There is a lack of sent/received counter in bytes.
Signed-off-by zhenwei pi <[email protected]>--- drivers/infiniband/sw/rxe/rxe_hw_counters.c | 2 ++ drivers/infiniband/sw/rxe/rxe_hw_counters.h | 2 ++ drivers/infiniband/sw/rxe/rxe_net.c | 1 + drivers/infiniband/sw/rxe/rxe_recv.c | 1 + drivers/infiniband/sw/rxe/rxe_verbs.h | 6 ++++++ 5 files changed, 12 insertions(+)drivers/infiniband/sw/rxe/rxe_hw_counters.c +2
-0@@ -22,6 +22,8 @@ static const struct rdma_stat_desc rxe_counter_descs[] = { [RXE_CNT_LINK_DOWNED].name = "link_downed", [RXE_CNT_RDMA_SEND].name = "rdma_sends", [RXE_CNT_RDMA_RECV].name = "rdma_recvs", + [RXE_CNT_SENT_BYTES].name = "sent_bytes", + [RXE_CNT_RCVD_BYTES].name = "rcvd_bytes", }; int rxe_ib_get_hw_stats(struct ib_device *ibdev,drivers/infiniband/sw/rxe/rxe_hw_counters.h +2
-0@@ -26,6 +26,8 @@ enum rxe_counters {RXE_CNT_LINK_DOWNED ,RXE_CNT_RDMA_SEND ,RXE_CNT_RDMA_RECV , +RXE_CNT_SENT_BYTES , +RXE_CNT_RCVD_BYTES , RXE_NUM_OF_COUNTERS };drivers/infiniband/sw/rxe/rxe_net.c +0
-0
drivers/infiniband/sw/rxe/rxe_net.c int rxe_xmit_packet(struct rxe_qp *qp, struct rxe_pkt_info *pkt, · line 522
Quoted 4 lines
drivers/infiniband/sw/rxe/rxe_net.c +1
-0@@ -519,6 +519,7 @@ int rxe_xmit_packet(struct rxe_qp *qp, struct rxe_pkt_info *pkt, }rxe_counter_inc (rxe,RXE_CNT_SENT_PKTS ); +rxe_counter_add (rxe,RXE_CNT_SENT_BYTES , skb->len);
Are you certain that the skb is still valid at this point?
Thanks
Quoted 36 lines
drivers/infiniband/sw/rxe/rxe_net.c +0
-0goto done; drop:drivers/infiniband/sw/rxe/rxe_recv.c +1
-0@@ -342,6 +342,7 @@ void rxe_rcv(struct sk_buff *skb) goto drop;rxe_counter_inc (rxe,RXE_CNT_RCVD_PKTS ); +rxe_counter_add (rxe,RXE_CNT_RCVD_BYTES , skb->len);if (unlikely (bth_qpn (pkt) ==IB_MULTICAST_QPN ))rxe_rcv_mcast_pkt (rxe, skb);drivers/infiniband/sw/rxe/rxe_verbs.h +6
-0@@ -460,6 +460,12 @@ static inline void rxe_counter_inc(struct rxe_dev *rxe, enum rxe_counters index)atomic64_inc (&rxe->stats_counters[index]); } +static inline void rxe_counter_add (struct rxe_dev *rxe,enum rxe_counters index, +s64 val) +{ +atomic64_add (val, &rxe->stats_counters[index]); +} +static inline struct rxe_dev *to_rdev (struct ib_device *dev) { return dev ? container_of(dev, struct rxe_dev, ib_dev) : NULL;Signature · 1 line
2.43.0
Re: [PATCH v3 3/3] RDMA/rxe: support perf mgmt GET method
In reply to zhenwei piQuoted 174 lines · zhenwei pi
In RXE, hardware counters are already supported, but not in a standardized manner. For instance, user-space monitoring tools like atop only read from the *counters* directory. Therefore, it is necessary to add perf management support to RXE.
Also use
rxe_counter_getinstead of rawatomic64_readin hw-counters.Signed-off-by zhenwei pi <[email protected]>--- drivers/infiniband/sw/rxe/Makefile | 1 + drivers/infiniband/sw/rxe/rxe_hw_counters.c | 2 +- drivers/infiniband/sw/rxe/rxe_loc.h | 6 ++ drivers/infiniband/sw/rxe/rxe_mad.c | 93 +++++++++++++++++++++ drivers/infiniband/sw/rxe/rxe_verbs.c | 1 + drivers/infiniband/sw/rxe/rxe_verbs.h | 5 ++ 6 files changed, 107 insertions(+), 1 deletion(-) create mode 100644 drivers/infiniband/sw/rxe/rxe_mad.cdrivers/infiniband/sw/rxe/Makefile +1
-0@@ -22,6 +22,7 @@ rdma_rxe-y := \ rxe_mcast.o \ rxe_task.o \ rxe_net.o \ + rxe_mad.o \ rxe_hw_counters.o rdma_rxe-$(CONFIG_INFINIBAND_ON_DEMAND_PAGING) += rxe_odp.odrivers/infiniband/sw/rxe/rxe_hw_counters.c +1
-1@@ -37,7 +37,7 @@ int rxe_ib_get_hw_stats(struct ib_device *ibdev,return -EINVAL ;for (cnt =0 ; cnt <ARRAY_SIZE (rxe_counter_descs); cnt++) - stats->value[cnt] =atomic64_read (&dev->stats_counters[cnt]); + stats->value[cnt] =rxe_counter_get (dev, cnt);return ARRAY_SIZE (rxe_counter_descs); }drivers/infiniband/sw/rxe/rxe_loc.h +6
-0@@ -245,4 +245,10 @@ static inline int rxe_ib_advise_mr(struct ib_pd *pd,#endif /* CONFIG_INFINIBAND_ON_DEMAND_PAGING */ +/* rxe-mad.c */ +int rxe_process_mad (struct ib_device *ibdev,int mad_flags,u32 port_num, +const struct ib_wc *in_wc,const struct ib_grh *in_grh, +const struct ib_mad *in,struct ib_mad *out, +size_t *out_mad_size,u16 *out_mad_pkey_index); +#endif /* RXE_LOC_H */ drivers/infiniband/sw/rxe/rxe_mad.c new file +93
-0@@ -0,0 +1,93 @@ +// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB +/* +* Copyright (c) 2026 zhenwei pi <[email protected]> +*/ + +#include <rdma/ib_pma.h> +#include "rxe.h" +#include "rxe_hw_counters.h" + +static int rxe_get_pma_info (struct ib_mad *out) +{ +struct ib_class_port_info cpi = {}; + + cpi.capability_mask =IB_PMA_CLASS_CAP_EXT_WIDTH ; +memcpy ((out->data +40 ), &cpi,sizeof (cpi)); + +return IB_MAD_RESULT_SUCCESS |IB_MAD_RESULT_REPLY ; +} + +static int rxe_get_pma_counters (struct rxe_dev *rxe,struct ib_mad *out) +{ +struct ib_pma_portcounters *pma_cnt = (struct ib_pma_portcounters *)(out->data +40 ); +s64 val; + +/* IBA release 1.8, 16.1.3.5: During operation, instead of overflowing, they shall stop +* at all ones. +*/ + val =rxe_counter_get (rxe,RXE_CNT_LINK_DOWNED ); +if (val >U8_MAX ) + pma_cnt->link_downed_counter =U8_MAX ; +else + pma_cnt->link_downed_counter = (u8 )val; + +return IB_MAD_RESULT_SUCCESS |IB_MAD_RESULT_REPLY ; +} + +static int rxe_get_pma_counters_ext (struct rxe_dev *rxe,struct ib_mad *out) +{ +struct ib_pma_portcounters_ext *pma_cnt_ext = + (struct ib_pma_portcounters_ext *)(out->data +40 ); + + pma_cnt_ext->port_xmit_data =cpu_to_be64 (rxe_counter_get (rxe,RXE_CNT_SENT_BYTES ) >>2 ); + pma_cnt_ext->port_rcv_data =cpu_to_be64 (rxe_counter_get (rxe,RXE_CNT_RCVD_BYTES ) >>2 ); + pma_cnt_ext->port_xmit_packets =cpu_to_be64 (rxe_counter_get (rxe,RXE_CNT_SENT_PKTS )); + pma_cnt_ext->port_rcv_packets =cpu_to_be64 (rxe_counter_get (rxe,RXE_CNT_RCVD_PKTS )); + +return IB_MAD_RESULT_SUCCESS |IB_MAD_RESULT_REPLY ; +} + +static int rxe_get_perf_mgmt (struct rxe_dev *rxe,const struct ib_mad *in,struct ib_mad *out) +{ +switch (in->mad_hdr.attr_id) { +case IB_PMA_CLASS_PORT_INFO : +return rxe_get_pma_info (out); + +case IB_PMA_PORT_COUNTERS : +return rxe_get_pma_counters (rxe, out); + +case IB_PMA_PORT_COUNTERS_EXT : +return rxe_get_pma_counters_ext (rxe, out); + +default : + out->mad_hdr.status =cpu_to_be16 (IB_MGMT_MAD_STATUS_UNSUPPORTED_METHOD_ATTRIB ); +return IB_MAD_RESULT_SUCCESS |IB_MAD_RESULT_REPLY ; + } +} + +int rxe_process_mad (struct ib_device *ibdev,int mad_flags,u32 port_num, +const struct ib_wc *in_wc,const struct ib_grh *in_grh, +const struct ib_mad *in,struct ib_mad *out, +size_t *out_mad_size,u16 *out_mad_pkey_index) +{ +struct rxe_dev *rxe =to_rdev (ibdev); +u8 mgmt_class = in->mad_hdr.mgmt_class; +u8 method = in->mad_hdr.method; + +if (port_num !=RXE_PORT ) +return IB_MAD_RESULT_FAILURE ; + +memset (out,0 ,sizeof (*out)); +switch (mgmt_class) { +case IB_MGMT_CLASS_PERF_MGMT : +if (method ==IB_MGMT_METHOD_GET ) +return rxe_get_perf_mgmt (rxe, in, out); +break ; + +default : +break ; + } + + out->mad_hdr.status =cpu_to_be16 (IB_MGMT_MAD_STATUS_UNSUPPORTED_METHOD ); +return IB_MAD_RESULT_SUCCESS |IB_MAD_RESULT_REPLY ; +}drivers/infiniband/sw/rxe/rxe_verbs.c +1
-0@@ -1509,6 +1509,7 @@ static const struct ib_device_ops rxe_dev_ops = { .post_recv = rxe_post_recv, .post_send = rxe_post_send, .post_srq_recv = rxe_post_srq_recv, + .process_mad = rxe_process_mad, .query_ah = rxe_query_ah, .query_device = rxe_query_device, .query_pkey = rxe_query_pkey,drivers/infiniband/sw/rxe/rxe_verbs.h +0
-0
drivers/infiniband/sw/rxe/rxe_verbs.h static inline void rxe_counter_add(struct rxe_dev *rxe, enum rxe_counters index, · line 472
Quoted 7 lines
drivers/infiniband/sw/rxe/rxe_verbs.h +4
-0@@ -466,6 +466,11 @@ static inline void rxe_counter_add(struct rxe_dev *rxe, enum rxe_counters index,atomic64_add (val, &rxe->stats_counters[index]); } +static inline s64 rxe_counter_get (struct rxe_dev *rxe,enum rxe_counters index) +{ +return atomic64_read (&rxe->stats_counters[index]); +}
Same comment as usual: please avoid obscuring basic kernel primitives by introducing trivial one‑line wrapper functions.
Thanks
Quoted 8 lines
drivers/infiniband/sw/rxe/rxe_verbs.h +1
-0+static inline struct rxe_dev *to_rdev (struct ib_device *dev) { return dev ? container_of(dev, struct rxe_dev, ib_dev) : NULL;Signature · 1 line
2.43.0