Support PERF MGMT for RXE
Subsystems (3)
v6 cover Support PERF MGMT for RXE
v6:
- return IB_MAD_RESULT_SUCCESS instead of 'IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_REPLY' on unsupported method
- add a script to verify the sent/rcvd bytes which is wrotten by Yanjun
v5:
- remove patch "RDMA/core: Fix memory free for GID table", it was applied by Jason separately.
- suggested by Yanjun, use 'skb_network_offset' to calculate the length of received packets.
v4:
- drop rxe_ib_device_get_netdev and RXE_PORT, use 1 instead
- avoid UAF to get skb length
- remove one-line wrapper rxe_counter_get, use atomic64_read instead
- fix memory free for GID table, this is a new patch in this series.
v3:
- merge 'RDMA/rxe: use rxe_counter_get' into previous commit
- zero *out* MAD memory
- return success with error status rather than failure to avoid uplayer hang
v2:
- Fix overflow for PMA counter *link_downed_counter*
- Use *rxe_counter_get* instead of *atomic64_read* for hw-counters
v1:
Support PERF MGMT for RXE, add sent/received bytes for RXE counters, also improve coding style.
Zhu Yanjun (1):
selftest/rxe: Add selftests for perf
zhenwei pi (3):
RDMA/rxe: remove rxe_ib_device_get_netdev() and RXE_PORT RDMA/rxe: add SENT/RCVD bytes
RDMA/rxe: support perf mgmt GET method
drivers/infiniband/sw/rxe/Makefile | 1 + drivers/infiniband/sw/rxe/rxe_hw_counters.c | 2 + drivers/infiniband/sw/rxe/rxe_hw_counters.h | 2 + drivers/infiniband/sw/rxe/rxe_loc.h | 6 ++ drivers/infiniband/sw/rxe/rxe_mad.c | 101 ++++++++++++++++++ drivers/infiniband/sw/rxe/rxe_mcast.c | 4 +- drivers/infiniband/sw/rxe/rxe_net.c | 9 +- drivers/infiniband/sw/rxe/rxe_recv.c | 2 + drivers/infiniband/sw/rxe/rxe_verbs.c | 5 +- drivers/infiniband/sw/rxe/rxe_verbs.h | 10 +- tools/testing/selftests/rdma/Makefile | 3 +- .../selftests/rdma/rxe_sent_rcvd_bytes.sh | 75 +++++++++++++ 12 files changed, 206 insertions(+), 14 deletions(-) create mode 100644 drivers/infiniband/sw/rxe/rxe_mad.c create mode 100755 tools/testing/selftests/rdma/rxe_sent_rcvd_bytes.sh
Signature · 1 line
2.43.0
PATCH v6 1/4 RDMA/rxe: remove rxe_ib_device_get_netdev() and RXE_PORT
Suggested by Leon, remove the rxe_ib_device_get_netdev() wrapper and the RXE_PORT definition. These additions do not improve readability, and RXE has always had only a single port.
--- drivers/infiniband/sw/rxe/rxe_mcast.c | 4 ++-- drivers/infiniband/sw/rxe/rxe_net.c | 7 +++---- drivers/infiniband/sw/rxe/rxe_verbs.c | 4 ++-- drivers/infiniband/sw/rxe/rxe_verbs.h | 6 ------ 4 files changed, 7 insertions(+), 14 deletions(-)
drivers/infiniband/sw/rxe/rxe_mcast.c +2 -2
@@ -34,7 +34,7 @@ static int rxe_mcast_add(struct rxe_dev *rxe, union ib_gid *mgid)struct net_device *ndev;int ret; - ndev =rxe_ib_device_get_netdev (&rxe->ib_dev); + ndev =ib_device_get_netdev (&rxe->ib_dev,1 );if (!ndev)return -ENODEV ;
@@ -59,7 +59,7 @@ static int rxe_mcast_del(struct rxe_dev *rxe, union ib_gid *mgid)struct net_device *ndev;int ret; - ndev =rxe_ib_device_get_netdev (&rxe->ib_dev); + ndev =ib_device_get_netdev (&rxe->ib_dev,1 );if (!ndev)return -ENODEV ;
drivers/infiniband/sw/rxe/rxe_net.c +3 -4
@@ -602,7 +602,7 @@ const char *rxe_parent_name(struct rxe_dev *rxe, unsigned int port_num)struct net_device *ndev;char *ndev_name; - ndev =rxe_ib_device_get_netdev (&rxe->ib_dev); + ndev =ib_device_get_netdev (&rxe->ib_dev,1 );if (!ndev)return NULL ; ndev_name = ndev->name;
@@ -646,12 +646,11 @@ static void rxe_sock_put(struct sock *sk,void rxe_net_del (struct ib_device *dev) { - struct rxe_dev *rxe = container_of(dev, struct rxe_dev, ib_dev);struct net_device *ndev;struct sock *sk;struct net *net; - ndev =rxe_ib_device_get_netdev (&rxe->ib_dev); + ndev =ib_device_get_netdev (dev,1 );if (!ndev)return ;
@@ -699,7 +698,7 @@ void rxe_set_port_state(struct rxe_dev *rxe) {struct net_device *ndev; - ndev =rxe_ib_device_get_netdev (&rxe->ib_dev); + ndev =ib_device_get_netdev (&rxe->ib_dev,1 );if (!ndev)return ;
drivers/infiniband/sw/rxe/rxe_verbs.c +2 -2
@@ -50,7 +50,7 @@ static int rxe_query_port(struct ib_device *ibdev, goto err_out; } - ndev =rxe_ib_device_get_netdev (ibdev); + ndev =ib_device_get_netdev (ibdev,1 );if (!ndev) { err = -ENODEV ; goto err_out;
@@ -1441,7 +1441,7 @@ static int rxe_enable_driver(struct ib_device *ib_dev) struct rxe_dev *rxe = container_of(ib_dev, struct rxe_dev, ib_dev);struct net_device *ndev; - ndev =rxe_ib_device_get_netdev (ib_dev); + ndev =ib_device_get_netdev (ib_dev,1 );if (!ndev)return -ENODEV ;
drivers/infiniband/sw/rxe/rxe_verbs.h +0 -6
@@ -415,7 +415,6 @@ struct rxe_port {u32 qp_gsi_index; }; -#define RXE_PORT 1struct rxe_dev {struct ib_device ib_dev;struct ib_device_attr attr;
@@ -451,11 +450,6 @@ struct rxe_dev {struct rxe_port port; }; -static inline struct net_device *rxe_ib_device_get_netdev (struct ib_device *dev) -{ -return ib_device_get_netdev (dev,RXE_PORT ); -} -static inline void rxe_counter_inc (struct rxe_dev *rxe,enum rxe_counters index) {atomic64_inc (&rxe->stats_counters[index]);
Signature · 1 line
2.43.0
PATCH v6 2/4 RDMA/rxe: add SENT/RCVD bytes
In reply to zhenwei piThere is a lack of sent/received counter in bytes.
--- drivers/infiniband/sw/rxe/rxe_hw_counters.c | 2 ++ drivers/infiniband/sw/rxe/rxe_hw_counters.h | 2 ++ drivers/infiniband/sw/rxe/rxe_net.c | 2 ++ drivers/infiniband/sw/rxe/rxe_recv.c | 2 ++ drivers/infiniband/sw/rxe/rxe_verbs.h | 6 ++++++ 5 files changed, 14 insertions(+)
drivers/infiniband/sw/rxe/rxe_hw_counters.c +2 -0
@@ -22,6 +22,8 @@ static const struct rdma_stat_desc rxe_counter_descs[] = { [RXE_CNT_LINK_DOWNED].name = "link_downed", [RXE_CNT_RDMA_SEND].name = "rdma_sends", [RXE_CNT_RDMA_RECV].name = "rdma_recvs", + [RXE_CNT_SENT_BYTES].name = "sent_bytes", + [RXE_CNT_RCVD_BYTES].name = "rcvd_bytes", }; int rxe_ib_get_hw_stats(struct ib_device *ibdev,
drivers/infiniband/sw/rxe/rxe_hw_counters.h +2 -0
@@ -26,6 +26,8 @@ enum rxe_counters {RXE_CNT_LINK_DOWNED ,RXE_CNT_RDMA_SEND ,RXE_CNT_RDMA_RECV , +RXE_CNT_SENT_BYTES , +RXE_CNT_RCVD_BYTES , RXE_NUM_OF_COUNTERS };
drivers/infiniband/sw/rxe/rxe_net.c +2 -0
@@ -503,6 +503,7 @@ int rxe_xmit_packet(struct rxe_qp *qp, struct rxe_pkt_info *pkt,int err;int is_request = pkt->mask &RXE_REQ_MASK ;struct rxe_dev *rxe =to_rdev (qp->ibqp.device); +unsigned int skblen = skb->len;unsigned long flags;spin_lock_irqsave (&qp->state_lock, flags);
@@ -526,6 +527,7 @@ int rxe_xmit_packet(struct rxe_qp *qp, struct rxe_pkt_info *pkt, }rxe_counter_inc (rxe,RXE_CNT_SENT_PKTS ); +rxe_counter_add (rxe,RXE_CNT_SENT_BYTES , skblen); goto done; drop:
drivers/infiniband/sw/rxe/rxe_recv.c +2 -0
@@ -318,6 +318,7 @@ void rxe_rcv(struct sk_buff *skb)int err;struct rxe_pkt_info *pkt =SKB_TO_PKT (skb);struct rxe_dev *rxe = pkt->rxe; +unsigned int skblen = skb->len -skb_network_offset (skb);if (unlikely (skb->len <RXE_BTH_BYTES )) goto drop;
@@ -341,6 +342,7 @@ void rxe_rcv(struct sk_buff *skb)if (unlikely (err)) goto drop; +rxe_counter_add (rxe,RXE_CNT_RCVD_BYTES , skblen);rxe_counter_inc (rxe,RXE_CNT_RCVD_PKTS );if (unlikely (bth_qpn (pkt) ==IB_MULTICAST_QPN ))
drivers/infiniband/sw/rxe/rxe_verbs.h +6 -0
@@ -455,6 +455,12 @@ static inline void rxe_counter_inc(struct rxe_dev *rxe, enum rxe_counters index)atomic64_inc (&rxe->stats_counters[index]); } +static inline void rxe_counter_add (struct rxe_dev *rxe,enum rxe_counters index, +s64 val) +{ +atomic64_add (val, &rxe->stats_counters[index]); +} +static inline struct rxe_dev *to_rdev (struct ib_device *dev) { return dev ? container_of(dev, struct rxe_dev, ib_dev) : NULL;
Signature · 1 line
2.43.0
PATCH v6 3/4 RDMA/rxe: support perf mgmt GET method
In reply to zhenwei piIn RXE, hardware counters are already supported, but not in a standardized manner. For instance, user-space monitoring tools like atop only read from the *counters* directory. Therefore, it is necessary to add perf management support to RXE.
Also use rxe_counter_get instead of raw atomic64_read in hw-counters.
--- drivers/infiniband/sw/rxe/Makefile | 1 + drivers/infiniband/sw/rxe/rxe_loc.h | 6 ++ drivers/infiniband/sw/rxe/rxe_mad.c | 101 ++++++++++++++++++++++++++ drivers/infiniband/sw/rxe/rxe_verbs.c | 1 + 4 files changed, 109 insertions(+) create mode 100644 drivers/infiniband/sw/rxe/rxe_mad.c
drivers/infiniband/sw/rxe/Makefile +1 -0
@@ -23,6 +23,7 @@ rdma_rxe-y := \ rxe_task.o \ rxe_net.o \ rxe_hw_counters.o \ + rxe_mad.o \ rxe_ns.o rdma_rxe-$(CONFIG_INFINIBAND_ON_DEMAND_PAGING) += rxe_odp.o
drivers/infiniband/sw/rxe/rxe_loc.h +6 -0
@@ -242,4 +242,10 @@ static inline int rxe_ib_advise_mr(struct ib_pd *pd,#endif /* CONFIG_INFINIBAND_ON_DEMAND_PAGING */ +/* rxe-mad.c */ +int rxe_process_mad (struct ib_device *ibdev,int mad_flags,u32 port_num, +const struct ib_wc *in_wc,const struct ib_grh *in_grh, +const struct ib_mad *in,struct ib_mad *out, +size_t *out_mad_size,u16 *out_mad_pkey_index); +#endif /* RXE_LOC_H */
drivers/infiniband/sw/rxe/rxe_mad.c new file +101 -0
101 lines not on this page · raw message
drivers/infiniband/sw/rxe/rxe_verbs.c +1 -0
@@ -1496,6 +1496,7 @@ static const struct ib_device_ops rxe_dev_ops = { .post_recv = rxe_post_recv, .post_send = rxe_post_send, .post_srq_recv = rxe_post_srq_recv, + .process_mad = rxe_process_mad, .query_ah = rxe_query_ah, .query_device = rxe_query_device, .query_pkey = rxe_query_pkey,
Signature · 1 line
2.43.0
PATCH v6 4/4 selftest/rxe: Add selftests for perf
In reply to zhenwei piFrom: Zhu Yanjun <[email protected]>
Create a virtual TUN net device with RXE support, then run rping server and client to invoke networking packets, finally compare both *sent_bytes* and *rcvd_bytes* of such device.
--- tools/testing/selftests/rdma/Makefile | 3 +- .../selftests/rdma/rxe_sent_rcvd_bytes.sh | 75 +++++++++++++++++++ 2 files changed, 77 insertions(+), 1 deletion(-) create mode 100755 tools/testing/selftests/rdma/rxe_sent_rcvd_bytes.sh
tools/testing/selftests/rdma/Makefile +2 -1
@@ -2,6 +2,7 @@ TEST_PROGS := rxe_rping_between_netns.sh \ rxe_ipv6.sh \ rxe_socket_with_netns.sh \ - rxe_test_NETDEV_UNREGISTER.sh + rxe_test_NETDEV_UNREGISTER.sh \ + rxe_sent_rcvd_bytes.sh include ../lib.mk
tools/testing/selftests/rdma/rxe_sent_rcvd_bytes.sh new file +75 -0
75 lines not on this page · raw message
Signature · 1 line
2.43.0