Support PERF MGMT for RXE
Subsystems (2)
v5 cover Support PERF MGMT for RXE
v5:
- remove patch "RDMA/core: Fix memory free for GID table", it was applied by Jason separately.
- suggested by Yanjun, use 'skb_network_offset' to calculate the length of received packets.
v4:
- drop rxe_ib_device_get_netdev and RXE_PORT, use 1 instead
- avoid UAF to get skb length
- remove one-line wrapper rxe_counter_get, use atomic64_read instead
- fix memory free for GID table, this is a new patch in this series.
v3:
- merge 'RDMA/rxe: use rxe_counter_get' into previous commit
- zero *out* MAD memory
- return success with error status rather than failure to avoid uplayer hang
v2:
- Fix overflow for PMA counter *link_downed_counter*
- Use *rxe_counter_get* instead of *atomic64_read* for hw-counters
v1:
Support PERF MGMT for RXE, add sent/received bytes for RXE counters, also improve coding style.
zhenwei pi (3):
RDMA/rxe: remove rxe_ib_device_get_netdev() and RXE_PORT RDMA/rxe: add SENT/RCVD bytes
RDMA/rxe: support perf mgmt GET method
drivers/infiniband/sw/rxe/Makefile | 1 + drivers/infiniband/sw/rxe/rxe_hw_counters.c | 2 + drivers/infiniband/sw/rxe/rxe_hw_counters.h | 2 + drivers/infiniband/sw/rxe/rxe_loc.h | 6 ++ drivers/infiniband/sw/rxe/rxe_mad.c | 101 ++++++++++++++++++++ drivers/infiniband/sw/rxe/rxe_mcast.c | 4 +- drivers/infiniband/sw/rxe/rxe_net.c | 9 +- drivers/infiniband/sw/rxe/rxe_recv.c | 2 + drivers/infiniband/sw/rxe/rxe_verbs.c | 5 +- drivers/infiniband/sw/rxe/rxe_verbs.h | 10 +- 10 files changed, 129 insertions(+), 13 deletions(-) create mode 100644 drivers/infiniband/sw/rxe/rxe_mad.c
Signature · 1 line
2.43.0
PATCH v5 1/3 RDMA/rxe: remove rxe_ib_device_get_netdev() and RXE_PORT
Suggested by Leon, remove the rxe_ib_device_get_netdev() wrapper and the RXE_PORT definition. These additions do not improve readability, and RXE has always had only a single port.
--- drivers/infiniband/sw/rxe/rxe_mcast.c | 4 ++-- drivers/infiniband/sw/rxe/rxe_net.c | 7 +++---- drivers/infiniband/sw/rxe/rxe_verbs.c | 4 ++-- drivers/infiniband/sw/rxe/rxe_verbs.h | 6 ------ 4 files changed, 7 insertions(+), 14 deletions(-)
drivers/infiniband/sw/rxe/rxe_mcast.c +2 -2
@@ -34,7 +34,7 @@ static int rxe_mcast_add(struct rxe_dev *rxe, union ib_gid *mgid)struct net_device *ndev;int ret; - ndev =rxe_ib_device_get_netdev (&rxe->ib_dev); + ndev =ib_device_get_netdev (&rxe->ib_dev,1 );if (!ndev)return -ENODEV ;
@@ -59,7 +59,7 @@ static int rxe_mcast_del(struct rxe_dev *rxe, union ib_gid *mgid)struct net_device *ndev;int ret; - ndev =rxe_ib_device_get_netdev (&rxe->ib_dev); + ndev =ib_device_get_netdev (&rxe->ib_dev,1 );if (!ndev)return -ENODEV ;
drivers/infiniband/sw/rxe/rxe_net.c +3 -4
@@ -602,7 +602,7 @@ const char *rxe_parent_name(struct rxe_dev *rxe, unsigned int port_num)struct net_device *ndev;char *ndev_name; - ndev =rxe_ib_device_get_netdev (&rxe->ib_dev); + ndev =ib_device_get_netdev (&rxe->ib_dev,1 );if (!ndev)return NULL ; ndev_name = ndev->name;
@@ -646,12 +646,11 @@ static void rxe_sock_put(struct sock *sk,void rxe_net_del (struct ib_device *dev) { - struct rxe_dev *rxe = container_of(dev, struct rxe_dev, ib_dev);struct net_device *ndev;struct sock *sk;struct net *net; - ndev =rxe_ib_device_get_netdev (&rxe->ib_dev); + ndev =ib_device_get_netdev (dev,1 );if (!ndev)return ;
@@ -699,7 +698,7 @@ void rxe_set_port_state(struct rxe_dev *rxe) {struct net_device *ndev; - ndev =rxe_ib_device_get_netdev (&rxe->ib_dev); + ndev =ib_device_get_netdev (&rxe->ib_dev,1 );if (!ndev)return ;
drivers/infiniband/sw/rxe/rxe_verbs.c +2 -2
@@ -50,7 +50,7 @@ static int rxe_query_port(struct ib_device *ibdev, goto err_out; } - ndev =rxe_ib_device_get_netdev (ibdev); + ndev =ib_device_get_netdev (ibdev,1 );if (!ndev) { err = -ENODEV ; goto err_out;
@@ -1441,7 +1441,7 @@ static int rxe_enable_driver(struct ib_device *ib_dev) struct rxe_dev *rxe = container_of(ib_dev, struct rxe_dev, ib_dev);struct net_device *ndev; - ndev =rxe_ib_device_get_netdev (ib_dev); + ndev =ib_device_get_netdev (ib_dev,1 );if (!ndev)return -ENODEV ;
drivers/infiniband/sw/rxe/rxe_verbs.h +0 -6
@@ -415,7 +415,6 @@ struct rxe_port {u32 qp_gsi_index; }; -#define RXE_PORT 1struct rxe_dev {struct ib_device ib_dev;struct ib_device_attr attr;
@@ -451,11 +450,6 @@ struct rxe_dev {struct rxe_port port; }; -static inline struct net_device *rxe_ib_device_get_netdev (struct ib_device *dev) -{ -return ib_device_get_netdev (dev,RXE_PORT ); -} -static inline void rxe_counter_inc (struct rxe_dev *rxe,enum rxe_counters index) {atomic64_inc (&rxe->stats_counters[index]);
Signature · 1 line
2.43.0
PATCH v5 2/3 RDMA/rxe: add SENT/RCVD bytes
In reply to zhenwei piThere is a lack of sent/received counter in bytes.
--- drivers/infiniband/sw/rxe/rxe_hw_counters.c | 2 ++ drivers/infiniband/sw/rxe/rxe_hw_counters.h | 2 ++ drivers/infiniband/sw/rxe/rxe_net.c | 2 ++ drivers/infiniband/sw/rxe/rxe_recv.c | 2 ++ drivers/infiniband/sw/rxe/rxe_verbs.h | 6 ++++++ 5 files changed, 14 insertions(+)
drivers/infiniband/sw/rxe/rxe_hw_counters.c +2 -0
@@ -22,6 +22,8 @@ static const struct rdma_stat_desc rxe_counter_descs[] = { [RXE_CNT_LINK_DOWNED].name = "link_downed", [RXE_CNT_RDMA_SEND].name = "rdma_sends", [RXE_CNT_RDMA_RECV].name = "rdma_recvs", + [RXE_CNT_SENT_BYTES].name = "sent_bytes", + [RXE_CNT_RCVD_BYTES].name = "rcvd_bytes", }; int rxe_ib_get_hw_stats(struct ib_device *ibdev,
drivers/infiniband/sw/rxe/rxe_hw_counters.h +2 -0
@@ -26,6 +26,8 @@ enum rxe_counters {RXE_CNT_LINK_DOWNED ,RXE_CNT_RDMA_SEND ,RXE_CNT_RDMA_RECV , +RXE_CNT_SENT_BYTES , +RXE_CNT_RCVD_BYTES , RXE_NUM_OF_COUNTERS };
drivers/infiniband/sw/rxe/rxe_net.c +2 -0
@@ -503,6 +503,7 @@ int rxe_xmit_packet(struct rxe_qp *qp, struct rxe_pkt_info *pkt,int err;int is_request = pkt->mask &RXE_REQ_MASK ;struct rxe_dev *rxe =to_rdev (qp->ibqp.device); +unsigned int skblen = skb->len;unsigned long flags;spin_lock_irqsave (&qp->state_lock, flags);
@@ -526,6 +527,7 @@ int rxe_xmit_packet(struct rxe_qp *qp, struct rxe_pkt_info *pkt, }rxe_counter_inc (rxe,RXE_CNT_SENT_PKTS ); +rxe_counter_add (rxe,RXE_CNT_SENT_BYTES , skblen); goto done; drop:
drivers/infiniband/sw/rxe/rxe_recv.c +2 -0
@@ -318,6 +318,7 @@ void rxe_rcv(struct sk_buff *skb)int err;struct rxe_pkt_info *pkt =SKB_TO_PKT (skb);struct rxe_dev *rxe = pkt->rxe; +unsigned int skblen = skb->len -skb_network_offset (skb);if (unlikely (skb->len <RXE_BTH_BYTES )) goto drop;
@@ -341,6 +342,7 @@ void rxe_rcv(struct sk_buff *skb)if (unlikely (err)) goto drop; +rxe_counter_add (rxe,RXE_CNT_RCVD_BYTES , skblen);rxe_counter_inc (rxe,RXE_CNT_RCVD_PKTS );if (unlikely (bth_qpn (pkt) ==IB_MULTICAST_QPN ))
drivers/infiniband/sw/rxe/rxe_verbs.h +6 -0
@@ -455,6 +455,12 @@ static inline void rxe_counter_inc(struct rxe_dev *rxe, enum rxe_counters index)atomic64_inc (&rxe->stats_counters[index]); } +static inline void rxe_counter_add (struct rxe_dev *rxe,enum rxe_counters index, +s64 val) +{ +atomic64_add (val, &rxe->stats_counters[index]); +} +static inline struct rxe_dev *to_rdev (struct ib_device *dev) { return dev ? container_of(dev, struct rxe_dev, ib_dev) : NULL;
Signature · 1 line
2.43.0
Re: [PATCH v5 2/3] RDMA/rxe: add SENT/RCVD bytes
Quoted 70 lines · zhenwei pi
There is a lack of sent/received counter in bytes.
Signed-off-by zhenwei pi <[email protected]>--- drivers/infiniband/sw/rxe/rxe_hw_counters.c | 2 ++ drivers/infiniband/sw/rxe/rxe_hw_counters.h | 2 ++ drivers/infiniband/sw/rxe/rxe_net.c | 2 ++ drivers/infiniband/sw/rxe/rxe_recv.c | 2 ++ drivers/infiniband/sw/rxe/rxe_verbs.h | 6 ++++++ 5 files changed, 14 insertions(+)drivers/infiniband/sw/rxe/rxe_hw_counters.c +2
-0@@ -22,6 +22,8 @@ static const struct rdma_stat_desc rxe_counter_descs[] = { [RXE_CNT_LINK_DOWNED].name = "link_downed", [RXE_CNT_RDMA_SEND].name = "rdma_sends", [RXE_CNT_RDMA_RECV].name = "rdma_recvs", + [RXE_CNT_SENT_BYTES].name = "sent_bytes", + [RXE_CNT_RCVD_BYTES].name = "rcvd_bytes", }; int rxe_ib_get_hw_stats(struct ib_device *ibdev,drivers/infiniband/sw/rxe/rxe_hw_counters.h +2
-0@@ -26,6 +26,8 @@ enum rxe_counters {RXE_CNT_LINK_DOWNED ,RXE_CNT_RDMA_SEND ,RXE_CNT_RDMA_RECV , +RXE_CNT_SENT_BYTES , +RXE_CNT_RCVD_BYTES , RXE_NUM_OF_COUNTERS };drivers/infiniband/sw/rxe/rxe_net.c +2
-0@@ -503,6 +503,7 @@ int rxe_xmit_packet(struct rxe_qp *qp, struct rxe_pkt_info *pkt,int err;int is_request = pkt->mask &RXE_REQ_MASK ;struct rxe_dev *rxe =to_rdev (qp->ibqp.device); +unsigned int skblen = skb->len;unsigned long flags;spin_lock_irqsave (&qp->state_lock, flags);@@ -526,6 +527,7 @@ int rxe_xmit_packet(struct rxe_qp *qp, struct rxe_pkt_info *pkt, }rxe_counter_inc (rxe,RXE_CNT_SENT_PKTS ); +rxe_counter_add (rxe,RXE_CNT_SENT_BYTES , skblen); goto done; drop:drivers/infiniband/sw/rxe/rxe_recv.c +1
-0@@ -318,6 +318,7 @@ void rxe_rcv(struct sk_buff *skb)int err;struct rxe_pkt_info *pkt =SKB_TO_PKT (skb);struct rxe_dev *rxe = pkt->rxe; +unsigned int skblen = skb->len -skb_network_offset (skb);if (unlikely (skb->len <RXE_BTH_BYTES )) goto drop;
drivers/infiniband/sw/rxe/rxe_recv.c void rxe_rcv(struct sk_buff *skb) · line 348
Quoted 7 lines
drivers/infiniband/sw/rxe/rxe_recv.c +1
-0@@ -341,6 +342,7 @@ void rxe_rcv(struct sk_buff *skb)if (unlikely (err)) goto drop; +rxe_counter_add (rxe,RXE_CNT_RCVD_BYTES , skblen);rxe_counter_inc (rxe,RXE_CNT_RCVD_PKTS );if (unlikely (bth_qpn (pkt) ==IB_MULTICAST_QPN ))
int rxe_xmit_packet(struct rxe_qp *qp, struct rxe_pkt_info *pkt,
struct sk_buff *skb)
{
int err;
int is_request = pkt->mask & RXE_REQ_MASK;
struct rxe_dev *rxe = to_rdev(qp->ibqp.device);
unsigned long flags;skb->len is printed here, that is len1
...
if (pkt->mask & RXE_LOOPBACK_MASK)
err = rxe_loopback(skb, pkt);
else
err = rxe_send(skb, pkt);...
}
In the following function
static int rxe_loopback(struct sk_buff *skb, struct rxe_pkt_info *pkt) {
...
if (skb->protocol == htons(ETH_P_IP))
skb_pull(skb, sizeof(struct iphdr));
else
skb_pull(skb, sizeof(struct ipv6hdr));...
/* remove udp header */
skb_pull(skb, sizeof(struct udphdr));print skb->len here, that is len2
rxe_rcv(skb);
...
}
Does len1 equal to len2?
If not, the transmitted length appears to differ from the received length when using loopback.
I am not sure whether this is expected behavior.
The same observation also applies to the non-loopback case.
Zhu Yanjun
Quoted 17 lines
drivers/infiniband/sw/rxe/rxe_verbs.h +6
-0@@ -455,6 +455,12 @@ static inline void rxe_counter_inc(struct rxe_dev *rxe, enum rxe_counters index)atomic64_inc (&rxe->stats_counters[index]); } +static inline void rxe_counter_add (struct rxe_dev *rxe,enum rxe_counters index, +s64 val) +{ +atomic64_add (val, &rxe->stats_counters[index]); +} +static inline struct rxe_dev *to_rdev (struct ib_device *dev) { return dev ? container_of(dev, struct rxe_dev, ib_dev) : NULL;
Re: [PATCH v5 2/3] RDMA/rxe: add SENT/RCVD bytes
Quoted 105 lines · yanjun.zhu
Quoted 67 lines · zhenwei pi
There is a lack of sent/received counter in bytes.
Signed-off-by zhenwei pi <[email protected]>---
drivers/infiniband/sw/rxe/rxe_hw_counters.c| 2 ++drivers/infiniband/sw/rxe/rxe_hw_counters.h| 2 ++ drivers/infiniband/sw/rxe/rxe_net.c | 2 ++ drivers/infiniband/sw/rxe/rxe_recv.c | 2 ++ drivers/infiniband/sw/rxe/rxe_verbs.h | 6 ++++++5 files changed, 14 insertions(+)drivers/infiniband/sw/rxe/rxe_hw_counters.c +2
-0@@ -22,6 +22,8 @@ static const struct rdma_stat_desc rxe_counter_descs[] = { [RXE_CNT_LINK_DOWNED].name = "link_downed", [RXE_CNT_RDMA_SEND].name = "rdma_sends", [RXE_CNT_RDMA_RECV].name = "rdma_recvs", + [RXE_CNT_SENT_BYTES].name = "sent_bytes", + [RXE_CNT_RCVD_BYTES].name = "rcvd_bytes", }; int rxe_ib_get_hw_stats(struct ib_device *ibdev,drivers/infiniband/sw/rxe/rxe_hw_counters.h +2
-0@@ -26,6 +26,8 @@ enum rxe_counters { RXE_CNT_LINK_DOWNED, RXE_CNT_RDMA_SEND, RXE_CNT_RDMA_RECV, + RXE_CNT_SENT_BYTES, + RXE_CNT_RCVD_BYTES,
RXE_NUM_OF_COUNTERS
};
diff --gita/drivers/infiniband/sw/rxe/rxe_net.cb/drivers/infiniband/sw/rxe/rxe_net.cindex6621d01ac32d..86660031ffa2100644drivers/infiniband/sw/rxe/rxe_net.c +2
-0@@ -503,6 +503,7 @@ int rxe_xmit_packet(struct rxe_qp *qp, structrxe_pkt_info *pkt, int err; int is_request = pkt->mask & RXE_REQ_MASK; struct rxe_dev *rxe = to_rdev(qp->ibqp.device); + unsigned int skblen = skb->len; unsigned long flags; spin_lock_irqsave(&qp->state_lock, flags);@@ -526,6 +527,7 @@ int rxe_xmit_packet(struct rxe_qp *qp, structrxe_pkt_info *pkt, } rxe_counter_inc(rxe, RXE_CNT_SENT_PKTS); + rxe_counter_add(rxe, RXE_CNT_SENT_BYTES, skblen); goto done;drop:
drivers/infiniband/sw/rxe/rxe_recv.c +1
-0@@ -318,6 +318,7 @@ void rxe_rcv(struct sk_buff *skb) int err; struct rxe_pkt_info *pkt = SKB_TO_PKT(skb); struct rxe_dev *rxe = pkt->rxe; + unsigned int skblen = skb->len - skb_network_offset(skb); if (unlikely(skb->len < RXE_BTH_BYTES)) goto drop;drivers/infiniband/sw/rxe/rxe_recv.c void rxe_rcv(struct sk_buff *skb) · line 346
Quoted 5 lines
drivers/infiniband/sw/rxe/rxe_recv.c +0
-0@@ -341,6 +342,7 @@ void rxe_rcv(struct sk_buff *skb) if (unlikely(err)) goto drop; + rxe_counter_add(rxe, RXE_CNT_RCVD_BYTES, skblen); rxe_counter_inc(rxe, RXE_CNT_RCVD_PKTS); if (unlikely(bth_qpn(pkt) == IB_MULTICAST_QPN))int
rxe_xmit_packet(struct rxe_qp *qp, struct rxe_pkt_info *pkt, struct sk_buff *skb)
{
int err;
intis_request=pkt->mask&RXE_REQ_MASK; structrxe_dev*rxe =to_rdev(qp->ibqp.device); unsigned long flags;
skb->lenis printed here, that is len1
...
if (pkt->mask&RXE_LOOPBACK_MASK)
err =rxe_loopback(skb, pkt);
else
err =rxe_send(skb, pkt);
...
}In the following function
static int
rxe_loopback(struct sk_buff *skb, struct rxe_pkt_info *pkt){
...
if (skb->protocol==htons(ETH_P_IP))
skb_pull(skb, sizeof(struct iphdr));
else
skb_pull(skb, sizeof(struct ipv6hdr));...
/* remove udp header */
skb_pull(skb, sizeof(struct udphdr));
Quoted 8 lines
skb->lenhere, that is len2
rxe_rcv(skb);...
}Does len1 equal to len2?
I have made tests. The difference between len1 and len2 is 28.
It should be the total of ipv4 header + udp header because I use ipv4 address to make tests.
I am not sure if the bytes of recv data should equal to the bytes of xmit data.
Zhu Yanjun
Quoted 26 lines
If not, the transmitted length appears to differ from the received length when using loopback.
I am not sure whether this is expected behavior.
The same observation also applies to the non-loopback case.
Zhu Yanjun
Quoted 17 lines
drivers/infiniband/sw/rxe/rxe_verbs.h +5
-0@@ -455,6 +455,12 @@ static inline void rxe_counter_inc(struct rxe_dev *rxe, enum rxe_counters index) atomic64_inc(&rxe->stats_counters[index]); } +static inline void rxe_counter_add(struct rxe_dev *rxe, enum rxe_counters index, + s64 val) +{ + atomic64_add(val, &rxe->stats_counters[index]); +} + static inline struct rxe_dev *to_rdev(struct ib_device *dev) {return dev ?
container_of(dev, struct rxe_dev, ib_dev): NULL;
Signature · 2 lines
Best Regards, Yanjun.Zhu
Re: [PATCH v5 2/3] RDMA/rxe: add SENT/RCVD bytes
Quoted 116 lines · Zhu Yanjun
Quoted 105 lines · yanjun.zhu
Quoted 67 lines · zhenwei pi
There is a lack of sent/received counter in bytes.
Signed-off-by zhenwei pi <[email protected]>---
drivers/infiniband/sw/rxe/rxe_hw_counters.c| 2 ++drivers/infiniband/sw/rxe/rxe_hw_counters.h| 2 ++ drivers/infiniband/sw/rxe/rxe_net.c | 2 ++ drivers/infiniband/sw/rxe/rxe_recv.c | 2 ++ drivers/infiniband/sw/rxe/rxe_verbs.h | 6 ++++++5 files changed, 14 insertions(+)drivers/infiniband/sw/rxe/rxe_hw_counters.c +2
-0@@ -22,6 +22,8 @@ static const struct rdma_stat_desc rxe_counter_descs[] = { [RXE_CNT_LINK_DOWNED].name = "link_downed", [RXE_CNT_RDMA_SEND].name = "rdma_sends", [RXE_CNT_RDMA_RECV].name = "rdma_recvs", + [RXE_CNT_SENT_BYTES].name = "sent_bytes", + [RXE_CNT_RCVD_BYTES].name = "rcvd_bytes", }; int rxe_ib_get_hw_stats(struct ib_device *ibdev,drivers/infiniband/sw/rxe/rxe_hw_counters.h +2
-0@@ -26,6 +26,8 @@ enum rxe_counters { RXE_CNT_LINK_DOWNED, RXE_CNT_RDMA_SEND, RXE_CNT_RDMA_RECV, + RXE_CNT_SENT_BYTES, + RXE_CNT_RCVD_BYTES,
RXE_NUM_OF_COUNTERS
};
diff --gita/drivers/infiniband/sw/rxe/rxe_net.cb/drivers/infiniband/sw/rxe/rxe_net.cindex6621d01ac32d..86660031ffa2100644drivers/infiniband/sw/rxe/rxe_net.c +2
-0@@ -503,6 +503,7 @@ int rxe_xmit_packet(struct rxe_qp *qp, structrxe_pkt_info *pkt, int err; int is_request = pkt->mask & RXE_REQ_MASK; struct rxe_dev *rxe = to_rdev(qp->ibqp.device); + unsigned int skblen = skb->len; unsigned long flags; spin_lock_irqsave(&qp->state_lock, flags);@@ -526,6 +527,7 @@ int rxe_xmit_packet(struct rxe_qp *qp, structrxe_pkt_info *pkt, } rxe_counter_inc(rxe, RXE_CNT_SENT_PKTS); + rxe_counter_add(rxe, RXE_CNT_SENT_BYTES, skblen); goto done;drop:
drivers/infiniband/sw/rxe/rxe_recv.c +1
-0@@ -318,6 +318,7 @@ void rxe_rcv(struct sk_buff *skb) int err; struct rxe_pkt_info *pkt = SKB_TO_PKT(skb); struct rxe_dev *rxe = pkt->rxe; + unsigned int skblen = skb->len - skb_network_offset(skb); if (unlikely(skb->len < RXE_BTH_BYTES)) goto drop;drivers/infiniband/sw/rxe/rxe_recv.c void rxe_rcv(struct sk_buff *skb) · line 346
Quoted 5 lines
drivers/infiniband/sw/rxe/rxe_recv.c +0
-0@@ -341,6 +342,7 @@ void rxe_rcv(struct sk_buff *skb) if (unlikely(err)) goto drop; + rxe_counter_add(rxe, RXE_CNT_RCVD_BYTES, skblen); rxe_counter_inc(rxe, RXE_CNT_RCVD_PKTS); if (unlikely(bth_qpn(pkt) == IB_MULTICAST_QPN))int
rxe_xmit_packet(struct rxe_qp *qp, struct rxe_pkt_info *pkt, struct sk_buff *skb)
{
int err;
intis_request=pkt->mask&RXE_REQ_MASK; structrxe_dev*rxe =to_rdev(qp->ibqp.device); unsigned long flags;
skb->lenis printed here, that is len1
...
if (pkt->mask&RXE_LOOPBACK_MASK)
err =rxe_loopback(skb, pkt);
else
err =rxe_send(skb, pkt);
...
}In the following function
static int
rxe_loopback(struct sk_buff *skb, struct rxe_pkt_info *pkt){
...
if (skb->protocol==htons(ETH_P_IP))
skb_pull(skb, sizeof(struct iphdr));
else
skb_pull(skb, sizeof(struct ipv6hdr));...
/* remove udp header */
skb_pull(skb, sizeof(struct udphdr));Quoted 8 lines
skb->lenhere, that is len2
rxe_rcv(skb);...
}Does len1 equal to len2?
Quoted 9 lines
I have made tests. The difference between len1 and len2 is 28.
It should be the total of ipv4 header + udp header because I use ipv4 address to make tests.
I am not sure if the bytes of recv data should equal to the bytes of xmit data.
Zhu Yanjun
Yes, I got the same result of 28 bytes difference during v4 version. So I try to use such logic to recalculate the length:
+ unsigned int skblen = skb->len + sizeof(struct udphdr);
...
+ if (skb->protocol == htons(ETH_P_IP))
+ skblen += sizeof(struct iphdr);
+ else if (skb->protocol == htons(ETH_P_IPV6))
+ skblen += sizeof(struct ipv6hdr);
+ rxe_counter_add(rxe, RXE_CNT_RCVD_BYTES, skblen);
then use 'unsigned int skblen = skb->len - skb_network_offset(skb);' in v5.
Quoted 27 lines
Quoted 26 lines
If not, the transmitted length appears to differ from the received length when using loopback.
I am not sure whether this is expected behavior.
The same observation also applies to the non-loopback case.
Zhu Yanjun
Quoted 17 lines
drivers/infiniband/sw/rxe/rxe_verbs.h +5
-0@@ -455,6 +455,12 @@ static inline void rxe_counter_inc(struct rxe_dev *rxe, enum rxe_counters index) atomic64_inc(&rxe->stats_counters[index]); } +static inline void rxe_counter_add(struct rxe_dev *rxe, enum rxe_counters index, + s64 val) +{ + atomic64_add(val, &rxe->stats_counters[index]); +} + static inline struct rxe_dev *to_rdev(struct ib_device *dev) {return dev ?
container_of(dev, struct rxe_dev, ib_dev): NULL;
Re: [PATCH v5 2/3] RDMA/rxe: add SENT/RCVD bytes
Quoted 139 lines · zhenwei pi
Quoted 116 lines · Zhu Yanjun
Quoted 105 lines · yanjun.zhu
Quoted 67 lines · zhenwei pi
There is a lack of sent/received counter in bytes.
Signed-off-by zhenwei pi <[email protected]>---
drivers/infiniband/sw/rxe/rxe_hw_counters.c| 2 ++drivers/infiniband/sw/rxe/rxe_hw_counters.h| 2 ++ drivers/infiniband/sw/rxe/rxe_net.c | 2 ++ drivers/infiniband/sw/rxe/rxe_recv.c | 2 ++ drivers/infiniband/sw/rxe/rxe_verbs.h | 6 ++++++5 files changed, 14 insertions(+)drivers/infiniband/sw/rxe/rxe_hw_counters.c +2
-0@@ -22,6 +22,8 @@ static const struct rdma_stat_desc rxe_counter_descs[] = { [RXE_CNT_LINK_DOWNED].name = "link_downed", [RXE_CNT_RDMA_SEND].name = "rdma_sends", [RXE_CNT_RDMA_RECV].name = "rdma_recvs", + [RXE_CNT_SENT_BYTES].name = "sent_bytes", + [RXE_CNT_RCVD_BYTES].name = "rcvd_bytes", }; int rxe_ib_get_hw_stats(struct ib_device *ibdev,drivers/infiniband/sw/rxe/rxe_hw_counters.h +2
-0@@ -26,6 +26,8 @@ enum rxe_counters { RXE_CNT_LINK_DOWNED, RXE_CNT_RDMA_SEND, RXE_CNT_RDMA_RECV, + RXE_CNT_SENT_BYTES, + RXE_CNT_RCVD_BYTES,
RXE_NUM_OF_COUNTERS
};
diff --gita/drivers/infiniband/sw/rxe/rxe_net.cb/drivers/infiniband/sw/rxe/rxe_net.cindex6621d01ac32d..86660031ffa2100644drivers/infiniband/sw/rxe/rxe_net.c +2
-0@@ -503,6 +503,7 @@ int rxe_xmit_packet(struct rxe_qp *qp, structrxe_pkt_info *pkt, int err; int is_request = pkt->mask & RXE_REQ_MASK; struct rxe_dev *rxe = to_rdev(qp->ibqp.device); + unsigned int skblen = skb->len; unsigned long flags; spin_lock_irqsave(&qp->state_lock, flags);@@ -526,6 +527,7 @@ int rxe_xmit_packet(struct rxe_qp *qp, structrxe_pkt_info *pkt, } rxe_counter_inc(rxe, RXE_CNT_SENT_PKTS); + rxe_counter_add(rxe, RXE_CNT_SENT_BYTES, skblen); goto done;drop:
drivers/infiniband/sw/rxe/rxe_recv.c +1
-0@@ -318,6 +318,7 @@ void rxe_rcv(struct sk_buff *skb) int err; struct rxe_pkt_info *pkt = SKB_TO_PKT(skb); struct rxe_dev *rxe = pkt->rxe; + unsigned int skblen = skb->len - skb_network_offset(skb); if (unlikely(skb->len < RXE_BTH_BYTES)) goto drop;drivers/infiniband/sw/rxe/rxe_recv.c void rxe_rcv(struct sk_buff *skb) · line 346
Quoted 5 lines
drivers/infiniband/sw/rxe/rxe_recv.c +0
-0@@ -341,6 +342,7 @@ void rxe_rcv(struct sk_buff *skb) if (unlikely(err)) goto drop; + rxe_counter_add(rxe, RXE_CNT_RCVD_BYTES, skblen); rxe_counter_inc(rxe, RXE_CNT_RCVD_PKTS); if (unlikely(bth_qpn(pkt) == IB_MULTICAST_QPN))int
rxe_xmit_packet(struct rxe_qp *qp, struct rxe_pkt_info *pkt, struct sk_buff *skb)
{
int err;
intis_request=pkt->mask&RXE_REQ_MASK; structrxe_dev*rxe =to_rdev(qp->ibqp.device); unsigned long flags;
skb->lenis printed here, that is len1
...
if (pkt->mask&RXE_LOOPBACK_MASK)
err =rxe_loopback(skb, pkt);
else
err =rxe_send(skb, pkt);
...
}In the following function
static int
rxe_loopback(struct sk_buff *skb, struct rxe_pkt_info *pkt){
...
if (skb->protocol==htons(ETH_P_IP))
skb_pull(skb, sizeof(struct iphdr));
else
skb_pull(skb, sizeof(struct ipv6hdr));...
/* remove udp header */
skb_pull(skb, sizeof(struct udphdr));Quoted 8 lines
skb->lenhere, that is len2
rxe_rcv(skb);...
}Does len1 equal to len2?
Quoted 9 lines
I have made tests. The difference between len1 and len2 is 28.
It should be the total of ipv4 header + udp header because I use ipv4 address to make tests.
I am not sure if the bytes of recv data should equal to the bytes of xmit data.
Zhu Yanjun
Yes, I got the same result of 28 bytes difference during v4 version. So I try to use such logic to recalculate the length:
+ unsigned int skblen =
skb->len+ sizeof(struct udphdr);
...
+ if (skb->protocol==htons(ETH_P_IP))
+ skblen += sizeof(struct iphdr);
+ else if (skb->protocol==htons(ETH_P_IPV6)) + skblen += sizeof(struct ipv6hdr);
+rxe_counter_add(rxe, RXE_CNT_RCVD_BYTES, skblen);
Quoted 1 line
then use 'unsigned int skblen =
skb->len-skb_network_offset(skb);' in v5.
Thanks. Not necessary to use this 'unsigned int skblen = skb->len - skb_network_offset(skb);'.
The above udphdr and iphdr/ip6hdr should be OK.
Thanks a lot.
Zhu Yanjun
Quoted 29 lines
Quoted 28 lines
Quoted 27 lines
If not, the transmitted length appears to differ from the received length when using loopback.
I am not sure whether this is expected behavior.
The same observation also applies to the non-loopback case.
Zhu Yanjun
Quoted 17 lines
drivers/infiniband/sw/rxe/rxe_verbs.h +5
-0@@ -455,6 +455,12 @@ static inline void rxe_counter_inc(struct rxe_dev *rxe, enum rxe_counters index) atomic64_inc(&rxe->stats_counters[index]); } +static inline void rxe_counter_add(struct rxe_dev *rxe, enum rxe_counters index, + s64 val) +{ + atomic64_add(val, &rxe->stats_counters[index]); +} + static inline struct rxe_dev *to_rdev(struct ib_device *dev) {return dev ?
container_of(dev, struct rxe_dev, ib_dev): NULL;
PATCH v5 3/3 RDMA/rxe: support perf mgmt GET method
In reply to zhenwei piIn RXE, hardware counters are already supported, but not in a standardized manner. For instance, user-space monitoring tools like atop only read from the *counters* directory. Therefore, it is necessary to add perf management support to RXE.
Also use rxe_counter_get instead of raw atomic64_read in hw-counters.
--- drivers/infiniband/sw/rxe/Makefile | 1 + drivers/infiniband/sw/rxe/rxe_loc.h | 6 ++ drivers/infiniband/sw/rxe/rxe_mad.c | 101 ++++++++++++++++++++++++++ drivers/infiniband/sw/rxe/rxe_verbs.c | 1 + 4 files changed, 109 insertions(+) create mode 100644 drivers/infiniband/sw/rxe/rxe_mad.c
drivers/infiniband/sw/rxe/Makefile +1 -0
@@ -23,6 +23,7 @@ rdma_rxe-y := \ rxe_task.o \ rxe_net.o \ rxe_hw_counters.o \ + rxe_mad.o \ rxe_ns.o rdma_rxe-$(CONFIG_INFINIBAND_ON_DEMAND_PAGING) += rxe_odp.o
drivers/infiniband/sw/rxe/rxe_loc.h +6 -0
@@ -242,4 +242,10 @@ static inline int rxe_ib_advise_mr(struct ib_pd *pd,#endif /* CONFIG_INFINIBAND_ON_DEMAND_PAGING */ +/* rxe-mad.c */ +int rxe_process_mad (struct ib_device *ibdev,int mad_flags,u32 port_num, +const struct ib_wc *in_wc,const struct ib_grh *in_grh, +const struct ib_mad *in,struct ib_mad *out, +size_t *out_mad_size,u16 *out_mad_pkey_index); +#endif /* RXE_LOC_H */
drivers/infiniband/sw/rxe/rxe_mad.c new file +101 -0
101 lines not on this page · raw message
drivers/infiniband/sw/rxe/rxe_verbs.c +1 -0
@@ -1496,6 +1496,7 @@ static const struct ib_device_ops rxe_dev_ops = { .post_recv = rxe_post_recv, .post_send = rxe_post_send, .post_srq_recv = rxe_post_srq_recv, + .process_mad = rxe_process_mad, .query_ah = rxe_query_ah, .query_device = rxe_query_device, .query_pkey = rxe_query_pkey,
Signature · 1 line
2.43.0
Quoted 6 lines · zhenwei pi
v5:
- remove patch "RDMA/core: Fix memory free for GID table", it was applied by Jason separately.
- suggested by Yanjun, use 'skb_network_offset' to calculate the length of received packets.
I am not sure whether you would prefer to add a test case under tools/testing/selftests/rdma or in rdma-core to verify this feature.
If it is possible to include a test case in either selftests or rdma-core, it would be very helpful for the supporters to validate this patch set.
Just my two cents.
Zhu Yanjun
Quoted 38 lines
v4:
- droprxe_ib_device_get_netdevandRXE_PORT, use 1 instead
- avoid UAF to get skb length
- remove one-line wrapperrxe_counter_get, useatomic64_readinstead
- fix memory free for GID table, this is a new patch in this series.v3:
- merge 'RDMA/rxe: userxe_counter_get' into previous commit
- zero *out* MAD memory
- return success with error status rather than failure to avoid uplayer hangv2:
- Fix overflow for PMA counter *link_downed_counter*
- Use *rxe_counter_get* instead of *atomic64_read* for hw-countersv1:
Support PERF MGMT for RXE, add sent/received bytes for RXE counters, also improve coding style.zhenwei pi (3):
RDMA/rxe: removerxe_ib_device_get_netdev()andRXE_PORTRDMA/rxe: add SENT/RCVD bytes
RDMA/rxe: support perf mgmt GET methoddrivers/infiniband/sw/rxe/Makefile | 1 + drivers/infiniband/sw/rxe/rxe_hw_counters.c | 2 + drivers/infiniband/sw/rxe/rxe_hw_counters.h | 2 + drivers/infiniband/sw/rxe/rxe_loc.h | 6 ++ drivers/infiniband/sw/rxe/rxe_mad.c | 101 ++++++++++++++++++++ drivers/infiniband/sw/rxe/rxe_mcast.c | 4 +- drivers/infiniband/sw/rxe/rxe_net.c | 9 +- drivers/infiniband/sw/rxe/rxe_recv.c | 2 + drivers/infiniband/sw/rxe/rxe_verbs.c | 5 +- drivers/infiniband/sw/rxe/rxe_verbs.h | 10 +- 10 files changed, 129 insertions(+), 13 deletions(-)create mode 100644
drivers/infiniband/sw/rxe/rxe_mad.c