Skip to contentSkip to conversation

Support PERF MGMT for RXE

PATCH Superseded zhenwei pi 2 participants 9 messages Apr 8 – Apr 11

Subsystems (2)

v5 of 7 Superseded Latest is v7 →

lore
Share & tools
lore

zhenwei pi

author
#0 lore raw

v5 cover Support PERF MGMT for RXE

v5:
- remove patch "RDMA/core: Fix memory free for GID table", it was applied by Jason separately.
- suggested by Yanjun, use 'skb_network_offset' to calculate the length of received packets.

v4:
- drop rxe_ib_device_get_netdev and RXE_PORT, use 1 instead
- avoid UAF to get skb length
- remove one-line wrapper rxe_counter_get, use atomic64_read instead
- fix memory free for GID table, this is a new patch in this series.

v3:
- merge 'RDMA/rxe: use rxe_counter_get' into previous commit
- zero *out* MAD memory
- return success with error status rather than failure to avoid uplayer hang

v2:
- Fix overflow for PMA counter *link_downed_counter*
- Use *rxe_counter_get* instead of *atomic64_read* for hw-counters

v1:
Support PERF MGMT for RXE, add sent/received bytes for RXE counters, also improve coding style.

zhenwei pi (3):
RDMA/rxe: remove rxe_ib_device_get_netdev() and RXE_PORT RDMA/rxe: add SENT/RCVD bytes
RDMA/rxe: support perf mgmt GET method

 drivers/infiniband/sw/rxe/Makefile          |   1 +
 drivers/infiniband/sw/rxe/rxe_hw_counters.c |   2 +
 drivers/infiniband/sw/rxe/rxe_hw_counters.h |   2 +
 drivers/infiniband/sw/rxe/rxe_loc.h         |   6 ++
 drivers/infiniband/sw/rxe/rxe_mad.c         | 101 ++++++++++++++++++++
 drivers/infiniband/sw/rxe/rxe_mcast.c       |   4 +-
 drivers/infiniband/sw/rxe/rxe_net.c         |   9 +-
 drivers/infiniband/sw/rxe/rxe_recv.c        |   2 +
 drivers/infiniband/sw/rxe/rxe_verbs.c       |   5 +-
 drivers/infiniband/sw/rxe/rxe_verbs.h       |  10 +-
 10 files changed, 129 insertions(+), 13 deletions(-)
 create mode 100644 drivers/infiniband/sw/rxe/rxe_mad.c
Signature · 1 line
2.43.0
author
#1 lore raw

PATCH v5 1/3 RDMA/rxe: remove rxe_ib_device_get_netdev() and RXE_PORT

Suggested by Leon, remove the rxe_ib_device_get_netdev() wrapper and the RXE_PORT definition. These additions do not improve readability, and RXE has always had only a single port.

---
 drivers/infiniband/sw/rxe/rxe_mcast.c | 4 ++--
 drivers/infiniband/sw/rxe/rxe_net.c   | 7 +++----
 drivers/infiniband/sw/rxe/rxe_verbs.c | 4 ++--
 drivers/infiniband/sw/rxe/rxe_verbs.h | 6 ------
 4 files changed, 7 insertions(+), 14 deletions(-)
drivers/infiniband/sw/rxe/rxe_mcast.c +2 -2
index 5cad72073eca..acd03bd87794 100644
@@ -34,7 +34,7 @@ static int rxe_mcast_add(struct rxe_dev *rxe, union ib_gid *mgid)
 	struct net_device *ndev;
 	int ret;
 
-	ndev = rxe_ib_device_get_netdev(&rxe->ib_dev);
+	ndev = ib_device_get_netdev(&rxe->ib_dev, 1);
 	if (!ndev)
 		return -ENODEV;
 
@@ -59,7 +59,7 @@ static int rxe_mcast_del(struct rxe_dev *rxe, union ib_gid *mgid)
 	struct net_device *ndev;
 	int ret;
 
-	ndev = rxe_ib_device_get_netdev(&rxe->ib_dev);
+	ndev = ib_device_get_netdev(&rxe->ib_dev, 1);
 	if (!ndev)
 		return -ENODEV;
 
drivers/infiniband/sw/rxe/rxe_net.c +3 -4
index 211bd3000acc..6621d01ac32d 100644
@@ -602,7 +602,7 @@ const char *rxe_parent_name(struct rxe_dev *rxe, unsigned int port_num)
 	struct net_device *ndev;
 	char *ndev_name;
 
-	ndev = rxe_ib_device_get_netdev(&rxe->ib_dev);
+	ndev = ib_device_get_netdev(&rxe->ib_dev, 1);
 	if (!ndev)
 		return NULL;
 	ndev_name = ndev->name;
@@ -646,12 +646,11 @@ static void rxe_sock_put(struct sock *sk,
 
 void rxe_net_del(struct ib_device *dev)
 {
-	struct rxe_dev *rxe = container_of(dev, struct rxe_dev, ib_dev);
 	struct net_device *ndev;
 	struct sock *sk;
 	struct net *net;
 
-	ndev = rxe_ib_device_get_netdev(&rxe->ib_dev);
+	ndev = ib_device_get_netdev(dev, 1);
 	if (!ndev)
 		return;
 
@@ -699,7 +698,7 @@ void rxe_set_port_state(struct rxe_dev *rxe)
 {
 	struct net_device *ndev;
 
-	ndev = rxe_ib_device_get_netdev(&rxe->ib_dev);
+	ndev = ib_device_get_netdev(&rxe->ib_dev, 1);
 	if (!ndev)
 		return;
 
drivers/infiniband/sw/rxe/rxe_verbs.c +2 -2
index 4d4891dc2884..eb17b6086d5e 100644
@@ -50,7 +50,7 @@ static int rxe_query_port(struct ib_device *ibdev,
 		goto err_out;
 	}
 
-	ndev = rxe_ib_device_get_netdev(ibdev);
+	ndev = ib_device_get_netdev(ibdev, 1);
 	if (!ndev) {
 		err = -ENODEV;
 		goto err_out;
@@ -1441,7 +1441,7 @@ static int rxe_enable_driver(struct ib_device *ib_dev)
 	struct rxe_dev *rxe = container_of(ib_dev, struct rxe_dev, ib_dev);
 	struct net_device *ndev;
 
-	ndev = rxe_ib_device_get_netdev(ib_dev);
+	ndev = ib_device_get_netdev(ib_dev, 1);
 	if (!ndev)
 		return -ENODEV;
 
drivers/infiniband/sw/rxe/rxe_verbs.h +0 -6
index d92f80d16f78..e800545d1046 100644
@@ -415,7 +415,6 @@ struct rxe_port {
 	u32			qp_gsi_index;
 };
 
-#define	RXE_PORT	1
 struct rxe_dev {
 	struct ib_device	ib_dev;
 	struct ib_device_attr	attr;
@@ -451,11 +450,6 @@ struct rxe_dev {
 	struct rxe_port		port;
 };
 
-static inline struct net_device *rxe_ib_device_get_netdev(struct ib_device *dev)
-{
-	return ib_device_get_netdev(dev, RXE_PORT);
-}
-
 static inline void rxe_counter_inc(struct rxe_dev *rxe, enum rxe_counters index)
 {
 	atomic64_inc(&rxe->stats_counters[index]);
Signature · 1 line
2.43.0
author
#2 lore raw

PATCH v5 2/3 RDMA/rxe: add SENT/RCVD bytes

In reply to zhenwei pi

There is a lack of sent/received counter in bytes.

---
 drivers/infiniband/sw/rxe/rxe_hw_counters.c | 2 ++
 drivers/infiniband/sw/rxe/rxe_hw_counters.h | 2 ++
 drivers/infiniband/sw/rxe/rxe_net.c         | 2 ++
 drivers/infiniband/sw/rxe/rxe_recv.c        | 2 ++
 drivers/infiniband/sw/rxe/rxe_verbs.h       | 6 ++++++
 5 files changed, 14 insertions(+)
drivers/infiniband/sw/rxe/rxe_hw_counters.c +2 -0
index 437917a7d8f2..17edaa9a9b9b 100644
@@ -22,6 +22,8 @@ static const struct rdma_stat_desc rxe_counter_descs[] = {
 	[RXE_CNT_LINK_DOWNED].name         =  "link_downed",
 	[RXE_CNT_RDMA_SEND].name           =  "rdma_sends",
 	[RXE_CNT_RDMA_RECV].name           =  "rdma_recvs",
+	[RXE_CNT_SENT_BYTES].name          =  "sent_bytes",
+	[RXE_CNT_RCVD_BYTES].name          =  "rcvd_bytes",
 };
 
 int rxe_ib_get_hw_stats(struct ib_device *ibdev,
drivers/infiniband/sw/rxe/rxe_hw_counters.h +2 -0
index 051f9e1c3852..01b355103cbc 100644
@@ -26,6 +26,8 @@ enum rxe_counters {
 	RXE_CNT_LINK_DOWNED,
 	RXE_CNT_RDMA_SEND,
 	RXE_CNT_RDMA_RECV,
+	RXE_CNT_SENT_BYTES,
+	RXE_CNT_RCVD_BYTES,
 	RXE_NUM_OF_COUNTERS
 };
 
drivers/infiniband/sw/rxe/rxe_net.c +2 -0
index 6621d01ac32d..86660031ffa2 100644
@@ -503,6 +503,7 @@ int rxe_xmit_packet(struct rxe_qp *qp, struct rxe_pkt_info *pkt,
 	int err;
 	int is_request = pkt->mask & RXE_REQ_MASK;
 	struct rxe_dev *rxe = to_rdev(qp->ibqp.device);
+	unsigned int skblen = skb->len;
 	unsigned long flags;
 
 	spin_lock_irqsave(&qp->state_lock, flags);
@@ -526,6 +527,7 @@ int rxe_xmit_packet(struct rxe_qp *qp, struct rxe_pkt_info *pkt,
 	}
 
 	rxe_counter_inc(rxe, RXE_CNT_SENT_PKTS);
+	rxe_counter_add(rxe, RXE_CNT_SENT_BYTES, skblen);
 	goto done;
 
 drop:
drivers/infiniband/sw/rxe/rxe_recv.c +2 -0
index 5861e4244049..e7bab89e7d8d 100644
@@ -318,6 +318,7 @@ void rxe_rcv(struct sk_buff *skb)
 	int err;
 	struct rxe_pkt_info *pkt = SKB_TO_PKT(skb);
 	struct rxe_dev *rxe = pkt->rxe;
+	unsigned int skblen = skb->len - skb_network_offset(skb);
 
 	if (unlikely(skb->len < RXE_BTH_BYTES))
 		goto drop;
@@ -341,6 +342,7 @@ void rxe_rcv(struct sk_buff *skb)
 	if (unlikely(err))
 		goto drop;
 
+	rxe_counter_add(rxe, RXE_CNT_RCVD_BYTES, skblen);
 	rxe_counter_inc(rxe, RXE_CNT_RCVD_PKTS);
 
 	if (unlikely(bth_qpn(pkt) == IB_MULTICAST_QPN))
drivers/infiniband/sw/rxe/rxe_verbs.h +6 -0
index e800545d1046..0f5ffd94643f 100644
@@ -455,6 +455,12 @@ static inline void rxe_counter_inc(struct rxe_dev *rxe, enum rxe_counters index)
 	atomic64_inc(&rxe->stats_counters[index]);
 }
 
+static inline void rxe_counter_add(struct rxe_dev *rxe, enum rxe_counters index,
+				   s64 val)
+{
+	atomic64_add(val, &rxe->stats_counters[index]);
+}
+
 static inline struct rxe_dev *to_rdev(struct ib_device *dev)
 {
 	return dev ? container_of(dev, struct rxe_dev, ib_dev) : NULL;
Signature · 1 line
2.43.0
author
#7 lore raw

PATCH v5 3/3 RDMA/rxe: support perf mgmt GET method

In reply to zhenwei pi

In RXE, hardware counters are already supported, but not in a standardized manner. For instance, user-space monitoring tools like atop only read from the *counters* directory. Therefore, it is necessary to add perf management support to RXE.

Also use rxe_counter_get instead of raw atomic64_read in hw-counters.

---
 drivers/infiniband/sw/rxe/Makefile    |   1 +
 drivers/infiniband/sw/rxe/rxe_loc.h   |   6 ++
 drivers/infiniband/sw/rxe/rxe_mad.c   | 101 ++++++++++++++++++++++++++
 drivers/infiniband/sw/rxe/rxe_verbs.c |   1 +
 4 files changed, 109 insertions(+)
 create mode 100644 drivers/infiniband/sw/rxe/rxe_mad.c
drivers/infiniband/sw/rxe/Makefile +1 -0
index 3977f4f13258..e097c1ca1874 100644
@@ -23,6 +23,7 @@ rdma_rxe-y := \
 	rxe_task.o \
 	rxe_net.o \
 	rxe_hw_counters.o \
+	rxe_mad.o \
 	rxe_ns.o
 
 rdma_rxe-$(CONFIG_INFINIBAND_ON_DEMAND_PAGING) += rxe_odp.o
drivers/infiniband/sw/rxe/rxe_loc.h +6 -0
index e095c12699cb..64d636bf80fd 100644
@@ -242,4 +242,10 @@ static inline int rxe_ib_advise_mr(struct ib_pd *pd,
 
 #endif /* CONFIG_INFINIBAND_ON_DEMAND_PAGING */
 
+/* rxe-mad.c */
+int rxe_process_mad(struct ib_device *ibdev, int mad_flags, u32 port_num,
+		    const struct ib_wc *in_wc, const struct ib_grh *in_grh,
+		    const struct ib_mad *in, struct ib_mad *out,
+		    size_t *out_mad_size, u16 *out_mad_pkey_index);
+
 #endif /* RXE_LOC_H */
drivers/infiniband/sw/rxe/rxe_mad.c new file +101 -0

101 lines not on this page · raw message

drivers/infiniband/sw/rxe/rxe_verbs.c +1 -0
index eb17b6086d5e..8edd4dd1f031 100644
@@ -1496,6 +1496,7 @@ static const struct ib_device_ops rxe_dev_ops = {
 	.post_recv = rxe_post_recv,
 	.post_send = rxe_post_send,
 	.post_srq_recv = rxe_post_srq_recv,
+	.process_mad = rxe_process_mad,
 	.query_ah = rxe_query_ah,
 	.query_device = rxe_query_device,
 	.query_pkey = rxe_query_pkey,
Signature · 1 line
2.43.0
#8 lore raw
In reply to zhenwei pi
Quoted 6 lines · zhenwei pi

v5:
- remove patch "RDMA/core: Fix memory free for GID table", it was applied by Jason separately.
- suggested by Yanjun, use 'skb_network_offset' to calculate the length of received packets.

I am not sure whether you would prefer to add a test case under tools/testing/selftests/rdma or in rdma-core to verify this feature.

If it is possible to include a test case in either selftests or rdma-core, it would be very helpful for the supporters to validate this patch set.

Just my two cents.

Zhu Yanjun

Quoted 38 lines

v4:
- drop rxe_ib_device_get_netdev and RXE_PORT, use 1 instead
- avoid UAF to get skb length
- remove one-line wrapper rxe_counter_get, use atomic64_read instead
- fix memory free for GID table, this is a new patch in this series.

v3:
- merge 'RDMA/rxe: use rxe_counter_get' into previous commit
- zero *out* MAD memory
- return success with error status rather than failure to avoid uplayer hang

v2:
- Fix overflow for PMA counter *link_downed_counter*
- Use *rxe_counter_get* instead of *atomic64_read* for hw-counters

v1:
Support PERF MGMT for RXE, add sent/received bytes for RXE counters, also improve coding style.

zhenwei pi (3):
RDMA/rxe: remove rxe_ib_device_get_netdev() and RXE_PORT RDMA/rxe: add SENT/RCVD bytes
RDMA/rxe: support perf mgmt GET method

  drivers/infiniband/sw/rxe/Makefile          |   1 +
  drivers/infiniband/sw/rxe/rxe_hw_counters.c |   2 +
  drivers/infiniband/sw/rxe/rxe_hw_counters.h |   2 +
  drivers/infiniband/sw/rxe/rxe_loc.h         |   6 ++
  drivers/infiniband/sw/rxe/rxe_mad.c         | 101 ++++++++++++++++++++
  drivers/infiniband/sw/rxe/rxe_mcast.c       |   4 +-
  drivers/infiniband/sw/rxe/rxe_net.c         |   9 +-
  drivers/infiniband/sw/rxe/rxe_recv.c        |   2 +
  drivers/infiniband/sw/rxe/rxe_verbs.c       |   5 +-
  drivers/infiniband/sw/rxe/rxe_verbs.h       |  10 +-
  10 files changed, 129 insertions(+), 13 deletions(-)

create mode 100644 drivers/infiniband/sw/rxe/rxe_mad.c

#3 lore raw

Re: [PATCH v5 2/3] RDMA/rxe: add SENT/RCVD bytes

In reply to zhenwei pi
Quoted 70 lines · zhenwei pi

There is a lack of sent/received counter in bytes.

---
  drivers/infiniband/sw/rxe/rxe_hw_counters.c | 2 ++
  drivers/infiniband/sw/rxe/rxe_hw_counters.h | 2 ++
  drivers/infiniband/sw/rxe/rxe_net.c         | 2 ++
  drivers/infiniband/sw/rxe/rxe_recv.c        | 2 ++
  drivers/infiniband/sw/rxe/rxe_verbs.h       | 6 ++++++
  5 files changed, 14 insertions(+)
drivers/infiniband/sw/rxe/rxe_hw_counters.c +2 -0
index 437917a7d8f2..17edaa9a9b9b 100644
@@ -22,6 +22,8 @@ static const struct rdma_stat_desc rxe_counter_descs[] = {
  	[RXE_CNT_LINK_DOWNED].name         =  "link_downed",
  	[RXE_CNT_RDMA_SEND].name           =  "rdma_sends",
  	[RXE_CNT_RDMA_RECV].name           =  "rdma_recvs",
+	[RXE_CNT_SENT_BYTES].name          =  "sent_bytes",
+	[RXE_CNT_RCVD_BYTES].name          =  "rcvd_bytes",
  };
 
  int rxe_ib_get_hw_stats(struct ib_device *ibdev,
drivers/infiniband/sw/rxe/rxe_hw_counters.h +2 -0
index 051f9e1c3852..01b355103cbc 100644
@@ -26,6 +26,8 @@ enum rxe_counters {
  	RXE_CNT_LINK_DOWNED,
  	RXE_CNT_RDMA_SEND,
  	RXE_CNT_RDMA_RECV,
+	RXE_CNT_SENT_BYTES,
+	RXE_CNT_RCVD_BYTES,
  	RXE_NUM_OF_COUNTERS
  };
 
drivers/infiniband/sw/rxe/rxe_net.c +2 -0
index 6621d01ac32d..86660031ffa2 100644
@@ -503,6 +503,7 @@ int rxe_xmit_packet(struct rxe_qp *qp, struct rxe_pkt_info *pkt,
  	int err;
  	int is_request = pkt->mask & RXE_REQ_MASK;
  	struct rxe_dev *rxe = to_rdev(qp->ibqp.device);
+	unsigned int skblen = skb->len;
  	unsigned long flags;
 
  	spin_lock_irqsave(&qp->state_lock, flags);
@@ -526,6 +527,7 @@ int rxe_xmit_packet(struct rxe_qp *qp, struct rxe_pkt_info *pkt,
  	}
 
  	rxe_counter_inc(rxe, RXE_CNT_SENT_PKTS);
+	rxe_counter_add(rxe, RXE_CNT_SENT_BYTES, skblen);
  	goto done;
 
  drop:
drivers/infiniband/sw/rxe/rxe_recv.c +1 -0
index 5861e4244049..e7bab89e7d8d 100644
@@ -318,6 +318,7 @@ void rxe_rcv(struct sk_buff *skb)
  	int err;
  	struct rxe_pkt_info *pkt = SKB_TO_PKT(skb);
  	struct rxe_dev *rxe = pkt->rxe;
+	unsigned int skblen = skb->len - skb_network_offset(skb);
 
  	if (unlikely(skb->len < RXE_BTH_BYTES))
  		goto drop;

drivers/infiniband/sw/rxe/rxe_recv.c void rxe_rcv(struct sk_buff *skb) · line 348

Quoted 7 lines
drivers/infiniband/sw/rxe/rxe_recv.c +1 -0
@@ -341,6 +342,7 @@ void rxe_rcv(struct sk_buff *skb)
  	if (unlikely(err))
  		goto drop;
 
+	rxe_counter_add(rxe, RXE_CNT_RCVD_BYTES, skblen);
  	rxe_counter_inc(rxe, RXE_CNT_RCVD_PKTS);
 
  	if (unlikely(bth_qpn(pkt) == IB_MULTICAST_QPN))

int rxe_xmit_packet(struct rxe_qp *qp, struct rxe_pkt_info *pkt,
struct sk_buff *skb)

{
        int err;
        int is_request = pkt->mask & RXE_REQ_MASK;
        struct rxe_dev *rxe = to_rdev(qp->ibqp.device);
        unsigned long flags;

skb->len is printed here, that is len1
...

        if (pkt->mask & RXE_LOOPBACK_MASK)
                err = rxe_loopback(skb, pkt);
        else
                err = rxe_send(skb, pkt);

...
}

In the following function

static int rxe_loopback(struct sk_buff *skb, struct rxe_pkt_info *pkt) {
...

        if (skb->protocol == htons(ETH_P_IP))
                skb_pull(skb, sizeof(struct iphdr));
        else
                skb_pull(skb, sizeof(struct ipv6hdr));

...

        /* remove udp header */
        skb_pull(skb, sizeof(struct udphdr));

print skb->len here, that is len2

        rxe_rcv(skb);

...
}

Does len1 equal to len2?

If not, the transmitted length appears to differ from the received length when using loopback.

I am not sure whether this is expected behavior.

The same observation also applies to the non-loopback case.

Zhu Yanjun

Quoted 17 lines
drivers/infiniband/sw/rxe/rxe_verbs.h +6 -0
index e800545d1046..0f5ffd94643f 100644
@@ -455,6 +455,12 @@ static inline void rxe_counter_inc(struct rxe_dev *rxe, enum rxe_counters index)
  	atomic64_inc(&rxe->stats_counters[index]);
  }
 
+static inline void rxe_counter_add(struct rxe_dev *rxe, enum rxe_counters index,
+				   s64 val)
+{
+	atomic64_add(val, &rxe->stats_counters[index]);
+}
+
  static inline struct rxe_dev *to_rdev(struct ib_device *dev)
  {
  	return dev ? container_of(dev, struct rxe_dev, ib_dev) : NULL;
#4 lore raw

Re: [PATCH v5 2/3] RDMA/rxe: add SENT/RCVD bytes

Quoted 105 lines · yanjun.zhu
Quoted 67 lines · zhenwei pi

There is a lack of sent/received counter in bytes.

---
  drivers/infiniband/sw/rxe/rxe_hw_counters.c | 2 ++ drivers/infiniband/sw/rxe/rxe_hw_counters.h | 2 ++ drivers/infiniband/sw/rxe/rxe_net.c         | 2 ++ drivers/infiniband/sw/rxe/rxe_recv.c        | 2 ++ drivers/infiniband/sw/rxe/rxe_verbs.h       | 6 ++++++

  5 files changed, 14 insertions(+)
drivers/infiniband/sw/rxe/rxe_hw_counters.c +2 -0
index 437917a7d8f2..17edaa9a9b9b 100644
@@ -22,6 +22,8 @@ static const struct rdma_stat_desc 
 rxe_counter_descs[] = {
       [RXE_CNT_LINK_DOWNED].name         =  "link_downed",
       [RXE_CNT_RDMA_SEND].name           =  "rdma_sends",
       [RXE_CNT_RDMA_RECV].name           =  "rdma_recvs",
+    [RXE_CNT_SENT_BYTES].name          =  "sent_bytes",
+    [RXE_CNT_RCVD_BYTES].name          =  "rcvd_bytes",
   };
     int rxe_ib_get_hw_stats(struct ib_device *ibdev,
drivers/infiniband/sw/rxe/rxe_hw_counters.h +2 -0
index 051f9e1c3852..01b355103cbc 100644
@@ -26,6 +26,8 @@ enum rxe_counters {
       RXE_CNT_LINK_DOWNED,
       RXE_CNT_RDMA_SEND,
       RXE_CNT_RDMA_RECV,
+    RXE_CNT_SENT_BYTES,
+    RXE_CNT_RCVD_BYTES,

      RXE_NUM_OF_COUNTERS
  };
  diff --git a/drivers/infiniband/sw/rxe/rxe_net.c b/drivers/infiniband/sw/rxe/rxe_net.c index 6621d01ac32d..86660031ffa2 100644

drivers/infiniband/sw/rxe/rxe_net.c +2 -0
@@ -503,6 +503,7 @@ int rxe_xmit_packet(struct rxe_qp *qp, struct 
 rxe_pkt_info *pkt,
       int err;
       int is_request = pkt->mask & RXE_REQ_MASK;
       struct rxe_dev *rxe = to_rdev(qp->ibqp.device);
+    unsigned int skblen = skb->len;
       unsigned long flags;
         spin_lock_irqsave(&qp->state_lock, flags);
@@ -526,6 +527,7 @@ int rxe_xmit_packet(struct rxe_qp *qp, struct 
 rxe_pkt_info *pkt,
       }
         rxe_counter_inc(rxe, RXE_CNT_SENT_PKTS);
+    rxe_counter_add(rxe, RXE_CNT_SENT_BYTES, skblen);
       goto done;

    drop:

drivers/infiniband/sw/rxe/rxe_recv.c +1 -0
index 5861e4244049..e7bab89e7d8d 100644
@@ -318,6 +318,7 @@ void rxe_rcv(struct sk_buff *skb)
       int err;
       struct rxe_pkt_info *pkt = SKB_TO_PKT(skb);
       struct rxe_dev *rxe = pkt->rxe;
+    unsigned int skblen = skb->len - skb_network_offset(skb);
         if (unlikely(skb->len < RXE_BTH_BYTES))
           goto drop;

drivers/infiniband/sw/rxe/rxe_recv.c void rxe_rcv(struct sk_buff *skb) · line 346

Quoted 5 lines
drivers/infiniband/sw/rxe/rxe_recv.c +0 -0
@@ -341,6 +342,7 @@ void rxe_rcv(struct sk_buff *skb)
       if (unlikely(err))
           goto drop;
   +    rxe_counter_add(rxe, RXE_CNT_RCVD_BYTES, skblen);
       rxe_counter_inc(rxe, RXE_CNT_RCVD_PKTS);
         if (unlikely(bth_qpn(pkt) == IB_MULTICAST_QPN))

int rxe_xmit_packet(struct rxe_qp *qp, struct rxe_pkt_info *pkt, struct sk_buff *skb)
{
        int err;
        int is_request = pkt->mask & RXE_REQ_MASK; struct rxe_dev *rxe = to_rdev(qp->ibqp.device); unsigned long flags;

skb->len is printed here, that is len1
...
        if (pkt->mask & RXE_LOOPBACK_MASK)
                err = rxe_loopback(skb, pkt);
        else
                err = rxe_send(skb, pkt);
...
}

In the following function

static int rxe_loopback(struct sk_buff *skb, struct rxe_pkt_info *pkt) {
...
        if (skb->protocol == htons(ETH_P_IP))
                skb_pull(skb, sizeof(struct iphdr));
        else
                skb_pull(skb, sizeof(struct ipv6hdr));

...
        /* remove udp header */
        skb_pull(skb, sizeof(struct udphdr));

Quoted 8 lines

print skb->len here, that is len2

        rxe_rcv(skb);

...
}

Does len1 equal to len2?

I have made tests. The difference between len1 and len2 is 28.

It should be the total of ipv4 header + udp header because I use ipv4 address to make tests.

I am not sure if the bytes of recv data should equal to the bytes of xmit data.

Zhu Yanjun

Quoted 26 lines

If not, the transmitted length appears to differ from the received length when using loopback.

I am not sure whether this is expected behavior.

The same observation also applies to the non-loopback case.

Zhu Yanjun

Quoted 17 lines
drivers/infiniband/sw/rxe/rxe_verbs.h +5 -0
index e800545d1046..0f5ffd94643f 100644
@@ -455,6 +455,12 @@ static inline void rxe_counter_inc(struct 
 rxe_dev *rxe, enum rxe_counters index)
       atomic64_inc(&rxe->stats_counters[index]);
   }
   +static inline void rxe_counter_add(struct rxe_dev *rxe, enum rxe_counters index,
+                   s64 val)
+{
+    atomic64_add(val, &rxe->stats_counters[index]);
+}
+
   static inline struct rxe_dev *to_rdev(struct ib_device *dev)
   {

      return dev ? container_of(dev, struct rxe_dev, ib_dev) : NULL;

Signature · 2 lines
Best Regards,
Yanjun.Zhu
author
#5 lore raw

Re: [PATCH v5 2/3] RDMA/rxe: add SENT/RCVD bytes

Quoted 116 lines · Zhu Yanjun
Quoted 105 lines · yanjun.zhu
Quoted 67 lines · zhenwei pi

There is a lack of sent/received counter in bytes.

---
  drivers/infiniband/sw/rxe/rxe_hw_counters.c | 2 ++ drivers/infiniband/sw/rxe/rxe_hw_counters.h | 2 ++ drivers/infiniband/sw/rxe/rxe_net.c         | 2 ++ drivers/infiniband/sw/rxe/rxe_recv.c        | 2 ++ drivers/infiniband/sw/rxe/rxe_verbs.h       | 6 ++++++

  5 files changed, 14 insertions(+)
drivers/infiniband/sw/rxe/rxe_hw_counters.c +2 -0
index 437917a7d8f2..17edaa9a9b9b 100644
@@ -22,6 +22,8 @@ static const struct rdma_stat_desc 
 rxe_counter_descs[] = {
       [RXE_CNT_LINK_DOWNED].name         =  "link_downed",
       [RXE_CNT_RDMA_SEND].name           =  "rdma_sends",
       [RXE_CNT_RDMA_RECV].name           =  "rdma_recvs",
+    [RXE_CNT_SENT_BYTES].name          =  "sent_bytes",
+    [RXE_CNT_RCVD_BYTES].name          =  "rcvd_bytes",
   };
     int rxe_ib_get_hw_stats(struct ib_device *ibdev,
drivers/infiniband/sw/rxe/rxe_hw_counters.h +2 -0
index 051f9e1c3852..01b355103cbc 100644
@@ -26,6 +26,8 @@ enum rxe_counters {
       RXE_CNT_LINK_DOWNED,
       RXE_CNT_RDMA_SEND,
       RXE_CNT_RDMA_RECV,
+    RXE_CNT_SENT_BYTES,
+    RXE_CNT_RCVD_BYTES,

      RXE_NUM_OF_COUNTERS
  };
  diff --git a/drivers/infiniband/sw/rxe/rxe_net.c b/drivers/ infiniband/sw/rxe/rxe_net.c index 6621d01ac32d..86660031ffa2 100644

drivers/infiniband/sw/rxe/rxe_net.c +2 -0
@@ -503,6 +503,7 @@ int rxe_xmit_packet(struct rxe_qp *qp, struct 
 rxe_pkt_info *pkt,
       int err;
       int is_request = pkt->mask & RXE_REQ_MASK;
       struct rxe_dev *rxe = to_rdev(qp->ibqp.device);
+    unsigned int skblen = skb->len;
       unsigned long flags;
         spin_lock_irqsave(&qp->state_lock, flags);
@@ -526,6 +527,7 @@ int rxe_xmit_packet(struct rxe_qp *qp, struct 
 rxe_pkt_info *pkt,
       }
         rxe_counter_inc(rxe, RXE_CNT_SENT_PKTS);
+    rxe_counter_add(rxe, RXE_CNT_SENT_BYTES, skblen);
       goto done;

    drop:

drivers/infiniband/sw/rxe/rxe_recv.c +1 -0
index 5861e4244049..e7bab89e7d8d 100644
@@ -318,6 +318,7 @@ void rxe_rcv(struct sk_buff *skb)
       int err;
       struct rxe_pkt_info *pkt = SKB_TO_PKT(skb);
       struct rxe_dev *rxe = pkt->rxe;
+    unsigned int skblen = skb->len - skb_network_offset(skb);
         if (unlikely(skb->len < RXE_BTH_BYTES))
           goto drop;

drivers/infiniband/sw/rxe/rxe_recv.c void rxe_rcv(struct sk_buff *skb) · line 346

Quoted 5 lines
drivers/infiniband/sw/rxe/rxe_recv.c +0 -0
@@ -341,6 +342,7 @@ void rxe_rcv(struct sk_buff *skb)
       if (unlikely(err))
           goto drop;
   +    rxe_counter_add(rxe, RXE_CNT_RCVD_BYTES, skblen);
       rxe_counter_inc(rxe, RXE_CNT_RCVD_PKTS);
         if (unlikely(bth_qpn(pkt) == IB_MULTICAST_QPN))

int rxe_xmit_packet(struct rxe_qp *qp, struct rxe_pkt_info *pkt, struct sk_buff *skb)
{
        int err;
        int is_request = pkt->mask & RXE_REQ_MASK; struct rxe_dev *rxe = to_rdev(qp->ibqp.device); unsigned long flags;

skb->len is printed here, that is len1
...
        if (pkt->mask & RXE_LOOPBACK_MASK)
                err = rxe_loopback(skb, pkt);
        else
                err = rxe_send(skb, pkt);
...
}

In the following function

static int rxe_loopback(struct sk_buff *skb, struct rxe_pkt_info *pkt) {
...
        if (skb->protocol == htons(ETH_P_IP))
                skb_pull(skb, sizeof(struct iphdr));
        else
                skb_pull(skb, sizeof(struct ipv6hdr));

...
        /* remove udp header */
        skb_pull(skb, sizeof(struct udphdr));

Quoted 8 lines

print skb->len here, that is len2

        rxe_rcv(skb);

...
}

Does len1 equal to len2?

Quoted 9 lines

I have made tests. The difference between len1 and len2 is 28.

It should be the total of ipv4 header + udp header because I use ipv4 address to make tests.

I am not sure if the bytes of recv data should equal to the bytes of xmit data.

Zhu Yanjun

Yes, I got the same result of 28 bytes difference during v4 version. So I try to use such logic to recalculate the length:

+ unsigned int skblen = skb->len + sizeof(struct udphdr);
...
+ if (skb->protocol == htons(ETH_P_IP))
+ skblen += sizeof(struct iphdr);
+ else if (skb->protocol == htons(ETH_P_IPV6))
+ skblen += sizeof(struct ipv6hdr);
+ rxe_counter_add(rxe, RXE_CNT_RCVD_BYTES, skblen);

then use 'unsigned int skblen = skb->len - skb_network_offset(skb);' in v5.

Quoted 27 lines
Quoted 26 lines

If not, the transmitted length appears to differ from the received length when using loopback.

I am not sure whether this is expected behavior.

The same observation also applies to the non-loopback case.

Zhu Yanjun

Quoted 17 lines
drivers/infiniband/sw/rxe/rxe_verbs.h +5 -0
index e800545d1046..0f5ffd94643f 100644
@@ -455,6 +455,12 @@ static inline void rxe_counter_inc(struct 
 rxe_dev *rxe, enum rxe_counters index)
       atomic64_inc(&rxe->stats_counters[index]);
   }
   +static inline void rxe_counter_add(struct rxe_dev *rxe, enum rxe_counters index,
+                   s64 val)
+{
+    atomic64_add(val, &rxe->stats_counters[index]);
+}
+
   static inline struct rxe_dev *to_rdev(struct ib_device *dev)
   {

      return dev ? container_of(dev, struct rxe_dev, ib_dev) : NULL;

#6 lore raw

Re: [PATCH v5 2/3] RDMA/rxe: add SENT/RCVD bytes

Quoted 139 lines · zhenwei pi
Quoted 116 lines · Zhu Yanjun
Quoted 105 lines · yanjun.zhu
Quoted 67 lines · zhenwei pi

There is a lack of sent/received counter in bytes.

---
  drivers/infiniband/sw/rxe/rxe_hw_counters.c | 2 ++ drivers/infiniband/sw/rxe/rxe_hw_counters.h | 2 ++ drivers/infiniband/sw/rxe/rxe_net.c         | 2 ++ drivers/infiniband/sw/rxe/rxe_recv.c        | 2 ++ drivers/infiniband/sw/rxe/rxe_verbs.h       | 6 ++++++

  5 files changed, 14 insertions(+)
drivers/infiniband/sw/rxe/rxe_hw_counters.c +2 -0
index 437917a7d8f2..17edaa9a9b9b 100644
@@ -22,6 +22,8 @@ static const struct rdma_stat_desc 
 rxe_counter_descs[] = {
       [RXE_CNT_LINK_DOWNED].name         =  "link_downed",
       [RXE_CNT_RDMA_SEND].name           =  "rdma_sends",
       [RXE_CNT_RDMA_RECV].name           =  "rdma_recvs",
+    [RXE_CNT_SENT_BYTES].name          =  "sent_bytes",
+    [RXE_CNT_RCVD_BYTES].name          =  "rcvd_bytes",
   };
     int rxe_ib_get_hw_stats(struct ib_device *ibdev,
drivers/infiniband/sw/rxe/rxe_hw_counters.h +2 -0
index 051f9e1c3852..01b355103cbc 100644
@@ -26,6 +26,8 @@ enum rxe_counters {
       RXE_CNT_LINK_DOWNED,
       RXE_CNT_RDMA_SEND,
       RXE_CNT_RDMA_RECV,
+    RXE_CNT_SENT_BYTES,
+    RXE_CNT_RCVD_BYTES,

      RXE_NUM_OF_COUNTERS
  };
  diff --git a/drivers/infiniband/sw/rxe/rxe_net.c b/drivers/ infiniband/sw/rxe/rxe_net.c index 6621d01ac32d..86660031ffa2 100644

drivers/infiniband/sw/rxe/rxe_net.c +2 -0
@@ -503,6 +503,7 @@ int rxe_xmit_packet(struct rxe_qp *qp, struct 
 rxe_pkt_info *pkt,
       int err;
       int is_request = pkt->mask & RXE_REQ_MASK;
       struct rxe_dev *rxe = to_rdev(qp->ibqp.device);
+    unsigned int skblen = skb->len;
       unsigned long flags;
         spin_lock_irqsave(&qp->state_lock, flags);
@@ -526,6 +527,7 @@ int rxe_xmit_packet(struct rxe_qp *qp, struct 
 rxe_pkt_info *pkt,
       }
         rxe_counter_inc(rxe, RXE_CNT_SENT_PKTS);
+    rxe_counter_add(rxe, RXE_CNT_SENT_BYTES, skblen);
       goto done;

    drop:

drivers/infiniband/sw/rxe/rxe_recv.c +1 -0
index 5861e4244049..e7bab89e7d8d 100644
@@ -318,6 +318,7 @@ void rxe_rcv(struct sk_buff *skb)
       int err;
       struct rxe_pkt_info *pkt = SKB_TO_PKT(skb);
       struct rxe_dev *rxe = pkt->rxe;
+    unsigned int skblen = skb->len - skb_network_offset(skb);
         if (unlikely(skb->len < RXE_BTH_BYTES))
           goto drop;

drivers/infiniband/sw/rxe/rxe_recv.c void rxe_rcv(struct sk_buff *skb) · line 346

Quoted 5 lines
drivers/infiniband/sw/rxe/rxe_recv.c +0 -0
@@ -341,6 +342,7 @@ void rxe_rcv(struct sk_buff *skb)
       if (unlikely(err))
           goto drop;
   +    rxe_counter_add(rxe, RXE_CNT_RCVD_BYTES, skblen);
       rxe_counter_inc(rxe, RXE_CNT_RCVD_PKTS);
         if (unlikely(bth_qpn(pkt) == IB_MULTICAST_QPN))

int rxe_xmit_packet(struct rxe_qp *qp, struct rxe_pkt_info *pkt, struct sk_buff *skb)
{
        int err;
        int is_request = pkt->mask & RXE_REQ_MASK; struct rxe_dev *rxe = to_rdev(qp->ibqp.device); unsigned long flags;

skb->len is printed here, that is len1
...
        if (pkt->mask & RXE_LOOPBACK_MASK)
                err = rxe_loopback(skb, pkt);
        else
                err = rxe_send(skb, pkt);
...
}

In the following function

static int rxe_loopback(struct sk_buff *skb, struct rxe_pkt_info *pkt) {
...
        if (skb->protocol == htons(ETH_P_IP))
                skb_pull(skb, sizeof(struct iphdr));
        else
                skb_pull(skb, sizeof(struct ipv6hdr));

...
        /* remove udp header */
        skb_pull(skb, sizeof(struct udphdr));

Quoted 8 lines

print skb->len here, that is len2

        rxe_rcv(skb);

...
}

Does len1 equal to len2?

Quoted 9 lines

I have made tests. The difference between len1 and len2 is 28.

It should be the total of ipv4 header + udp header because I use ipv4 address to make tests.

I am not sure if the bytes of recv data should equal to the bytes of xmit data.

Zhu Yanjun

Yes, I got the same result of 28 bytes difference during v4 version. So I try to use such logic to recalculate the length:

+    unsigned int skblen = skb->len + sizeof(struct udphdr);
...
+    if (skb->protocol == htons(ETH_P_IP))
+        skblen += sizeof(struct iphdr);
+    else if (skb->protocol == htons(ETH_P_IPV6)) +        skblen += sizeof(struct ipv6hdr);
+    rxe_counter_add(rxe, RXE_CNT_RCVD_BYTES, skblen);

Quoted 1 line

then use 'unsigned int skblen = skb->len - skb_network_offset(skb);' in v5.

Thanks. Not necessary to use this 'unsigned int skblen = skb->len - skb_network_offset(skb);'.

The above udphdr and iphdr/ip6hdr should be OK.

Thanks a lot.
Zhu Yanjun

Quoted 29 lines
Quoted 28 lines
Quoted 27 lines

If not, the transmitted length appears to differ from the received length when using loopback.

I am not sure whether this is expected behavior.

The same observation also applies to the non-loopback case.

Zhu Yanjun

Quoted 17 lines
drivers/infiniband/sw/rxe/rxe_verbs.h +5 -0
index e800545d1046..0f5ffd94643f 100644
@@ -455,6 +455,12 @@ static inline void rxe_counter_inc(struct 
 rxe_dev *rxe, enum rxe_counters index)
       atomic64_inc(&rxe->stats_counters[index]);
   }
   +static inline void rxe_counter_add(struct rxe_dev *rxe, enum rxe_counters index,
+                   s64 val)
+{
+    atomic64_add(val, &rxe->stats_counters[index]);
+}
+
   static inline struct rxe_dev *to_rdev(struct ib_device *dev)
   {

      return dev ? container_of(dev, struct rxe_dev, ib_dev) : NULL;