IB/rxe: Add port protocol stats
Subsystems (2)
PATCH 1/1 IB/rxe: Add port protocol stats
From: Yonatan Cohen <yonatanc-VPRAkNaXOzVWk0Htik3J/[email protected]>
Expose new counters using the get_hw_stats callback. We expose the following counters:
+---------------------+----------------------------------------+ | Name | Description | |---------------------+----------------------------------------| |sent_pkts | number of sent pkts | |---------------------+----------------------------------------| |rcvd_pkts | number of received packets | |---------------------+----------------------------------------| |out_of_sequence | number of errors due to packet | | | transport sequence number | |---------------------+----------------------------------------| |duplicate_request | number of received duplicated packets. | | | A request that previously executed is | | | named duplicated. | |---------------------+----------------------------------------| |rcvd_rnr_err | number of received RNR by completer | |---------------------+----------------------------------------| |send_rnr_err | number of sent RNR by responder | |---------------------+----------------------------------------| |rcvd_seq_err | number of out of sequence packets | | | received | |---------------------+----------------------------------------| |ack_deffered | number of deferred handling of ack | | | packets. | |---------------------+----------------------------------------| |retry_exceeded_err | number of times retry exceeded | |---------------------+----------------------------------------| |completer_retry_err | number of times completer decided to | | | retry | |---------------------+----------------------------------------| |send_err | number of failed send packet | +---------------------+----------------------------------------+
Signed-off-by Leon Romanovsky <[email protected]>
---
Hi Doug,
This patch applies cleanly on latest Linus's master branch. It was prepared and tested a long time ago, but I delayed it due to merge conflicts and my vacation. In case, you have plans to create another pull request to Linus with Danny's code [1], I would like to consider inclusion of this patch too.
Thanks
[1] http://marc.info/?l=linux-rdma&m=148839266418779&w=2
--- drivers/infiniband/sw/rxe/Makefile | 3 +- drivers/infiniband/sw/rxe/rxe_comp.c | 12 +++++ drivers/infiniband/sw/rxe/rxe_hw_counters.c | 78 +++++++++++++++++++++++++++++ drivers/infiniband/sw/rxe/rxe_hw_counters.h | 61 ++++++++++++++++++++++ drivers/infiniband/sw/rxe/rxe_loc.h | 2 + drivers/infiniband/sw/rxe/rxe_recv.c | 2 + drivers/infiniband/sw/rxe/rxe_resp.c | 5 ++ drivers/infiniband/sw/rxe/rxe_verbs.c | 3 ++ drivers/infiniband/sw/rxe/rxe_verbs.h | 8 +++ 9 files changed, 173 insertions(+), 1 deletion(-) create mode 100644 drivers/infiniband/sw/rxe/rxe_hw_counters.c create mode 100644 drivers/infiniband/sw/rxe/rxe_hw_counters.h
drivers/infiniband/sw/rxe/Makefile +2 -1
@@ -20,4 +20,5 @@ rdma_rxe-y := \ rxe_mcast.o \ rxe_task.o \ rxe_net.o \ - rxe_sysfs.o + rxe_sysfs.o \ + rxe_hw_counters.o
drivers/infiniband/sw/rxe/rxe_comp.c +12 -0
@@ -154,6 +154,8 @@ void rxe_comp_queue_pkt(struct rxe_dev *rxe, struct rxe_qp *qp,skb_queue_tail (&qp->resp_pkts, skb); must_sched =skb_queue_len (&qp->resp_pkts) >1 ; +if (must_sched !=0 ) +rxe_counter_inc (rxe,RXE_CNT_COMPLETER_SCHED );rxe_run_task (&qp->comp.task, must_sched); }
@@ -236,6 +238,7 @@ static inline enum comp_state check_ack(struct rxe_qp *qp, {unsigned int mask = pkt->mask;u8 syn; +struct rxe_dev *rxe =to_rdev (qp->ibqp.device);/* Check the sequence only */ switch (qp->comp.opcode) {
@@ -298,6 +301,7 @@ static inline enum comp_state check_ack(struct rxe_qp *qp,return COMPST_WRITE_SEND ;case AETH_RNR_NAK : +rxe_counter_inc (rxe,RXE_CNT_RCV_RNR );return COMPST_RNR_RETRY ;case AETH_NAK :
@@ -307,6 +311,8 @@ static inline enum comp_state check_ack(struct rxe_qp *qp,* before */ if (psn_compare (pkt->psn, qp->comp.psn) >0 ) { +rxe_counter_inc (rxe, +RXE_CNT_RCV_SEQ_ERR ); qp->comp.psn = pkt->psn;if (qp->req.wait_psn) { qp->req.wait_psn =0 ;
@@ -534,6 +540,7 @@ static void rxe_drain_resp_pkts(struct rxe_qp *qp, bool notify)int rxe_completer (void *arg) {struct rxe_qp *qp = (struct rxe_qp *)arg; +struct rxe_dev *rxe =to_rdev (qp->ibqp.device);struct rxe_send_wqe *wqe = wqe;struct sk_buff *skb =NULL ;struct rxe_pkt_info *pkt =NULL ;
@@ -685,6 +692,8 @@ int rxe_completer(void *arg)/* tell the requester to retry the * send send queue next time around */ +rxe_counter_inc (rxe, +RXE_CNT_COMP_RETRY ); qp->req.need_retry =1 ;rxe_run_task (&qp->req.task,1 ); }
@@ -699,6 +708,7 @@ int rxe_completer(void *arg) goto exit; }else { +rxe_counter_inc (rxe,RXE_CNT_RETRY_EXCEEDED ); wqe->status =IB_WC_RETRY_EXC_ERR ; state =COMPST_ERROR ; }
@@ -720,6 +730,8 @@ int rxe_completer(void *arg) skb =NULL ; goto exit; }else { +rxe_counter_inc (rxe, +RXE_CNT_RNR_RETRY_EXCEEDED ); wqe->status =IB_WC_RNR_RETRY_EXC_ERR ; state =COMPST_ERROR ; }
drivers/infiniband/sw/rxe/rxe_hw_counters.c new file +78 -0
78 lines not on this page · raw message
drivers/infiniband/sw/rxe/rxe_hw_counters.h new file +61 -0
61 lines not on this page · raw message
drivers/infiniband/sw/rxe/rxe_loc.h +2 -0
@@ -278,6 +278,7 @@ static inline int rxe_xmit_packet(struct rxe_dev *rxe, struct rxe_qp *qp,if (err) { rxe->xmit_errors++; +rxe_counter_inc (rxe,RXE_CNT_SEND_ERR );return err; }
@@ -287,6 +288,7 @@ static inline int rxe_xmit_packet(struct rxe_dev *rxe, struct rxe_qp *qp,rxe_run_task (&qp->comp.task,1 ); } +rxe_counter_inc (rxe,RXE_CNT_SENT_PKTS ); goto done; drop:
drivers/infiniband/sw/rxe/rxe_recv.c +2 -0
@@ -403,6 +403,8 @@ int rxe_rcv(struct sk_buff *skb) goto drop; } +rxe_counter_inc (rxe,RXE_CNT_RCVD_PKTS ); +if (unlikely (bth_qpn (pkt) ==IB_MULTICAST_QPN ))rxe_rcv_mcast_pkt (rxe, skb);else
drivers/infiniband/sw/rxe/rxe_resp.c +5 -0
@@ -149,6 +149,7 @@ static enum resp_states check_psn(struct rxe_qp *qp, struct rxe_pkt_info *pkt) {int diff =psn_compare (pkt->psn, qp->resp.psn); +struct rxe_dev *rxe =to_rdev (qp->ibqp.device);switch (qp_type (qp)) {case IB_QPT_RC :
@@ -157,9 +158,11 @@ static enum resp_states check_psn(struct rxe_qp *qp,return RESPST_CLEANUP ; qp->resp.sent_psn_nak =1 ; +rxe_counter_inc (rxe,RXE_CNT_OUT_OF_SEQ_REQ );return RESPST_ERR_PSN_OUT_OF_SEQ ; }else if (diff <0 ) { +rxe_counter_inc (rxe,RXE_CNT_DUP_REQ );return RESPST_DUPLICATE_REQUEST ; }
@@ -1224,6 +1227,7 @@ void rxe_drain_req_pkts(struct rxe_qp *qp, bool notify)int rxe_responder (void *arg) {struct rxe_qp *qp = (struct rxe_qp *)arg; +struct rxe_dev *rxe =to_rdev (qp->ibqp.device);enum resp_states state;struct rxe_pkt_info *pkt =NULL ;int ret =0 ;
@@ -1312,6 +1316,7 @@ int rxe_responder(void *arg)break ;case RESPST_ERR_RNR :if (qp_type (qp) ==IB_QPT_RC ) { +rxe_counter_inc (rxe,RXE_CNT_SND_RNR );/* RC - class B */ send_ack(qp, pkt, AETH_RNR_NAK | (~AETH_TYPE_MASK &
drivers/infiniband/sw/rxe/rxe_verbs.c +3 -0
@@ -35,6 +35,7 @@#include "rxe.h" #include "rxe_loc.h" #include "rxe_queue.h" +#include "rxe_hw_counters.h" static int rxe_query_device(struct ib_device *dev, struct ib_device_attr *attr,
@@ -1318,6 +1319,8 @@ int rxe_register_device(struct rxe_dev *rxe) dev->map_mr_sg = rxe_map_mr_sg; dev->attach_mcast = rxe_attach_mcast; dev->detach_mcast = rxe_detach_mcast; + dev->get_hw_stats = rxe_ib_get_hw_stats; + dev->alloc_hw_stats = rxe_ib_alloc_hw_stats; err =ib_register_device (dev,NULL );if (err) {
drivers/infiniband/sw/rxe/rxe_verbs.h +8 -0
@@ -38,6 +38,7 @@#include <rdma/rdma_user_rxe.h> #include "rxe_pool.h" #include "rxe_task.h" +#include "rxe_hw_counters.h" static inline int pkey_match (u16 key1,u16 key2) {
@@ -401,10 +402,17 @@ struct rxe_dev {spinlock_t mmap_offset_lock;/* guard mmap_offset */ int mmap_offset; +u64 stats_counters[RXE_NUM_OF_COUNTERS ]; +struct rxe_port port;struct list_head list; }; +static inline void rxe_counter_inc (struct rxe_dev *rxe,enum rxe_counters cnt) +{ + rxe->stats_counters[cnt]++; +} +static inline struct rxe_dev *to_rdev (struct ib_device *dev) { return dev ? container_of(dev, struct rxe_dev, ib_dev) : NULL;
--
2.12.0
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html
A message that is not in the archive (sent off-list or not yet received): 20170302081554.5012-1-leon-DgEjT+Ai2ygdnm+yROfE0A@public.gmane.org
Boyer, Andrew
Reviewed-byQuoted 62 lines · On 3/2/17, 3:15 AM, "[email protected]…
From: Yonatan Cohen <yonatanc-VPRAkNaXOzVWk0Htik3J/[email protected]>
Expose new counters using the
get_hw_statscallback. We expose the following counters:+---------------------+----------------------------------------+ | Name | Description | |---------------------+----------------------------------------| |sent_pkts | number of sent pkts | |---------------------+----------------------------------------| |rcvd_pkts | number of received packets | |---------------------+----------------------------------------| |out_of_sequence | number of errors due to packet | | | transport sequence number | |---------------------+----------------------------------------| |duplicate_request | number of received duplicated packets. | | | A request that previously executed is | | | named duplicated. | |---------------------+----------------------------------------| |rcvd_rnr_err | number of received RNR by completer | |---------------------+----------------------------------------| |send_rnr_err | number of sent RNR by responder | |---------------------+----------------------------------------| |rcvd_seq_err | number of out of sequence packets | | | received | |---------------------+----------------------------------------| |ack_deffered | number of deferred handling of ack | | | packets. | |---------------------+----------------------------------------| |retry_exceeded_err | number of times retry exceeded | |---------------------+----------------------------------------| |completer_retry_err | number of times completer decided to | | | retry | |---------------------+----------------------------------------| |send_err | number of failed send packet | +---------------------+----------------------------------------+Signed-off-by Leon Romanovsky <[email protected]>---
drivers/infiniband/sw/rxe/Makefile | 3 +- drivers/infiniband/sw/rxe/rxe_comp.c | 12 +++++ drivers/infiniband/sw/rxe/rxe_hw_counters.c | 78 +++++++++++++++++++++++++++++ drivers/infiniband/sw/rxe/rxe_hw_counters.h | 61 ++++++++++++++++++++++ drivers/infiniband/sw/rxe/rxe_loc.h | 2 + drivers/infiniband/sw/rxe/rxe_recv.c | 2 + drivers/infiniband/sw/rxe/rxe_resp.c | 5 ++ drivers/infiniband/sw/rxe/rxe_verbs.c | 3 ++ drivers/infiniband/sw/rxe/rxe_verbs.h | 8 +++ 9 files changed, 173 insertions(+), 1 deletion(-)create mode 100644
drivers/infiniband/sw/rxe/rxe_hw_counters.ccreate mode 100644drivers/infiniband/sw/rxe/rxe_hw_counters.hdrivers/infiniband/sw/rxe/rxe_comp.c +0
-0
drivers/infiniband/sw/rxe/rxe_comp.c int rxe_completer(void *arg) · line 699
Quoted 8 lines
drivers/infiniband/sw/rxe/rxe_comp.c +2
-0@@ -685,6 +692,8 @@ int rxe_completer(void *arg)/* tell the requester to retry the * send send queue next time around */ +rxe_counter_inc (rxe, +RXE_CNT_COMP_RETRY ); qp->req.need_retry =1 ;rxe_run_task (&qp->req.task,1 ); }
While here, fix ³send send queue² in comments?
Quoted 14 lines
drivers/infiniband/sw/rxe/rxe_verbs.h +1
-0@@ -38,6 +38,7 @@#include <rdma/rdma_user_rxe.h> #include "rxe_pool.h" #include "rxe_task.h" +#include "rxe_hw_counters.h" static inline int pkey_match (u16 key1,u16 key2) {
drivers/infiniband/sw/rxe/rxe_verbs.h struct rxe_dev { · line 409
Quoted 8 lines
drivers/infiniband/sw/rxe/rxe_verbs.h +2
-0@@ -401,10 +402,17 @@ struct rxe_dev {spinlock_t mmap_offset_lock;/* guard mmap_offset */ int mmap_offset; +u64 stats_counters[RXE_NUM_OF_COUNTERS ]; +struct rxe_port port;struct list_head list; };
Match indentation with other fields?
Looks good otherwise.
Reviewed-by Andrew Boyer <[email protected]>
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html
A message that is not in the archive (sent off-list or not yet received): D4E6FBE1.13839%[email protected]
Quoted 103 lines · Andrew
Quoted 62 lines · On 3/2/17, 3:15 AM, "[email protected]…
From: Yonatan Cohen <yonatanc-VPRAkNaXOzVWk0Htik3J/[email protected]>
Expose new counters using the
get_hw_statscallback. We expose the following counters:+---------------------+----------------------------------------+ | Name | Description | |---------------------+----------------------------------------| |sent_pkts | number of sent pkts | |---------------------+----------------------------------------| |rcvd_pkts | number of received packets | |---------------------+----------------------------------------| |out_of_sequence | number of errors due to packet | | | transport sequence number | |---------------------+----------------------------------------| |duplicate_request | number of received duplicated packets. | | | A request that previously executed is | | | named duplicated. | |---------------------+----------------------------------------| |rcvd_rnr_err | number of received RNR by completer | |---------------------+----------------------------------------| |send_rnr_err | number of sent RNR by responder | |---------------------+----------------------------------------| |rcvd_seq_err | number of out of sequence packets | | | received | |---------------------+----------------------------------------| |ack_deffered | number of deferred handling of ack | | | packets. | |---------------------+----------------------------------------| |retry_exceeded_err | number of times retry exceeded | |---------------------+----------------------------------------| |completer_retry_err | number of times completer decided to | | | retry | |---------------------+----------------------------------------| |send_err | number of failed send packet | +---------------------+----------------------------------------+Signed-off-by Leon Romanovsky <[email protected]>---
drivers/infiniband/sw/rxe/Makefile | 3 +- drivers/infiniband/sw/rxe/rxe_comp.c | 12 +++++ drivers/infiniband/sw/rxe/rxe_hw_counters.c | 78 +++++++++++++++++++++++++++++ drivers/infiniband/sw/rxe/rxe_hw_counters.h | 61 ++++++++++++++++++++++ drivers/infiniband/sw/rxe/rxe_loc.h | 2 + drivers/infiniband/sw/rxe/rxe_recv.c | 2 + drivers/infiniband/sw/rxe/rxe_resp.c | 5 ++ drivers/infiniband/sw/rxe/rxe_verbs.c | 3 ++ drivers/infiniband/sw/rxe/rxe_verbs.h | 8 +++ 9 files changed, 173 insertions(+), 1 deletion(-)create mode 100644
drivers/infiniband/sw/rxe/rxe_hw_counters.ccreate mode 100644drivers/infiniband/sw/rxe/rxe_hw_counters.hdrivers/infiniband/sw/rxe/rxe_comp.c +0
-0drivers/infiniband/sw/rxe/rxe_comp.c int rxe_completer(void *arg) · line 699
Quoted 8 lines
drivers/infiniband/sw/rxe/rxe_comp.c +2
-0@@ -685,6 +692,8 @@ int rxe_completer(void *arg)/* tell the requester to retry the * send send queue next time around */ +rxe_counter_inc (rxe, +RXE_CNT_COMP_RETRY ); qp->req.need_retry =1 ;rxe_run_task (&qp->req.task,1 ); }While here, fix ³send send queue² in comments?
Quoted 14 lines
drivers/infiniband/sw/rxe/rxe_verbs.h +1
-0@@ -38,6 +38,7 @@#include <rdma/rdma_user_rxe.h> #include "rxe_pool.h" #include "rxe_task.h" +#include "rxe_hw_counters.h" static inline int pkey_match (u16 key1,u16 key2) {drivers/infiniband/sw/rxe/rxe_verbs.h struct rxe_dev { · line 409
Quoted 8 lines
drivers/infiniband/sw/rxe/rxe_verbs.h +2
-0@@ -401,10 +402,17 @@ struct rxe_dev {spinlock_t mmap_offset_lock;/* guard mmap_offset */ int mmap_offset; +u64 stats_counters[RXE_NUM_OF_COUNTERS ]; +struct rxe_port port;struct list_head list; };Match indentation with other fields?
Quoted 2 lines
Looks good otherwise.
Reviewed-by Andrew Boyer <[email protected]>
Thanks for the review,
I'll post v1.
Quoted 5 lines
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html