Skip to contentSkip to conversation

examples/ipsec-secgw: add per core packet stats

PATCH Superseded Anoob Joseph 3 participants 19 messages Apr 21, 2020 – Jul 1, 2020

lore
Share & tools
lore
#0 lore raw

PATCH 1/1 examples/ipsec-secgw: add per core packet stats

Adding per core packet handling stats to analyze traffic distribution when multiple cores are engaged.

Since aggregating the packet stats across cores would affect performance, keeping the feature disabled using compile time flags.

---
 examples/ipsec-secgw/ipsec-secgw.c   | 112 +++++++++++++++++++++++++++++++++--
 examples/ipsec-secgw/ipsec-secgw.h   |   2 +
 examples/ipsec-secgw/ipsec.h         |  22 +++++++
 examples/ipsec-secgw/ipsec_process.c |   5 ++
 4 files changed, 137 insertions(+), 4 deletions(-)
examples/ipsec-secgw/ipsec-secgw.c +108 -4
index 6d02341..eb94187 100644
@@ -288,6 +288,61 @@ adjust_ipv6_pktlen(struct rte_mbuf *m, const struct rte_ipv6_hdr *iph,
 	}
 }
 
+#ifdef ENABLE_STATS
+static uint64_t timer_period = 10; /* default period is 10 seconds */
+
+/* Print out statistics on packet distribution */
+static void
+print_stats(void)
+{
+	uint64_t total_packets_dropped, total_packets_tx, total_packets_rx;
+	unsigned int coreid;
+	float burst_percent;
+
+	total_packets_dropped = 0;
+	total_packets_tx = 0;
+	total_packets_rx = 0;
+
+	const char clr[] = { 27, '[', '2', 'J', '\0' };
+	const char topLeft[] = { 27, '[', '1', ';', '1', 'H', '\0' };
+
+	/* Clear screen and move to top left */
+	printf("%s%s", clr, topLeft);
+
+	printf("\nCore statistics ====================================");
+
+	for (coreid = 0; coreid < RTE_MAX_LCORE; coreid++) {
+		/* skip disabled cores */
+		if (rte_lcore_is_enabled(coreid) == 0)
+			continue;
+		burst_percent = (float)(core_statistics[coreid].burst_rx * 100)/
+					core_statistics[coreid].rx;
+		printf("\nStatistics for core %u ------------------------------"
+			   "\nPackets received: %20"PRIu64
+			   "\nPackets sent: %24"PRIu64
+			   "\nPackets dropped: %21"PRIu64
+			   "\nBurst percent: %23.2f",
+			   coreid,
+			   core_statistics[coreid].rx,
+			   core_statistics[coreid].tx,
+			   core_statistics[coreid].dropped,
+			   burst_percent);
+
+		total_packets_dropped += core_statistics[coreid].dropped;
+		total_packets_tx += core_statistics[coreid].tx;
+		total_packets_rx += core_statistics[coreid].rx;
+	}
+	printf("\nAggregate statistics ==============================="
+		   "\nTotal packets received: %14"PRIu64
+		   "\nTotal packets sent: %18"PRIu64
+		   "\nTotal packets dropped: %15"PRIu64,
+		   total_packets_rx,
+		   total_packets_tx,
+		   total_packets_dropped);
+	printf("\n====================================================\n");
+}
+#endif /* ENABLE_STATS */
+
 static inline void
 prepare_one_packet(struct rte_mbuf *pkt, struct ipsec_traffic *t)
 {
@@ -351,6 +406,7 @@ prepare_one_packet(struct rte_mbuf *pkt, struct ipsec_traffic *t)
 		RTE_LOG(ERR, IPSEC, "Unsupported packet type 0x%x\n",
 			rte_be_to_cpu_16(eth->ether_type));
 		rte_pktmbuf_free(pkt);
+		core_stats_update_drop(1);
 		return;
 	}
 
@@ -471,6 +527,11 @@ send_burst(struct lcore_conf *qconf, uint16_t n, uint16_t port)
 	int32_t ret;
 	uint16_t queueid;
 
+#ifdef ENABLE_STATS
+	int lcore_id = rte_lcore_id();
+	core_statistics[lcore_id].tx += n;
+#endif /* ENABLE_STATS */
+
 	queueid = qconf->tx_queue_id[port];
 	m_table = (struct rte_mbuf **)qconf->tx_mbufs[port].m_table;
 
@@ -478,6 +539,9 @@ send_burst(struct lcore_conf *qconf, uint16_t n, uint16_t port)
 
 	ret = rte_eth_tx_burst(port, queueid, m_table, n);
 	if (unlikely(ret < n)) {
+#ifdef ENABLE_STATS
+		core_statistics[lcore_id].dropped += n-ret;
+#endif /* ENABLE_STATS */
 		do {
 			rte_pktmbuf_free(m_table[ret]);
 		} while (++ret < n);
@@ -584,18 +648,21 @@ inbound_sp_sa(struct sp_ctx *sp, struct sa_ctx *sa, struct traffic_type *ip,
 			continue;
 		}
 		if (res == DISCARD) {
+			core_stats_update_drop(1);
 			rte_pktmbuf_free(m);
 			continue;
 		}
 
 		/* Only check SPI match for processed IPSec packets */
 		if (i < lim && ((m->ol_flags & PKT_RX_SEC_OFFLOAD) == 0)) {
+			core_stats_update_drop(1);
 			rte_pktmbuf_free(m);
 			continue;
 		}
 
 		sa_idx = res - 1;
 		if (!inbound_sa_check(sa, m, sa_idx)) {
+			core_stats_update_drop(1);
 			rte_pktmbuf_free(m);
 			continue;
 		}
@@ -630,8 +697,10 @@ split46_traffic(struct ipsec_traffic *trf, struct rte_mbuf *mb[], uint32_t num)
 					uint8_t *,
 					offsetof(struct ip6_hdr, ip6_nxt));
 			n6++;
-		} else
+		} else {
+			core_stats_update_drop(1);
 			rte_pktmbuf_free(m);
+		}
 	}
 
 	trf->ip4.num = n4;
@@ -682,11 +751,12 @@ outbound_sp(struct sp_ctx *sp, struct traffic_type *ip,
 	for (i = 0; i < ip->num; i++) {
 		m = ip->pkts[i];
 		sa_idx = ip->res[i] - 1;
-		if (ip->res[i] == DISCARD)
+		if (ip->res[i] == DISCARD) {
+			core_stats_update_drop(1);
 			rte_pktmbuf_free(m);
-		else if (ip->res[i] == BYPASS)
+		} else if (ip->res[i] == BYPASS) {
 			ip->pkts[j++] = m;
-		else {
+		} else {
 			ipsec->res[ipsec->num] = sa_idx;
 			ipsec->pkts[ipsec->num++] = m;
 		}
@@ -705,6 +775,8 @@ process_pkts_outbound(struct ipsec_ctx *ipsec_ctx,
 	for (i = 0; i < traffic->ipsec.num; i++)
 		rte_pktmbuf_free(traffic->ipsec.pkts[i]);
 
+	core_stats_update_drop(traffic->ipsec.num);
+
 	traffic->ipsec.num = 0;
 
 	outbound_sp(ipsec_ctx->sp4_ctx, &traffic->ip4, &traffic->ipsec);
@@ -745,12 +817,14 @@ process_pkts_inbound_nosp(struct ipsec_ctx *ipsec_ctx,
 	/* Drop any IPv4 traffic from unprotected ports */
 	for (i = 0; i < traffic->ip4.num; i++)
 		rte_pktmbuf_free(traffic->ip4.pkts[i]);
+	core_stats_update_drop(traffic->ip4.num);
 
 	traffic->ip4.num = 0;
 
 	/* Drop any IPv6 traffic from unprotected ports */
 	for (i = 0; i < traffic->ip6.num; i++)
 		rte_pktmbuf_free(traffic->ip6.pkts[i]);
+	core_stats_update_drop(traffic->ip6.num);
 
 	traffic->ip6.num = 0;
 
@@ -788,6 +862,7 @@ process_pkts_outbound_nosp(struct ipsec_ctx *ipsec_ctx,
 	/* Drop any IPsec traffic from protected ports */
 	for (i = 0; i < traffic->ipsec.num; i++)
 		rte_pktmbuf_free(traffic->ipsec.pkts[i]);
+	core_stats_update_drop(traffic->ipsec.num);
 
 	n = 0;
 
@@ -901,6 +976,7 @@ route4_pkts(struct rt_ctx *rt_ctx, struct rte_mbuf *pkts[], uint8_t nb_pkts)
 		}
 
 		if ((pkt_hop & RTE_LPM_LOOKUP_SUCCESS) == 0) {
+			core_stats_update_drop(1);
 			rte_pktmbuf_free(pkts[i]);
 			continue;
 		}
@@ -953,6 +1029,7 @@ route6_pkts(struct rt_ctx *rt_ctx, struct rte_mbuf *pkts[], uint8_t nb_pkts)
 		}
 
 		if (pkt_hop == -1) {
+			core_stats_update_drop(1);
 			rte_pktmbuf_free(pkts[i]);
 			continue;
 		}
@@ -1099,6 +1176,9 @@ ipsec_poll_mode_worker(void)
 	const uint64_t drain_tsc = (rte_get_tsc_hz() + US_PER_S - 1)
 			/ US_PER_S * BURST_TX_DRAIN_US;
 	struct lcore_rx_queue *rxql;
+#ifdef ENABLE_STATS
+	uint64_t timer_tsc = 0;
+#endif /* ENABLE_STATS */
 
 	prev_tsc = 0;
 	lcore_id = rte_lcore_id();
@@ -1159,6 +1239,19 @@ ipsec_poll_mode_worker(void)
 			drain_tx_buffers(qconf);
 			drain_crypto_buffers(qconf);
 			prev_tsc = cur_tsc;
+#ifdef ENABLE_STATS
+			if (lcore_id == rte_get_master_lcore()) {
+				/* advance the timer */
+				timer_tsc += diff_tsc;
+
+				/* if timer has reached its timeout */
+				if (unlikely(timer_tsc >= timer_period)) {
+					print_stats();
+					/* reset the timer */
+					timer_tsc = 0;
+				}
+			}
+#endif /* ENABLE_STATS */
 		}
 
 		for (i = 0; i < qconf->nb_rx_queue; ++i) {
@@ -1169,6 +1262,12 @@ ipsec_poll_mode_worker(void)
 			nb_rx = rte_eth_rx_burst(portid, queueid,
 					pkts, MAX_PKT_BURST);
 
+#ifdef ENABLE_STATS
+			core_statistics[lcore_id].rx += nb_rx;
+			if (nb_rx == MAX_PKT_BURST)
+				core_statistics[lcore_id].burst_rx += nb_rx;
+#endif /* ENABLE_STATS */
+
 			if (nb_rx > 0)
 				process_pkts(qconf, pkts, nb_rx, portid);
 
@@ -2747,6 +2846,11 @@ main(int32_t argc, char **argv)
 	signal(SIGINT, signal_handler);
 	signal(SIGTERM, signal_handler);
 
+#ifdef ENABLE_STATS
+	/* convert to number of cycles */
+	timer_period *= rte_get_timer_hz();
+#endif /* ENABLE_STATS */
+
 	/* initialize event helper configuration */
 	eh_conf = eh_conf_init();
 	if (eh_conf == NULL)
examples/ipsec-secgw/ipsec-secgw.h +2 -0
index 4b53cb5..d886a35 100644
@@ -6,6 +6,8 @@
 
 #include <stdbool.h>
 
+//#define ENABLE_STATS
+
 #define NB_SOCKETS 4
 
 #define MAX_PKT_BURST 32
examples/ipsec-secgw/ipsec.h +22 -0
index 1e642d1..8519eab 100644
@@ -46,6 +46,17 @@
 
 #define IP6_VERSION (6)
 
+#ifdef ENABLE_STATS
+struct ipsec_core_statistics {
+	uint64_t tx;
+	uint64_t rx;
+	uint64_t dropped;
+	uint64_t burst_rx;
+} __rte_cache_aligned;
+
+struct ipsec_core_statistics core_statistics[RTE_MAX_ETHPORTS];
+#endif /* ENABLE_STATS */
+
 struct rte_crypto_xform;
 struct ipsec_xform;
 struct rte_mbuf;
@@ -416,4 +427,15 @@ check_flow_params(uint16_t fdir_portid, uint8_t fdir_qid);
 int
 create_ipsec_esp_flow(struct ipsec_sa *sa);
 
+static inline void
+core_stats_update_drop(int n)
+{
+#ifdef ENABLE_STATS
+	int lcore_id = rte_lcore_id();
+	core_statistics[lcore_id].dropped += n;
+#else
+	RTE_SET_USED(n);
+#endif /* ENABLE_STATS */
+}
+
 #endif /* __IPSEC_H__ */
examples/ipsec-secgw/ipsec_process.c +5 -0
index bb2f2b8..05cb3ad 100644
@@ -24,6 +24,11 @@ free_pkts(struct rte_mbuf *mb[], uint32_t n)
 {
 	uint32_t i;
 
+#ifdef ENABLE_STATS
+	int lcore_id = rte_lcore_id();
+	core_statistics[lcore_id].dropped += n;
+#endif /* ENABLE_STATS */
+
 	for (i = 0; i != n; i++)
 		rte_pktmbuf_free(mb[i]);
 }
Signature · 1 line
2.7.4
#1 lore raw

PATCH v2 1/1 examples/ipsec-secgw: add per core packet stats

Adding per core packet handling stats to analyze traffic distribution when multiple cores are engaged.

Since aggregating the packet stats across cores would affect performance, keeping the feature disabled using compile time flags.

---

v2:
* Added lookup failure cases to drop count

 examples/ipsec-secgw/ipsec-secgw.c   | 118 +++++++++++++++++++++++++++++++++--
 examples/ipsec-secgw/ipsec-secgw.h   |   2 +
 examples/ipsec-secgw/ipsec.c         |  13 +++-
 examples/ipsec-secgw/ipsec.h         |  22 +++++++
 examples/ipsec-secgw/ipsec_process.c |   5 ++
 5 files changed, 154 insertions(+), 6 deletions(-)
examples/ipsec-secgw/ipsec-secgw.c +113 -5
index 6d02341..db92ddc 100644
@@ -288,6 +288,61 @@ adjust_ipv6_pktlen(struct rte_mbuf *m, const struct rte_ipv6_hdr *iph,
 	}
 }
 
+#ifdef ENABLE_STATS
+static uint64_t timer_period = 10; /* default period is 10 seconds */
+
+/* Print out statistics on packet distribution */
+static void
+print_stats(void)
+{
+	uint64_t total_packets_dropped, total_packets_tx, total_packets_rx;
+	unsigned int coreid;
+	float burst_percent;
+
+	total_packets_dropped = 0;
+	total_packets_tx = 0;
+	total_packets_rx = 0;
+
+	const char clr[] = { 27, '[', '2', 'J', '\0' };
+	const char topLeft[] = { 27, '[', '1', ';', '1', 'H', '\0' };
+
+	/* Clear screen and move to top left */
+	printf("%s%s", clr, topLeft);
+
+	printf("\nCore statistics ====================================");
+
+	for (coreid = 0; coreid < RTE_MAX_LCORE; coreid++) {
+		/* skip disabled cores */
+		if (rte_lcore_is_enabled(coreid) == 0)
+			continue;
+		burst_percent = (float)(core_statistics[coreid].burst_rx * 100)/
+					core_statistics[coreid].rx;
+		printf("\nStatistics for core %u ------------------------------"
+			   "\nPackets received: %20"PRIu64
+			   "\nPackets sent: %24"PRIu64
+			   "\nPackets dropped: %21"PRIu64
+			   "\nBurst percent: %23.2f",
+			   coreid,
+			   core_statistics[coreid].rx,
+			   core_statistics[coreid].tx,
+			   core_statistics[coreid].dropped,
+			   burst_percent);
+
+		total_packets_dropped += core_statistics[coreid].dropped;
+		total_packets_tx += core_statistics[coreid].tx;
+		total_packets_rx += core_statistics[coreid].rx;
+	}
+	printf("\nAggregate statistics ==============================="
+		   "\nTotal packets received: %14"PRIu64
+		   "\nTotal packets sent: %18"PRIu64
+		   "\nTotal packets dropped: %15"PRIu64,
+		   total_packets_rx,
+		   total_packets_tx,
+		   total_packets_dropped);
+	printf("\n====================================================\n");
+}
+#endif /* ENABLE_STATS */
+
 static inline void
 prepare_one_packet(struct rte_mbuf *pkt, struct ipsec_traffic *t)
 {
@@ -333,6 +388,7 @@ prepare_one_packet(struct rte_mbuf *pkt, struct ipsec_traffic *t)
 
 		/* drop packet when IPv6 header exceeds first segment length */
 		if (unlikely(l3len > pkt->data_len)) {
+			core_stats_update_drop(1);
 			rte_pktmbuf_free(pkt);
 			return;
 		}
@@ -350,6 +406,7 @@ prepare_one_packet(struct rte_mbuf *pkt, struct ipsec_traffic *t)
 		/* Unknown/Unsupported type, drop the packet */
 		RTE_LOG(ERR, IPSEC, "Unsupported packet type 0x%x\n",
 			rte_be_to_cpu_16(eth->ether_type));
+		core_stats_update_drop(1);
 		rte_pktmbuf_free(pkt);
 		return;
 	}
@@ -471,6 +528,11 @@ send_burst(struct lcore_conf *qconf, uint16_t n, uint16_t port)
 	int32_t ret;
 	uint16_t queueid;
 
+#ifdef ENABLE_STATS
+	int lcore_id = rte_lcore_id();
+	core_statistics[lcore_id].tx += n;
+#endif /* ENABLE_STATS */
+
 	queueid = qconf->tx_queue_id[port];
 	m_table = (struct rte_mbuf **)qconf->tx_mbufs[port].m_table;
 
@@ -478,6 +540,9 @@ send_burst(struct lcore_conf *qconf, uint16_t n, uint16_t port)
 
 	ret = rte_eth_tx_burst(port, queueid, m_table, n);
 	if (unlikely(ret < n)) {
+#ifdef ENABLE_STATS
+		core_statistics[lcore_id].dropped += n-ret;
+#endif /* ENABLE_STATS */
 		do {
 			rte_pktmbuf_free(m_table[ret]);
 		} while (++ret < n);
@@ -525,6 +590,7 @@ send_fragment_packet(struct lcore_conf *qconf, struct rte_mbuf *m,
 			"error code: %d\n",
 			__func__, m->pkt_len, rte_errno);
 
+	core_stats_update_drop(1);
 	rte_pktmbuf_free(m);
 	return len;
 }
@@ -549,8 +615,10 @@ send_single_packet(struct rte_mbuf *m, uint16_t port, uint8_t proto)
 	/* need to fragment the packet */
 	} else if (frag_tbl_sz > 0)
 		len = send_fragment_packet(qconf, m, port, proto);
-	else
+	else {
+		core_stats_update_drop(1);
 		rte_pktmbuf_free(m);
+	}
 
 	/* enough pkts to be sent */
 	if (unlikely(len == MAX_PKT_BURST)) {
@@ -584,18 +652,21 @@ inbound_sp_sa(struct sp_ctx *sp, struct sa_ctx *sa, struct traffic_type *ip,
 			continue;
 		}
 		if (res == DISCARD) {
+			core_stats_update_drop(1);
 			rte_pktmbuf_free(m);
 			continue;
 		}
 
 		/* Only check SPI match for processed IPSec packets */
 		if (i < lim && ((m->ol_flags & PKT_RX_SEC_OFFLOAD) == 0)) {
+			core_stats_update_drop(1);
 			rte_pktmbuf_free(m);
 			continue;
 		}
 
 		sa_idx = res - 1;
 		if (!inbound_sa_check(sa, m, sa_idx)) {
+			core_stats_update_drop(1);
 			rte_pktmbuf_free(m);
 			continue;
 		}
@@ -630,8 +701,10 @@ split46_traffic(struct ipsec_traffic *trf, struct rte_mbuf *mb[], uint32_t num)
 					uint8_t *,
 					offsetof(struct ip6_hdr, ip6_nxt));
 			n6++;
-		} else
+		} else {
+			core_stats_update_drop(1);
 			rte_pktmbuf_free(m);
+		}
 	}
 
 	trf->ip4.num = n4;
@@ -682,11 +755,12 @@ outbound_sp(struct sp_ctx *sp, struct traffic_type *ip,
 	for (i = 0; i < ip->num; i++) {
 		m = ip->pkts[i];
 		sa_idx = ip->res[i] - 1;
-		if (ip->res[i] == DISCARD)
+		if (ip->res[i] == DISCARD) {
+			core_stats_update_drop(1);
 			rte_pktmbuf_free(m);
-		else if (ip->res[i] == BYPASS)
+		} else if (ip->res[i] == BYPASS) {
 			ip->pkts[j++] = m;
-		else {
+		} else {
 			ipsec->res[ipsec->num] = sa_idx;
 			ipsec->pkts[ipsec->num++] = m;
 		}
@@ -705,6 +779,8 @@ process_pkts_outbound(struct ipsec_ctx *ipsec_ctx,
 	for (i = 0; i < traffic->ipsec.num; i++)
 		rte_pktmbuf_free(traffic->ipsec.pkts[i]);
 
+	core_stats_update_drop(traffic->ipsec.num);
+
 	traffic->ipsec.num = 0;
 
 	outbound_sp(ipsec_ctx->sp4_ctx, &traffic->ip4, &traffic->ipsec);
@@ -745,12 +821,14 @@ process_pkts_inbound_nosp(struct ipsec_ctx *ipsec_ctx,
 	/* Drop any IPv4 traffic from unprotected ports */
 	for (i = 0; i < traffic->ip4.num; i++)
 		rte_pktmbuf_free(traffic->ip4.pkts[i]);
+	core_stats_update_drop(traffic->ip4.num);
 
 	traffic->ip4.num = 0;
 
 	/* Drop any IPv6 traffic from unprotected ports */
 	for (i = 0; i < traffic->ip6.num; i++)
 		rte_pktmbuf_free(traffic->ip6.pkts[i]);
+	core_stats_update_drop(traffic->ip6.num);
 
 	traffic->ip6.num = 0;
 
@@ -788,6 +866,7 @@ process_pkts_outbound_nosp(struct ipsec_ctx *ipsec_ctx,
 	/* Drop any IPsec traffic from protected ports */
 	for (i = 0; i < traffic->ipsec.num; i++)
 		rte_pktmbuf_free(traffic->ipsec.pkts[i]);
+	core_stats_update_drop(traffic->ipsec.num);
 
 	n = 0;
 
@@ -901,6 +980,7 @@ route4_pkts(struct rt_ctx *rt_ctx, struct rte_mbuf *pkts[], uint8_t nb_pkts)
 		}
 
 		if ((pkt_hop & RTE_LPM_LOOKUP_SUCCESS) == 0) {
+			core_stats_update_drop(1);
 			rte_pktmbuf_free(pkts[i]);
 			continue;
 		}
@@ -953,6 +1033,7 @@ route6_pkts(struct rt_ctx *rt_ctx, struct rte_mbuf *pkts[], uint8_t nb_pkts)
 		}
 
 		if (pkt_hop == -1) {
+			core_stats_update_drop(1);
 			rte_pktmbuf_free(pkts[i]);
 			continue;
 		}
@@ -1099,6 +1180,9 @@ ipsec_poll_mode_worker(void)
 	const uint64_t drain_tsc = (rte_get_tsc_hz() + US_PER_S - 1)
 			/ US_PER_S * BURST_TX_DRAIN_US;
 	struct lcore_rx_queue *rxql;
+#ifdef ENABLE_STATS
+	uint64_t timer_tsc = 0;
+#endif /* ENABLE_STATS */
 
 	prev_tsc = 0;
 	lcore_id = rte_lcore_id();
@@ -1159,6 +1243,19 @@ ipsec_poll_mode_worker(void)
 			drain_tx_buffers(qconf);
 			drain_crypto_buffers(qconf);
 			prev_tsc = cur_tsc;
+#ifdef ENABLE_STATS
+			if (lcore_id == rte_get_master_lcore()) {
+				/* advance the timer */
+				timer_tsc += diff_tsc;
+
+				/* if timer has reached its timeout */
+				if (unlikely(timer_tsc >= timer_period)) {
+					print_stats();
+					/* reset the timer */
+					timer_tsc = 0;
+				}
+			}
+#endif /* ENABLE_STATS */
 		}
 
 		for (i = 0; i < qconf->nb_rx_queue; ++i) {
@@ -1169,6 +1266,12 @@ ipsec_poll_mode_worker(void)
 			nb_rx = rte_eth_rx_burst(portid, queueid,
 					pkts, MAX_PKT_BURST);
 
+#ifdef ENABLE_STATS
+			core_statistics[lcore_id].rx += nb_rx;
+			if (nb_rx == MAX_PKT_BURST)
+				core_statistics[lcore_id].burst_rx += nb_rx;
+#endif /* ENABLE_STATS */
+
 			if (nb_rx > 0)
 				process_pkts(qconf, pkts, nb_rx, portid);
 
@@ -2747,6 +2850,11 @@ main(int32_t argc, char **argv)
 	signal(SIGINT, signal_handler);
 	signal(SIGTERM, signal_handler);
 
+#ifdef ENABLE_STATS
+	/* convert to number of cycles */
+	timer_period *= rte_get_timer_hz();
+#endif /* ENABLE_STATS */
+
 	/* initialize event helper configuration */
 	eh_conf = eh_conf_init();
 	if (eh_conf == NULL)
examples/ipsec-secgw/ipsec-secgw.h +2 -0
index 4b53cb5..d886a35 100644
@@ -6,6 +6,8 @@
 
 #include <stdbool.h>
 
+//#define ENABLE_STATS
+
 #define NB_SOCKETS 4
 
 #define MAX_PKT_BURST 32
examples/ipsec-secgw/ipsec.c +12 -1
index bf88d80..dcb9312 100644
@@ -499,8 +499,10 @@ enqueue_cop_burst(struct cdev_qp *cqp)
 			" enqueued %u crypto ops out of %u\n",
 			cqp->id, cqp->qp, ret, len);
 			/* drop packets that we fail to enqueue */
-			for (i = ret; i < len; i++)
+			for (i = ret; i < len; i++) {
+				core_stats_update_drop(1);
 				rte_pktmbuf_free(cqp->buf[i]->sym->m_src);
+			}
 	}
 	cqp->in_flight += ret;
 	cqp->len = 0;
@@ -528,6 +530,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 
 	for (i = 0; i < nb_pkts; i++) {
 		if (unlikely(sas[i] == NULL)) {
+			core_stats_update_drop(1);
 			rte_pktmbuf_free(pkts[i]);
 			continue;
 		}
@@ -549,6 +552,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 
 			if ((unlikely(ips->security.ses == NULL)) &&
 				create_lookaside_session(ipsec_ctx, sa, ips)) {
+				core_stats_update_drop(1);
 				rte_pktmbuf_free(pkts[i]);
 				continue;
 			}
@@ -563,6 +567,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 		case RTE_SECURITY_ACTION_TYPE_CPU_CRYPTO:
 			RTE_LOG(ERR, IPSEC, "CPU crypto is not supported by the"
 					" legacy mode.");
+			core_stats_update_drop(1);
 			rte_pktmbuf_free(pkts[i]);
 			continue;
 
@@ -575,6 +580,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 
 			if ((unlikely(ips->crypto.ses == NULL)) &&
 				create_lookaside_session(ipsec_ctx, sa, ips)) {
+				core_stats_update_drop(1);
 				rte_pktmbuf_free(pkts[i]);
 				continue;
 			}
@@ -584,6 +590,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 
 			ret = xform_func(pkts[i], sa, &priv->cop);
 			if (unlikely(ret)) {
+				core_stats_update_drop(1);
 				rte_pktmbuf_free(pkts[i]);
 				continue;
 			}
@@ -608,6 +615,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 
 			ret = xform_func(pkts[i], sa, &priv->cop);
 			if (unlikely(ret)) {
+				core_stats_update_drop(1);
 				rte_pktmbuf_free(pkts[i]);
 				continue;
 			}
@@ -643,6 +651,7 @@ ipsec_inline_dequeue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 		sa = priv->sa;
 		ret = xform_func(pkt, sa, &priv->cop);
 		if (unlikely(ret)) {
+			core_stats_update_drop(1);
 			rte_pktmbuf_free(pkt);
 			continue;
 		}
@@ -690,12 +699,14 @@ ipsec_dequeue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 				RTE_SECURITY_ACTION_TYPE_NONE) {
 				ret = xform_func(pkt, sa, cops[j]);
 				if (unlikely(ret)) {
+					core_stats_update_drop(1);
 					rte_pktmbuf_free(pkt);
 					continue;
 				}
 			} else if (ipsec_get_action_type(sa) ==
 				RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) {
 				if (cops[j]->status) {
+					core_stats_update_drop(1);
 					rte_pktmbuf_free(pkt);
 					continue;
 				}
examples/ipsec-secgw/ipsec.h +22 -0
index 1e642d1..8519eab 100644
@@ -46,6 +46,17 @@
 
 #define IP6_VERSION (6)
 
+#ifdef ENABLE_STATS
+struct ipsec_core_statistics {
+	uint64_t tx;
+	uint64_t rx;
+	uint64_t dropped;
+	uint64_t burst_rx;
+} __rte_cache_aligned;
+
+struct ipsec_core_statistics core_statistics[RTE_MAX_ETHPORTS];
+#endif /* ENABLE_STATS */
+
 struct rte_crypto_xform;
 struct ipsec_xform;
 struct rte_mbuf;
@@ -416,4 +427,15 @@ check_flow_params(uint16_t fdir_portid, uint8_t fdir_qid);
 int
 create_ipsec_esp_flow(struct ipsec_sa *sa);
 
+static inline void
+core_stats_update_drop(int n)
+{
+#ifdef ENABLE_STATS
+	int lcore_id = rte_lcore_id();
+	core_statistics[lcore_id].dropped += n;
+#else
+	RTE_SET_USED(n);
+#endif /* ENABLE_STATS */
+}
+
 #endif /* __IPSEC_H__ */
examples/ipsec-secgw/ipsec_process.c +5 -0
index bb2f2b8..05cb3ad 100644
@@ -24,6 +24,11 @@ free_pkts(struct rte_mbuf *mb[], uint32_t n)
 {
 	uint32_t i;
 
+#ifdef ENABLE_STATS
+	int lcore_id = rte_lcore_id();
+	core_statistics[lcore_id].dropped += n;
+#endif /* ENABLE_STATS */
+
 	for (i = 0; i != n; i++)
 		rte_pktmbuf_free(mb[i]);
 }
Signature · 1 line
2.7.4
#2 lore raw
Quoted 25 lines

Adding per core packet handling stats to analyze traffic distribution when multiple cores are engaged.

Since aggregating the packet stats across cores would affect performance, keeping the feature disabled using compile time flags.

---

v2:
* Added lookup failure cases to drop count

 examples/ipsec-secgw/ipsec-secgw.c   | 118 +++++++++++++++++++++++++++++++++--
 examples/ipsec-secgw/ipsec-secgw.h   |   2 +
 examples/ipsec-secgw/ipsec.c         |  13 +++-
 examples/ipsec-secgw/ipsec.h         |  22 +++++++
 examples/ipsec-secgw/ipsec_process.c |   5 ++
 5 files changed, 154 insertions(+), 6 deletions(-)
examples/ipsec-secgw/ipsec-secgw.c +0 -0
index 6d02341..db92ddc 100644

examples/ipsec-secgw/ipsec-secgw.c adjust_ipv6_pktlen(struct rte_mbuf *m, const struct rte_ipv6_hdr *iph, · line 292

Quoted 5 lines
examples/ipsec-secgw/ipsec-secgw.c +2 -0
@@ -288,6 +288,61 @@ adjust_ipv6_pktlen(struct rte_mbuf *m, const struct rte_ipv6_hdr *iph,
 	}
 }
 
+#ifdef ENABLE_STATS
+static uint64_t timer_period = 10; /* default period is 10 seconds */

I think it is better to add user ability to control stats period. Either runtime-option, or just compile time:
replace ENABLE_STATS with STATS_PERIOD (0 would mean stats disabled).

Quoted 8 lines
examples/ipsec-secgw/ipsec-secgw.c +8 -0
+
+/* Print out statistics on packet distribution */
+static void
+print_stats(void)
+{
+	uint64_t total_packets_dropped, total_packets_tx, total_packets_rx;
+	unsigned int coreid;
+	float burst_percent;

examples/ipsec-secgw/ipsec-secgw.c adjust_ipv6_pktlen(struct rte_mbuf *m, const struct rte_ipv6_hdr *iph, · line 310

Quoted 10 lines
examples/ipsec-secgw/ipsec-secgw.c +10 -0
+
+	total_packets_dropped = 0;
+	total_packets_tx = 0;
+	total_packets_rx = 0;
+
+	const char clr[] = { 27, '[', '2', 'J', '\0' };
+	const char topLeft[] = { 27, '[', '1', ';', '1', 'H', '\0' };
+
+	/* Clear screen and move to top left */
+	printf("%s%s", clr, topLeft);

Is that really needed?

examples/ipsec-secgw/ipsec-secgw.c adjust_ipv6_pktlen(struct rte_mbuf *m, const struct rte_ipv6_hdr *iph, · line 319

Quoted 9 lines
examples/ipsec-secgw/ipsec-secgw.c +9 -0
+
+	printf("\nCore statistics ====================================");
+
+	for (coreid = 0; coreid < RTE_MAX_LCORE; coreid++) {
+		/* skip disabled cores */
+		if (rte_lcore_is_enabled(coreid) == 0)
+			continue;
+		burst_percent = (float)(core_statistics[coreid].burst_rx * 100)/
+					core_statistics[coreid].rx;

Would float be always enough here? Might better long double?

Quoted 46 lines
examples/ipsec-secgw/ipsec-secgw.c +28 -0
+		printf("\nStatistics for core %u ------------------------------"
+			   "\nPackets received: %20"PRIu64
+			   "\nPackets sent: %24"PRIu64
+			   "\nPackets dropped: %21"PRIu64
+			   "\nBurst percent: %23.2f",
+			   coreid,
+			   core_statistics[coreid].rx,
+			   core_statistics[coreid].tx,
+			   core_statistics[coreid].dropped,
+			   burst_percent);
+
+		total_packets_dropped += core_statistics[coreid].dropped;
+		total_packets_tx += core_statistics[coreid].tx;
+		total_packets_rx += core_statistics[coreid].rx;
+	}
+	printf("\nAggregate statistics ==============================="
+		   "\nTotal packets received: %14"PRIu64
+		   "\nTotal packets sent: %18"PRIu64
+		   "\nTotal packets dropped: %15"PRIu64,
+		   total_packets_rx,
+		   total_packets_tx,
+		   total_packets_dropped);
+	printf("\n====================================================\n");
+}
+#endif /* ENABLE_STATS */
+
 static inline void
 prepare_one_packet(struct rte_mbuf *pkt, struct ipsec_traffic *t)
 {
@@ -333,6 +388,7 @@ prepare_one_packet(struct rte_mbuf *pkt, struct ipsec_traffic *t)
 
 		/* drop packet when IPv6 header exceeds first segment length */
 		if (unlikely(l3len > pkt->data_len)) {
+			core_stats_update_drop(1);
 			rte_pktmbuf_free(pkt);
 			return;
 		}
@@ -350,6 +406,7 @@ prepare_one_packet(struct rte_mbuf *pkt, struct ipsec_traffic *t)
 		/* Unknown/Unsupported type, drop the packet */
 		RTE_LOG(ERR, IPSEC, "Unsupported packet type 0x%x\n",
 			rte_be_to_cpu_16(eth->ether_type));
+		core_stats_update_drop(1);
 		rte_pktmbuf_free(pkt);
 		return;
 	}

examples/ipsec-secgw/ipsec-secgw.c send_burst(struct lcore_conf *qconf, uint16_t n, uint16_t port) · line 534

Quoted 7 lines
examples/ipsec-secgw/ipsec-secgw.c +4 -0
@@ -471,6 +528,11 @@ send_burst(struct lcore_conf *qconf, uint16_t n, uint16_t port)
 	int32_t ret;
 	uint16_t queueid;
 
+#ifdef ENABLE_STATS
+	int lcore_id = rte_lcore_id();
+	core_statistics[lcore_id].tx += n;
+#endif /* ENABLE_STATS */

Instead of polluting genric code with ifdefs, why not
to introduce 2 new functions: core_stats_update_rx(), core_stats_update_tx(), as you did for core_stats_drop()?

examples/ipsec-secgw/ipsec-secgw.c send_burst(struct lcore_conf *qconf, uint16_t n, uint16_t port) · line 545

Quoted 11 lines
examples/ipsec-secgw/ipsec-secgw.c +4 -0
+
 	queueid = qconf->tx_queue_id[port];
 	m_table = (struct rte_mbuf **)qconf->tx_mbufs[port].m_table;
 
@@ -478,6 +540,9 @@ send_burst(struct lcore_conf *qconf, uint16_t n, uint16_t port)
 
 	ret = rte_eth_tx_burst(port, queueid, m_table, n);
 	if (unlikely(ret < n)) {
+#ifdef ENABLE_STATS
+		core_statistics[lcore_id].dropped += n-ret;
+#endif /* ENABLE_STATS */

You have core_stats_update_drop() for that - use it.

Quoted 12 lines
examples/ipsec-secgw/ipsec-secgw.c +1 -0
 		do {
 			rte_pktmbuf_free(m_table[ret]);
 		} while (++ret < n);
@@ -525,6 +590,7 @@ send_fragment_packet(struct lcore_conf *qconf, struct rte_mbuf *m,
 			"error code: %d\n",
 			__func__, m->pkt_len, rte_errno);
 
+	core_stats_update_drop(1);
 	rte_pktmbuf_free(m);
 	return len;
 }

examples/ipsec-secgw/ipsec-secgw.c send_single_packet(struct rte_mbuf *m, uint16_t port, uint8_t proto) · line 620

Quoted 7 lines
examples/ipsec-secgw/ipsec-secgw.c +2 -1
@@ -549,8 +615,10 @@ send_single_packet(struct rte_mbuf *m, uint16_t port, uint8_t proto)
 	/* need to fragment the packet */
 	} else if (frag_tbl_sz > 0)
 		len = send_fragment_packet(qconf, m, port, proto);
-	else
+	else {
+		core_stats_update_drop(1);
 		rte_pktmbuf_free(m);

It looks like a lot of such places...
Would it be worth to unite core_stats_update_drop() and rte_pktmbuf_free(m) Into some inline function: ipsec_secgw_packet_drop(struct rte_mbuf *m[], uint32_t n) and use it all over such places.

Quoted 39 lines
examples/ipsec-secgw/ipsec-secgw.c +7 -1
+	}
 
 	/* enough pkts to be sent */
 	if (unlikely(len == MAX_PKT_BURST)) {
@@ -584,18 +652,21 @@ inbound_sp_sa(struct sp_ctx *sp, struct sa_ctx *sa, struct traffic_type *ip,
 			continue;
 		}
 		if (res == DISCARD) {
+			core_stats_update_drop(1);
 			rte_pktmbuf_free(m);
 			continue;
 		}
 
 		/* Only check SPI match for processed IPSec packets */
 		if (i < lim && ((m->ol_flags & PKT_RX_SEC_OFFLOAD) == 0)) {
+			core_stats_update_drop(1);
 			rte_pktmbuf_free(m);
 			continue;
 		}
 
 		sa_idx = res - 1;
 		if (!inbound_sa_check(sa, m, sa_idx)) {
+			core_stats_update_drop(1);
 			rte_pktmbuf_free(m);
 			continue;
 		}
@@ -630,8 +701,10 @@ split46_traffic(struct ipsec_traffic *trf, struct rte_mbuf *mb[], uint32_t num)
 					uint8_t *,
 					offsetof(struct ip6_hdr, ip6_nxt));
 			n6++;
-		} else
+		} else {
+			core_stats_update_drop(1);
 			rte_pktmbuf_free(m);
+		}
 	}
 
 	trf->ip4.num = n4;

examples/ipsec-secgw/ipsec-secgw.c outbound_sp(struct sp_ctx *sp, struct traffic_type *ip, · line 761

Quoted 9 lines
examples/ipsec-secgw/ipsec-secgw.c +3 -2
@@ -682,11 +755,12 @@ outbound_sp(struct sp_ctx *sp, struct traffic_type *ip,
 	for (i = 0; i < ip->num; i++) {
 		m = ip->pkts[i];
 		sa_idx = ip->res[i] - 1;
-		if (ip->res[i] == DISCARD)
+		if (ip->res[i] == DISCARD) {
+			core_stats_update_drop(1);
 			rte_pktmbuf_free(m);
-		else if (ip->res[i] == BYPASS)
+		} else if (ip->res[i] == BYPASS) {

Looks unnecessary.

Quoted 55 lines
examples/ipsec-secgw/ipsec-secgw.c +8 -1
 			ip->pkts[j++] = m;
-		else {
+		} else {
 			ipsec->res[ipsec->num] = sa_idx;
 			ipsec->pkts[ipsec->num++] = m;
 		}
@@ -705,6 +779,8 @@ process_pkts_outbound(struct ipsec_ctx *ipsec_ctx,
 	for (i = 0; i < traffic->ipsec.num; i++)
 		rte_pktmbuf_free(traffic->ipsec.pkts[i]);
 
+	core_stats_update_drop(traffic->ipsec.num);
+
 	traffic->ipsec.num = 0;
 
 	outbound_sp(ipsec_ctx->sp4_ctx, &traffic->ip4, &traffic->ipsec);
@@ -745,12 +821,14 @@ process_pkts_inbound_nosp(struct ipsec_ctx *ipsec_ctx,
 	/* Drop any IPv4 traffic from unprotected ports */
 	for (i = 0; i < traffic->ip4.num; i++)
 		rte_pktmbuf_free(traffic->ip4.pkts[i]);
+	core_stats_update_drop(traffic->ip4.num);
 
 	traffic->ip4.num = 0;
 
 	/* Drop any IPv6 traffic from unprotected ports */
 	for (i = 0; i < traffic->ip6.num; i++)
 		rte_pktmbuf_free(traffic->ip6.pkts[i]);
+	core_stats_update_drop(traffic->ip6.num);
 
 	traffic->ip6.num = 0;
 
@@ -788,6 +866,7 @@ process_pkts_outbound_nosp(struct ipsec_ctx *ipsec_ctx,
 	/* Drop any IPsec traffic from protected ports */
 	for (i = 0; i < traffic->ipsec.num; i++)
 		rte_pktmbuf_free(traffic->ipsec.pkts[i]);
+	core_stats_update_drop(traffic->ipsec.num);
 
 	n = 0;
 
@@ -901,6 +980,7 @@ route4_pkts(struct rt_ctx *rt_ctx, struct rte_mbuf *pkts[], uint8_t nb_pkts)
 		}
 
 		if ((pkt_hop & RTE_LPM_LOOKUP_SUCCESS) == 0) {
+			core_stats_update_drop(1);
 			rte_pktmbuf_free(pkts[i]);
 			continue;
 		}
@@ -953,6 +1033,7 @@ route6_pkts(struct rt_ctx *rt_ctx, struct rte_mbuf *pkts[], uint8_t nb_pkts)
 		}
 
 		if (pkt_hop == -1) {
+			core_stats_update_drop(1);
 			rte_pktmbuf_free(pkts[i]);
 			continue;
 		}

examples/ipsec-secgw/ipsec-secgw.c ipsec_poll_mode_worker(void) · line 1185

Quoted 6 lines
examples/ipsec-secgw/ipsec-secgw.c +3 -0
@@ -1099,6 +1180,9 @@ ipsec_poll_mode_worker(void)
 	const uint64_t drain_tsc = (rte_get_tsc_hz() + US_PER_S - 1)
 			/ US_PER_S * BURST_TX_DRAIN_US;
 	struct lcore_rx_queue *rxql;
+#ifdef ENABLE_STATS
+	uint64_t timer_tsc = 0;
+#endif /* ENABLE_STATS */

Probably better just RTE_SET_USED(timer_tsc);

Quoted 10 lines
examples/ipsec-secgw/ipsec-secgw.c +3 -0
 	prev_tsc = 0;
 	lcore_id = rte_lcore_id();
@@ -1159,6 +1243,19 @@ ipsec_poll_mode_worker(void)
 			drain_tx_buffers(qconf);
 			drain_crypto_buffers(qconf);
 			prev_tsc = cur_tsc;
+#ifdef ENABLE_STATS
+			if (lcore_id == rte_get_master_lcore()) {
+				/* advance the timer */

examples/ipsec-secgw/ipsec-secgw.c ipsec_poll_mode_worker(void) · line 1258

Quoted 10 lines
examples/ipsec-secgw/ipsec-secgw.c +10 -0
+				timer_tsc += diff_tsc;
+
+				/* if timer has reached its timeout */
+				if (unlikely(timer_tsc >= timer_period)) {
+					print_stats();
+					/* reset the timer */
+					timer_tsc = 0;
+				}
+			}
+#endif /* ENABLE_STATS */

Why to do stats collection/display inside data-path?
Why not use rte_timer/rte_alarm and make it happen in control thread? Another option - make stats printing at some signal to the process. In that case we don't need to bother with time period at all - it will be user to decide.
Again if we remove that print_stats() from data-path it might become cheap enough to always collect it, and we will not need ENABLE_STATS macro at all.

examples/ipsec-secgw/ipsec-secgw.c ipsec_poll_mode_worker(void) · line 1274

Quoted 13 lines
examples/ipsec-secgw/ipsec-secgw.c +6 -0
 		}
 
 		for (i = 0; i < qconf->nb_rx_queue; ++i) {
@@ -1169,6 +1266,12 @@ ipsec_poll_mode_worker(void)
 			nb_rx = rte_eth_rx_burst(portid, queueid,
 					pkts, MAX_PKT_BURST);
 
+#ifdef ENABLE_STATS
+			core_statistics[lcore_id].rx += nb_rx;
+			if (nb_rx == MAX_PKT_BURST)
+				core_statistics[lcore_id].burst_rx += nb_rx;
+#endif /* ENABLE_STATS */
+

Same for above for TX: no need to pollute the code with ifdefs. Better to introduce new function: stats_update_rx() or so.

Quoted 20 lines
examples/ipsec-secgw/ipsec-secgw.c +5 -0
 			if (nb_rx > 0)
 				process_pkts(qconf, pkts, nb_rx, portid);
 
@@ -2747,6 +2850,11 @@ main(int32_t argc, char **argv)
 	signal(SIGINT, signal_handler);
 	signal(SIGTERM, signal_handler);
 
+#ifdef ENABLE_STATS
+	/* convert to number of cycles */
+	timer_period *= rte_get_timer_hz();
+#endif /* ENABLE_STATS */
+
 	/* initialize event helper configuration */
 	eh_conf = eh_conf_init();
 	if (eh_conf == NULL)
examples/ipsec-secgw/ipsec-secgw.h +0 -0
index 4b53cb5..d886a35 100644

examples/ipsec-secgw/ipsec-secgw.h line 10

Quoted 5 lines
examples/ipsec-secgw/ipsec-secgw.h +2 -0
@@ -6,6 +6,8 @@
 
 #include <stdbool.h>
 
+//#define ENABLE_STATS
+

Should be removed I think.

Quoted 95 lines
examples/ipsec-secgw/ipsec-secgw.h +0 -0
 #define NB_SOCKETS 4
 
 #define MAX_PKT_BURST 32
examples/ipsec-secgw/ipsec.c +12 -1
index bf88d80..dcb9312 100644
@@ -499,8 +499,10 @@ enqueue_cop_burst(struct cdev_qp *cqp)
 			" enqueued %u crypto ops out of %u\n",
 			cqp->id, cqp->qp, ret, len);
 			/* drop packets that we fail to enqueue */
-			for (i = ret; i < len; i++)
+			for (i = ret; i < len; i++) {
+				core_stats_update_drop(1);
 				rte_pktmbuf_free(cqp->buf[i]->sym->m_src);
+			}
 	}
 	cqp->in_flight += ret;
 	cqp->len = 0;
@@ -528,6 +530,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 
 	for (i = 0; i < nb_pkts; i++) {
 		if (unlikely(sas[i] == NULL)) {
+			core_stats_update_drop(1);
 			rte_pktmbuf_free(pkts[i]);
 			continue;
 		}
@@ -549,6 +552,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 
 			if ((unlikely(ips->security.ses == NULL)) &&
 				create_lookaside_session(ipsec_ctx, sa, ips)) {
+				core_stats_update_drop(1);
 				rte_pktmbuf_free(pkts[i]);
 				continue;
 			}
@@ -563,6 +567,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 		case RTE_SECURITY_ACTION_TYPE_CPU_CRYPTO:
 			RTE_LOG(ERR, IPSEC, "CPU crypto is not supported by the"
 					" legacy mode.");
+			core_stats_update_drop(1);
 			rte_pktmbuf_free(pkts[i]);
 			continue;
 
@@ -575,6 +580,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 
 			if ((unlikely(ips->crypto.ses == NULL)) &&
 				create_lookaside_session(ipsec_ctx, sa, ips)) {
+				core_stats_update_drop(1);
 				rte_pktmbuf_free(pkts[i]);
 				continue;
 			}
@@ -584,6 +590,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 
 			ret = xform_func(pkts[i], sa, &priv->cop);
 			if (unlikely(ret)) {
+				core_stats_update_drop(1);
 				rte_pktmbuf_free(pkts[i]);
 				continue;
 			}
@@ -608,6 +615,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 
 			ret = xform_func(pkts[i], sa, &priv->cop);
 			if (unlikely(ret)) {
+				core_stats_update_drop(1);
 				rte_pktmbuf_free(pkts[i]);
 				continue;
 			}
@@ -643,6 +651,7 @@ ipsec_inline_dequeue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 		sa = priv->sa;
 		ret = xform_func(pkt, sa, &priv->cop);
 		if (unlikely(ret)) {
+			core_stats_update_drop(1);
 			rte_pktmbuf_free(pkt);
 			continue;
 		}
@@ -690,12 +699,14 @@ ipsec_dequeue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 				RTE_SECURITY_ACTION_TYPE_NONE) {
 				ret = xform_func(pkt, sa, cops[j]);
 				if (unlikely(ret)) {
+					core_stats_update_drop(1);
 					rte_pktmbuf_free(pkt);
 					continue;
 				}
 			} else if (ipsec_get_action_type(sa) ==
 				RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) {
 				if (cops[j]->status) {
+					core_stats_update_drop(1);
 					rte_pktmbuf_free(pkt);
 					continue;
 				}
examples/ipsec-secgw/ipsec.h +0 -0
index 1e642d1..8519eab 100644

examples/ipsec-secgw/ipsec.h line 54

Quoted 9 lines
examples/ipsec-secgw/ipsec.h +6 -0
@@ -46,6 +46,17 @@
 
 #define IP6_VERSION (6)
 
+#ifdef ENABLE_STATS
+struct ipsec_core_statistics {
+	uint64_t tx;
+	uint64_t rx;
+	uint64_t dropped;
+	uint64_t burst_rx;

A bit strange to have burst_rx and no similar counterpart for tx. BTW, do you need burst_rx?
Might be better:
nb_calls_rx, nb_calls_tx;
and then: rx/nb_calls_rx will give you average burst size.

examples/ipsec-secgw/ipsec.h line 57

Quoted 3 lines
examples/ipsec-secgw/ipsec.h +3 -0
+} __rte_cache_aligned;
+
+struct ipsec_core_statistics core_statistics[RTE_MAX_ETHPORTS];

Should be RTE_MAX_LCORES, I think.

Quoted 26 lines
examples/ipsec-secgw/ipsec.h +13 -0
+#endif /* ENABLE_STATS */
+
 struct rte_crypto_xform;
 struct ipsec_xform;
 struct rte_mbuf;
@@ -416,4 +427,15 @@ check_flow_params(uint16_t fdir_portid, uint8_t fdir_qid);
 int
 create_ipsec_esp_flow(struct ipsec_sa *sa);
 
+static inline void
+core_stats_update_drop(int n)
+{
+#ifdef ENABLE_STATS
+	int lcore_id = rte_lcore_id();
+	core_statistics[lcore_id].dropped += n;
+#else
+	RTE_SET_USED(n);
+#endif /* ENABLE_STATS */
+}
+
 #endif /* __IPSEC_H__ */
examples/ipsec-secgw/ipsec_process.c +0 -0
index bb2f2b8..05cb3ad 100644

examples/ipsec-secgw/ipsec_process.c free_pkts(struct rte_mbuf *mb[], uint32_t n) · line 31

Quoted 8 lines
examples/ipsec-secgw/ipsec_process.c +5 -0
@@ -24,6 +24,11 @@ free_pkts(struct rte_mbuf *mb[], uint32_t n)
 {
 	uint32_t i;
 
+#ifdef ENABLE_STATS
+	int lcore_id = rte_lcore_id();
+	core_statistics[lcore_id].dropped += n;
+#endif /* ENABLE_STATS */
+

Same as above - why not use stats_update_drop() here?

Quoted 5 lines
examples/ipsec-secgw/ipsec_process.c +0 -1
 	for (i = 0; i != n; i++)
 		rte_pktmbuf_free(mb[i]);
 }
--

2.7.4

#3 lore raw

Hi Konstantin,

Thanks for the review. Please see inline.

@Akhil, do you have any comments?

Thanks,
Anoob

Quoted 43 lines

-----Original Message-----
From: Ananyev, Konstantin <[email protected]>
Sent: Friday, April 24, 2020 4:44 PM
To: Anoob Joseph <[email protected]>; Akhil Goyal <[email protected]>; Nicolau, Radu <[email protected]>
Cc: Narayana Prasad Raju Athreya <[email protected]>; [email protected]
Subject: [EXT] RE: [dpdk-dev] [PATCH v2] examples/ipsec-secgw: add per core packet stats

External Email

----------------------------------------------------------------------

Quoted 6 lines

Adding per core packet handling stats to analyze traffic distribution when multiple cores are engaged.

Since aggregating the packet stats across cores would affect performance, keeping the feature disabled using compile time flags.

Quoted 7 lines

---

v2:
* Added lookup failure cases to drop count

 examples/ipsec-secgw/ipsec-secgw.c   | 118

+++++++++++++++++++++++++++++++++--

examples/ipsec-secgw/ipsec-secgw.c adjust_ipv6_pktlen(struct rte_mbuf *m, const struct

Quoted 12 lines
 examples/ipsec-secgw/ipsec-secgw.h   |   2 +
 examples/ipsec-secgw/ipsec.c         |  13 +++-
 examples/ipsec-secgw/ipsec.h         |  22 +++++++
 examples/ipsec-secgw/ipsec_process.c |   5 ++
 5 files changed, 154 insertions(+), 6 deletions(-)
examples/ipsec-secgw/ipsec-secgw.c +0 -0
index 6d02341..db92ddc 100644
@@ -288,6 +288,61 @@ adjust_ipv6_pktlen(struct rte_mbuf *m, const struct

rte_ipv6_hdr *iph,

Quoted 3 lines
examples/ipsec-secgw/ipsec-secgw.c +0 -0
 	}
 }
 
Quoted 6 lines

examples/ipsec-secgw/ipsec-secgw.c adjust_ipv6_pktlen(struct rte_mbuf *m, const struct rte_ipv6_hdr *iph, · line 292

Quoted 2 lines
examples/ipsec-secgw/ipsec-secgw.c +2 -0
+#ifdef ENABLE_STATS
+static uint64_t timer_period = 10; /* default period is 10 seconds */

I think it is better to add user ability to control stats period. Either runtime-option, or just compile time:
replace ENABLE_STATS with STATS_PERIOD (0 would mean stats disabled).

[Anoob] That's a good suggestion. Will make this change as part of v3.

Quoted 12 lines
Quoted 12 lines
examples/ipsec-secgw/ipsec-secgw.c +11 -0
+
+/* Print out statistics on packet distribution */ static void
+print_stats(void)
+{
+	uint64_t total_packets_dropped, total_packets_tx, total_packets_rx;
+	unsigned int coreid;
+	float burst_percent;
+
+	total_packets_dropped = 0;
+	total_packets_tx = 0;
+	total_packets_rx = 0;
Quoted 8 lines

examples/ipsec-secgw/ipsec-secgw.c adjust_ipv6_pktlen(struct rte_mbuf *m, const struct rte_ipv6_hdr *iph, · line 310

Quoted 6 lines
examples/ipsec-secgw/ipsec-secgw.c +6 -0
+
+	const char clr[] = { 27, '[', '2', 'J', '\0' };
+	const char topLeft[] = { 27, '[', '1', ';', '1', 'H', '\0' };
+
+	/* Clear screen and move to top left */
+	printf("%s%s", clr, topLeft);

Is that really needed?

[Anoob] I had just copied from l2fwd and worked from that. But it seems like, 'topleft' is not needed. Will remove that. Without a clearscreen, the prints would just crowd the screen. I would like to retain clearscreen.

Quoted 12 lines

examples/ipsec-secgw/ipsec-secgw.c adjust_ipv6_pktlen(struct rte_mbuf *m, const struct rte_ipv6_hdr *iph, · line 319

Quoted 10 lines
examples/ipsec-secgw/ipsec-secgw.c +9 -0
+
+	printf("\nCore statistics ====================================");
+
+	for (coreid = 0; coreid < RTE_MAX_LCORE; coreid++) {
+		/* skip disabled cores */
+		if (rte_lcore_is_enabled(coreid) == 0)
+			continue;
+		burst_percent = (float)(core_statistics[coreid].burst_rx * 100)/
+					core_statistics[coreid].rx;

Would float be always enough here? Might better long double?

[Anoob] This is just to get the percentage of traffic doing bursts. Since it's percentage float seemed enough. Do you think we need double instead? I can change if you think so.

Quoted 55 lines
Quoted 14 lines
examples/ipsec-secgw/ipsec-secgw.c +13 -0
+		printf("\nStatistics for core %u ------------------------------"
+			   "\nPackets received: %20"PRIu64
+			   "\nPackets sent: %24"PRIu64
+			   "\nPackets dropped: %21"PRIu64
+			   "\nBurst percent: %23.2f",
+			   coreid,
+			   core_statistics[coreid].rx,
+			   core_statistics[coreid].tx,
+			   core_statistics[coreid].dropped,
+			   burst_percent);
+
+		total_packets_dropped += core_statistics[coreid].dropped;
+		total_packets_tx += core_statistics[coreid].tx;

examples/ipsec-secgw/ipsec-secgw.c adjust_ipv6_pktlen(struct rte_mbuf *m, const struct rte_ipv6_hdr *iph, · line 345

Quoted 10 lines
examples/ipsec-secgw/ipsec-secgw.c +10 -0
+		total_packets_rx += core_statistics[coreid].rx;
+	}
+	printf("\nAggregate statistics ==============================="
+		   "\nTotal packets received: %14"PRIu64
+		   "\nTotal packets sent: %18"PRIu64
+		   "\nTotal packets dropped: %15"PRIu64,
+		   total_packets_rx,
+		   total_packets_tx,
+		   total_packets_dropped);
+
	printf("\n===================================================

=\n");

examples/ipsec-secgw/ipsec-secgw.c prepare_one_packet(struct rte_mbuf *pkt, struct · line 390

Quoted 9 lines
examples/ipsec-secgw/ipsec-secgw.c +3 -0
+}
+#endif /* ENABLE_STATS */
+
 static inline void
 prepare_one_packet(struct rte_mbuf *pkt, struct ipsec_traffic *t)  {
@@ -333,6 +388,7 @@ prepare_one_packet(struct rte_mbuf *pkt, struct
 ipsec_traffic *t)
 
 		/* drop packet when IPv6 header exceeds first segment length

*/

examples/ipsec-secgw/ipsec-secgw.c prepare_one_packet(struct rte_mbuf *pkt, struct

Quoted 6 lines
examples/ipsec-secgw/ipsec-secgw.c +1 -0
 		if (unlikely(l3len > pkt->data_len)) {
+			core_stats_update_drop(1);
 			rte_pktmbuf_free(pkt);
 			return;
examples/ipsec-secgw/ipsec-secgw.c +0 -0
 		}
@@ -350,6 +406,7 @@ prepare_one_packet(struct rte_mbuf *pkt, struct

ipsec_traffic *t)

examples/ipsec-secgw/ipsec-secgw.c send_burst(struct lcore_conf *qconf, uint16_t n,

Quoted 8 lines
examples/ipsec-secgw/ipsec-secgw.c +1 -0
 		/* Unknown/Unsupported type, drop the packet */
 		RTE_LOG(ERR, IPSEC, "Unsupported packet type 0x%x\n",
 			rte_be_to_cpu_16(eth->ether_type));
+		core_stats_update_drop(1);
 		rte_pktmbuf_free(pkt);
 		return;
 	}
@@ -471,6 +528,11 @@ send_burst(struct lcore_conf *qconf, uint16_t n,

uint16_t port)

Quoted 3 lines
examples/ipsec-secgw/ipsec-secgw.c +0 -0
 	int32_t ret;
 	uint16_t queueid;
 
Quoted 8 lines

examples/ipsec-secgw/ipsec-secgw.c send_burst(struct lcore_conf *qconf, uint16_t n, uint16_t port) · line 534

Quoted 4 lines
examples/ipsec-secgw/ipsec-secgw.c +4 -0
+#ifdef ENABLE_STATS
+	int lcore_id = rte_lcore_id();
+	core_statistics[lcore_id].tx += n;
+#endif /* ENABLE_STATS */

Instead of polluting genric code with ifdefs, why not to introduce 2 new
functions: core_stats_update_rx(), core_stats_update_tx(), as you did for core_stats_drop()?

[Anoob] The drop would happen from multiple places while Rx & Tx would happen only from one place each. I wasn't sure which would be the right approach. If you agree to introduce rx, tx & drop stats functions, I will update so.

Quoted 8 lines
Quoted 8 lines
examples/ipsec-secgw/ipsec-secgw.c +1 -0
+
 	queueid = qconf->tx_queue_id[port];
 	m_table = (struct rte_mbuf **)qconf->tx_mbufs[port].m_table;
 
@@ -478,6 +540,9 @@ send_burst(struct lcore_conf *qconf, uint16_t n,
 uint16_t port)
 
Quoted 8 lines

examples/ipsec-secgw/ipsec-secgw.c send_burst(struct lcore_conf *qconf, uint16_t n, uint16_t port)

Quoted 4 lines
examples/ipsec-secgw/ipsec-secgw.c +2 -0
 	ret = rte_eth_tx_burst(port, queueid, m_table, n);
 	if (unlikely(ret < n)) {
+#ifdef ENABLE_STATS
+		core_statistics[lcore_id].dropped += n-ret; #endif /*

ENABLE_STATS

examples/ipsec-secgw/ipsec-secgw.c send_burst(struct lcore_conf *qconf, uint16_t n, · line 546

Quoted 1 line
examples/ipsec-secgw/ipsec-secgw.c +1 -0
+*/

You have core_stats_update_drop() for that - use it.

[Anoob] lcore_id is already available as local variable. Will add the call instead.

Quoted 15 lines

examples/ipsec-secgw/ipsec-secgw.c send_fragment_packet(struct lcore_conf *qconf,

Quoted 5 lines
examples/ipsec-secgw/ipsec-secgw.c +0 -0
 		do {
 			rte_pktmbuf_free(m_table[ret]);
examples/ipsec-secgw/ipsec-secgw.c +0 -0
 		} while (++ret < n);
@@ -525,6 +590,7 @@ send_fragment_packet(struct lcore_conf *qconf,

struct rte_mbuf *m,

examples/ipsec-secgw/ipsec-secgw.c send_single_packet(struct rte_mbuf *m, uint16_t

Quoted 8 lines
examples/ipsec-secgw/ipsec-secgw.c +1 -0
 			"error code: %d\n",
 			__func__, m->pkt_len, rte_errno);
 
+	core_stats_update_drop(1);
 	rte_pktmbuf_free(m);
 	return len;
 }
@@ -549,8 +615,10 @@ send_single_packet(struct rte_mbuf *m, uint16_t

port, uint8_t proto)

Quoted 12 lines

examples/ipsec-secgw/ipsec-secgw.c send_single_packet(struct rte_mbuf *m, uint16_t port, uint8_t proto) · line 620

Quoted 7 lines
examples/ipsec-secgw/ipsec-secgw.c +2 -1
 	/* need to fragment the packet */
 	} else if (frag_tbl_sz > 0)
 		len = send_fragment_packet(qconf, m, port, proto);
-	else
+	else {
+		core_stats_update_drop(1);
 		rte_pktmbuf_free(m);

It looks like a lot of such places...
Would it be worth to unite core_stats_update_drop() and rte_pktmbuf_free(m) Into some inline function: ipsec_secgw_packet_drop(struct rte_mbuf *m[], uint32_t n) and use it all over such places.

[Anoob] Ack

Quoted 106 lines

examples/ipsec-secgw/ipsec-secgw.c split46_traffic(struct ipsec_traffic *trf, struct

Quoted 28 lines
examples/ipsec-secgw/ipsec-secgw.c +1 -0
+	}
 
 	/* enough pkts to be sent */
 	if (unlikely(len == MAX_PKT_BURST)) { @@ -584,18 +652,21 @@

inbound_sp_sa(struct sp_ctx *sp, struct sa_ctx *sa, struct traffic_type *ip,

examples/ipsec-secgw/ipsec-secgw.c +3 -0
 			continue;
 		}
 		if (res == DISCARD) {
+			core_stats_update_drop(1);
 			rte_pktmbuf_free(m);
 			continue;
 		}
 
 		/* Only check SPI match for processed IPSec packets */
 		if (i < lim && ((m->ol_flags & PKT_RX_SEC_OFFLOAD) == 0)) {
+			core_stats_update_drop(1);
 			rte_pktmbuf_free(m);
 			continue;
 		}
 
 		sa_idx = res - 1;
 		if (!inbound_sa_check(sa, m, sa_idx)) {
+			core_stats_update_drop(1);
 			rte_pktmbuf_free(m);
 			continue;
 		}
@@ -630,8 +701,10 @@ split46_traffic(struct ipsec_traffic *trf, struct

rte_mbuf *mb[], uint32_t num)

examples/ipsec-secgw/ipsec-secgw.c outbound_sp(struct sp_ctx *sp, struct traffic_type

Quoted 12 lines
examples/ipsec-secgw/ipsec-secgw.c +3 -1
 					uint8_t *,
 					offsetof(struct ip6_hdr, ip6_nxt));
 			n6++;
-		} else
+		} else {
+			core_stats_update_drop(1);
 			rte_pktmbuf_free(m);
+		}
 	}
 
 	trf->ip4.num = n4;
@@ -682,11 +755,12 @@ outbound_sp(struct sp_ctx *sp, struct traffic_type

*ip,

examples/ipsec-secgw/ipsec-secgw.c outbound_sp(struct sp_ctx *sp, struct traffic_type *ip, · line 761

Quoted 9 lines
examples/ipsec-secgw/ipsec-secgw.c +3 -2
 	for (i = 0; i < ip->num; i++) {
 		m = ip->pkts[i];
 		sa_idx = ip->res[i] - 1;
-		if (ip->res[i] == DISCARD)
+		if (ip->res[i] == DISCARD) {
+			core_stats_update_drop(1);
 			rte_pktmbuf_free(m);
-		else if (ip->res[i] == BYPASS)
+		} else if (ip->res[i] == BYPASS) {

Looks unnecessary.

examples/ipsec-secgw/ipsec-secgw.c process_pkts_outbound_nosp(struct ipsec_ctx

Quoted 31 lines
examples/ipsec-secgw/ipsec-secgw.c +5 -2
 			ip->pkts[j++] = m;
-		else {
+		} else {
 			ipsec->res[ipsec->num] = sa_idx;
 			ipsec->pkts[ipsec->num++] = m;
 		}
@@ -705,6 +779,8 @@ process_pkts_outbound(struct ipsec_ctx *ipsec_ctx,
 	for (i = 0; i < traffic->ipsec.num; i++)
 		rte_pktmbuf_free(traffic->ipsec.pkts[i]);
 
+	core_stats_update_drop(traffic->ipsec.num);
+
 	traffic->ipsec.num = 0;
 
 	outbound_sp(ipsec_ctx->sp4_ctx, &traffic->ip4, &traffic->ipsec); @@
-745,12 +821,14 @@ process_pkts_inbound_nosp(struct ipsec_ctx *ipsec_ctx,
 	/* Drop any IPv4 traffic from unprotected ports */
 	for (i = 0; i < traffic->ip4.num; i++)
 		rte_pktmbuf_free(traffic->ip4.pkts[i]);
+	core_stats_update_drop(traffic->ip4.num);
 
 	traffic->ip4.num = 0;
 
 	/* Drop any IPv6 traffic from unprotected ports */
 	for (i = 0; i < traffic->ip6.num; i++)
 		rte_pktmbuf_free(traffic->ip6.pkts[i]);
+	core_stats_update_drop(traffic->ip6.num);
 
 	traffic->ip6.num = 0;
@@ -788,6 +866,7 @@ process_pkts_outbound_nosp(struct ipsec_ctx

*ipsec_ctx,

examples/ipsec-secgw/ipsec-secgw.c route4_pkts(struct rt_ctx *rt_ctx, struct rte_mbuf

Quoted 8 lines
examples/ipsec-secgw/ipsec-secgw.c +1 -0
 	/* Drop any IPsec traffic from protected ports */
 	for (i = 0; i < traffic->ipsec.num; i++)
 		rte_pktmbuf_free(traffic->ipsec.pkts[i]);
+	core_stats_update_drop(traffic->ipsec.num);
 
 	n = 0;
 
@@ -901,6 +980,7 @@ route4_pkts(struct rt_ctx *rt_ctx, struct rte_mbuf

*pkts[], uint8_t nb_pkts)

examples/ipsec-secgw/ipsec-secgw.c route6_pkts(struct rt_ctx *rt_ctx, struct rte_mbuf

Quoted 8 lines
examples/ipsec-secgw/ipsec-secgw.c +1 -0
 		}
 
 		if ((pkt_hop & RTE_LPM_LOOKUP_SUCCESS) == 0) {
+			core_stats_update_drop(1);
 			rte_pktmbuf_free(pkts[i]);
 			continue;
 		}
@@ -953,6 +1033,7 @@ route6_pkts(struct rt_ctx *rt_ctx, struct rte_mbuf

*pkts[], uint8_t nb_pkts)

Quoted 2 lines
examples/ipsec-secgw/ipsec-secgw.c +0 -0
 		}
 
Quoted 14 lines

examples/ipsec-secgw/ipsec-secgw.c ipsec_poll_mode_worker(void) · line 1185

Quoted 12 lines
examples/ipsec-secgw/ipsec-secgw.c +4 -0
 		if (pkt_hop == -1) {
+			core_stats_update_drop(1);
 			rte_pktmbuf_free(pkts[i]);
 			continue;
 		}
@@ -1099,6 +1180,9 @@ ipsec_poll_mode_worker(void)
 	const uint64_t drain_tsc = (rte_get_tsc_hz() + US_PER_S - 1)
 			/ US_PER_S * BURST_TX_DRAIN_US;
 	struct lcore_rx_queue *rxql;
+#ifdef ENABLE_STATS
+	uint64_t timer_tsc = 0;
+#endif /* ENABLE_STATS */

Probably better just RTE_SET_USED(timer_tsc);

[Anoob] Isn't it better to keep all the code related an optional feature under on flag? Either way is fine.

Quoted 23 lines
Quoted 11 lines
examples/ipsec-secgw/ipsec-secgw.c +3 -0
 	prev_tsc = 0;
 	lcore_id = rte_lcore_id();
@@ -1159,6 +1243,19 @@ ipsec_poll_mode_worker(void)
 			drain_tx_buffers(qconf);
 			drain_crypto_buffers(qconf);
 			prev_tsc = cur_tsc;
+#ifdef ENABLE_STATS
+			if (lcore_id == rte_get_master_lcore()) {
+				/* advance the timer */

examples/ipsec-secgw/ipsec-secgw.c ipsec_poll_mode_worker(void) · line 1258

Quoted 10 lines
examples/ipsec-secgw/ipsec-secgw.c +10 -0
+				timer_tsc += diff_tsc;
+
+				/* if timer has reached its timeout */
+				if (unlikely(timer_tsc >= timer_period)) {
+					print_stats();
+					/* reset the timer */
+					timer_tsc = 0;
+				}
+			}
+#endif /* ENABLE_STATS */
Quoted 7 lines

Why to do stats collection/display inside data-path?
Why not use rte_timer/rte_alarm and make it happen in control thread? Another option - make stats printing at some signal to the process. In that case we don't need to bother with time period at all - it will be user to decide.
Again if we remove that print_stats() from data-path it might become cheap enough to always collect it, and we will not need ENABLE_STATS macro at all.

[Anoob] The stats collection will not be cheap because we are updating a common structure from multiple cores. And the idea is not to have a very efficient stats mechanism. When we have more cores engaged, IPsec performance is directly dependent on ability of RSS to split traffic to multiple cores. Such stats would come handy in understanding per core distribution of incoming traffic.

Quoted 14 lines

examples/ipsec-secgw/ipsec-secgw.c ipsec_poll_mode_worker(void) · line 1261

Quoted 4 lines
examples/ipsec-secgw/ipsec-secgw.c +0 -0
 		}
 
 		for (i = 0; i < qconf->nb_rx_queue; ++i) { @@ -1169,6 +1266,12

@@

examples/ipsec-secgw/ipsec-secgw.c ipsec_poll_mode_worker(void) · line 1272

Quoted 8 lines

ipsec_poll_mode_worker(void)

examples/ipsec-secgw/ipsec-secgw.c +4 -0
 			nb_rx = rte_eth_rx_burst(portid, queueid,
 					pkts, MAX_PKT_BURST);
 
+#ifdef ENABLE_STATS
+			core_statistics[lcore_id].rx += nb_rx;
+			if (nb_rx == MAX_PKT_BURST)
+				core_statistics[lcore_id].burst_rx += nb_rx;

#endif /*

Quoted 5 lines

examples/ipsec-secgw/ipsec-secgw.c

Quoted 2 lines
examples/ipsec-secgw/ipsec-secgw.c +2 -0
+ENABLE_STATS */
+

Same for above for TX: no need to pollute the code with ifdefs. Better to introduce new function: stats_update_rx() or so.

[Anoob] Ack

Quoted 139 lines
Quoted 23 lines
examples/ipsec-secgw/ipsec-secgw.c +5 -0
 			if (nb_rx > 0)
 				process_pkts(qconf, pkts, nb_rx, portid);
@@ -2747,6 +2850,11 @@ main(int32_t argc, char **argv)
 	signal(SIGINT, signal_handler);
 	signal(SIGTERM, signal_handler);
 
+#ifdef ENABLE_STATS
+	/* convert to number of cycles */
+	timer_period *= rte_get_timer_hz();
+#endif /* ENABLE_STATS */
+
 	/* initialize event helper configuration */
 	eh_conf = eh_conf_init();
 	if (eh_conf == NULL)
examples/ipsec-secgw/ipsec-secgw.h +0 -0
index 4b53cb5..d886a35 100644

examples/ipsec-secgw/ipsec-secgw.h line 10

Quoted 5 lines
examples/ipsec-secgw/ipsec-secgw.h +2 -0
@@ -6,6 +6,8 @@
 
 #include <stdbool.h>
 
+//#define ENABLE_STATS
+

Should be removed I think.

examples/ipsec-secgw/ipsec.c ipsec_enqueue(ipsec_xform_fn xform_func, struct

Quoted 38 lines
examples/ipsec-secgw/ipsec-secgw.h +0 -0
 #define NB_SOCKETS 4
 
 #define MAX_PKT_BURST 32
(quoted diff) +0 -0

b/examples/ipsec-secgw/ipsec.c index bf88d80..dcb9312 100644

examples/ipsec-secgw/ipsec.c +4 -1
@@ -499,8 +499,10 @@ enqueue_cop_burst(struct cdev_qp *cqp)
 			" enqueued %u crypto ops out of %u\n",
 			cqp->id, cqp->qp, ret, len);
 			/* drop packets that we fail to enqueue */
-			for (i = ret; i < len; i++)
+			for (i = ret; i < len; i++) {
+				core_stats_update_drop(1);
 				rte_pktmbuf_free(cqp->buf[i]->sym->m_src);
+			}
 	}
 	cqp->in_flight += ret;
 	cqp->len = 0;
@@ -528,6 +530,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct
 ipsec_ctx *ipsec_ctx,
 
 	for (i = 0; i < nb_pkts; i++) {
 		if (unlikely(sas[i] == NULL)) {
+			core_stats_update_drop(1);
 			rte_pktmbuf_free(pkts[i]);
 			continue;
examples/ipsec-secgw/ipsec.c +1 -0
 		}
@@ -549,6 +552,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct
 ipsec_ctx *ipsec_ctx,
 
 			if ((unlikely(ips->security.ses == NULL)) &&
 				create_lookaside_session(ipsec_ctx, sa, ips)) {
+				core_stats_update_drop(1);
 				rte_pktmbuf_free(pkts[i]);
 				continue;
examples/ipsec-secgw/ipsec.c +0 -0
 			}
@@ -563,6 +567,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct

ipsec_ctx *ipsec_ctx,

examples/ipsec-secgw/ipsec.c ipsec_enqueue(ipsec_xform_fn xform_func, struct · line 568

Quoted 2 lines
examples/ipsec-secgw/ipsec.c +0 -0
 		case RTE_SECURITY_ACTION_TYPE_CPU_CRYPTO:
 			RTE_LOG(ERR, IPSEC, "CPU crypto is not supported by

the"

examples/ipsec-secgw/ipsec.c ipsec_inline_dequeue(ipsec_xform_fn xform_func,

Quoted 33 lines
examples/ipsec-secgw/ipsec.c +2 -0
 					" legacy mode.");
+			core_stats_update_drop(1);
 			rte_pktmbuf_free(pkts[i]);
 			continue;
 
@@ -575,6 +580,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct
 ipsec_ctx *ipsec_ctx,
 
 			if ((unlikely(ips->crypto.ses == NULL)) &&
 				create_lookaside_session(ipsec_ctx, sa, ips)) {
+				core_stats_update_drop(1);
 				rte_pktmbuf_free(pkts[i]);
 				continue;
examples/ipsec-secgw/ipsec.c +1 -0
 			}
@@ -584,6 +590,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct
 ipsec_ctx *ipsec_ctx,
 
 			ret = xform_func(pkts[i], sa, &priv->cop);
 			if (unlikely(ret)) {
+				core_stats_update_drop(1);
 				rte_pktmbuf_free(pkts[i]);
 				continue;
examples/ipsec-secgw/ipsec.c +1 -0
 			}
@@ -608,6 +615,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct
 ipsec_ctx *ipsec_ctx,
 
 			ret = xform_func(pkts[i], sa, &priv->cop);
 			if (unlikely(ret)) {
+				core_stats_update_drop(1);
 				rte_pktmbuf_free(pkts[i]);
 				continue;
examples/ipsec-secgw/ipsec.c +0 -0
 			}
@@ -643,6 +651,7 @@ ipsec_inline_dequeue(ipsec_xform_fn xform_func,

struct ipsec_ctx *ipsec_ctx,

examples/ipsec-secgw/ipsec.c ipsec_dequeue(ipsec_xform_fn xform_func, struct

Quoted 8 lines
examples/ipsec-secgw/ipsec.c +1 -0
 		sa = priv->sa;
 		ret = xform_func(pkt, sa, &priv->cop);
 		if (unlikely(ret)) {
+			core_stats_update_drop(1);
 			rte_pktmbuf_free(pkt);
 			continue;
 		}
@@ -690,12 +699,14 @@ ipsec_dequeue(ipsec_xform_fn xform_func, struct

ipsec_ctx *ipsec_ctx,

examples/ipsec-secgw/ipsec.c ipsec_dequeue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,

Quoted 9 lines
examples/ipsec-secgw/ipsec.c +1 -0
 				RTE_SECURITY_ACTION_TYPE_NONE) {
 				ret = xform_func(pkt, sa, cops[j]);
 				if (unlikely(ret)) {
+					core_stats_update_drop(1);
 					rte_pktmbuf_free(pkt);
 					continue;
 				}
 			} else if (ipsec_get_action_type(sa) ==
 
	RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) {
Quoted 13 lines
examples/ipsec-secgw/ipsec.c +1 -0
 				if (cops[j]->status) {
+					core_stats_update_drop(1);
 					rte_pktmbuf_free(pkt);
 					continue;
 				}
(quoted diff) +0 -0

b/examples/ipsec-secgw/ipsec.h index 1e642d1..8519eab 100644

examples/ipsec-secgw/ipsec.h +0 -0
@@ -46,6 +46,17 @@
 
 #define IP6_VERSION (6)
 
Quoted 12 lines

examples/ipsec-secgw/ipsec.h line 54

Quoted 6 lines
examples/ipsec-secgw/ipsec.h +6 -0
+#ifdef ENABLE_STATS
+struct ipsec_core_statistics {
+	uint64_t tx;
+	uint64_t rx;
+	uint64_t dropped;
+	uint64_t burst_rx;

A bit strange to have burst_rx and no similar counterpart for tx. BTW, do you need burst_rx?
Might be better:
nb_calls_rx, nb_calls_tx;
and then: rx/nb_calls_rx will give you average burst size.

[Anoob] After the packets are received in the app, further IPsec processing is done, which is more efficient when done in batches. For example, SA lookup is better when done in a batch, as SIMD instructions would be used. Ability of traffic to be in burst or ability of ethdev to give burst will have an effect on the IPsec performance per core. Tx burst percentage isn't that important, since it is the app which submits for Tx.

Rx/nb_calls_rx would also give a similar kind of stat, but the difference is whether 1 packet cases and 31 packet cases are treated equal. I would prefer to keep it the current way, but what you suggested also would work.

Quoted 6 lines

examples/ipsec-secgw/ipsec.h line 57

Quoted 4 lines
examples/ipsec-secgw/ipsec.h +3 -0
+} __rte_cache_aligned;
+
+struct ipsec_core_statistics core_statistics[RTE_MAX_ETHPORTS];

Should be RTE_MAX_LCORES, I think.

[Anoob] Ack

Quoted 40 lines
Quoted 26 lines
examples/ipsec-secgw/ipsec.h +12 -0
+#endif /* ENABLE_STATS */
+
 struct rte_crypto_xform;
 struct ipsec_xform;
 struct rte_mbuf;
@@ -416,4 +427,15 @@ check_flow_params(uint16_t fdir_portid, uint8_t
 fdir_qid);  int  create_ipsec_esp_flow(struct ipsec_sa *sa);
 
+static inline void
+core_stats_update_drop(int n)
+{
+#ifdef ENABLE_STATS
+	int lcore_id = rte_lcore_id();
+	core_statistics[lcore_id].dropped += n; #else
+	RTE_SET_USED(n);
+#endif /* ENABLE_STATS */
+}
+
 #endif /* __IPSEC_H__ */
examples/ipsec-secgw/ipsec_process.c +0 -0
index bb2f2b8..05cb3ad 100644

examples/ipsec-secgw/ipsec_process.c free_pkts(struct rte_mbuf *mb[], uint32_t n) { · line 29

Quoted 6 lines
examples/ipsec-secgw/ipsec_process.c +4 -0
@@ -24,6 +24,11 @@ free_pkts(struct rte_mbuf *mb[], uint32_t n)  {
 	uint32_t i;
 
+#ifdef ENABLE_STATS
+	int lcore_id = rte_lcore_id();
+	core_statistics[lcore_id].dropped += n; #endif /* ENABLE_STATS */
+

Same as above - why not use stats_update_drop() here?

Quoted 5 lines
examples/ipsec-secgw/ipsec_process.c +0 -1
 	for (i = 0; i != n; i++)
 		rte_pktmbuf_free(mb[i]);
 }
--

2.7.4

#4 lore raw
Quoted 28 lines
Quoted 20 lines
Quoted 8 lines
examples/ipsec-secgw/ipsec-secgw.c +3 -0
@@ -1159,6 +1243,19 @@ ipsec_poll_mode_worker(void)
 			drain_tx_buffers(qconf);
 			drain_crypto_buffers(qconf);
 			prev_tsc = cur_tsc;
+#ifdef ENABLE_STATS
+			if (lcore_id == rte_get_master_lcore()) {
+				/* advance the timer */

examples/ipsec-secgw/ipsec-secgw.c ipsec_poll_mode_worker(void) · line 1258

Quoted 10 lines
examples/ipsec-secgw/ipsec-secgw.c +10 -0
+				timer_tsc += diff_tsc;
+
+				/* if timer has reached its timeout */
+				if (unlikely(timer_tsc >= timer_period)) {
+					print_stats();
+					/* reset the timer */
+					timer_tsc = 0;
+				}
+			}
+#endif /* ENABLE_STATS */
Quoted 7 lines

Why to do stats collection/display inside data-path?
Why not use rte_timer/rte_alarm and make it happen in control thread? Another option - make stats printing at some signal to the process. In that case we don't need to bother with time period at all - it will be user to decide.
Again if we remove that print_stats() from data-path it might become cheap enough to always collect it, and we will not need ENABLE_STATS macro at all.

Quoted 1 line

[Anoob] The stats collection will not be cheap because we are updating a common structure from multiple cores.

What I am saying - per-lcore stats collection (updating lcore stats by lcore itself) - should be cheap enough and can stay in data-path (probably even always, not only when particular compile flag is enabled). Global stats collection and display is quite expensive operation,
so we shouldn't keep it in data-path.
We have a control thread within dpdk (for alarms, signals, timers, etc.),
why not to call print_stats from there?

Quoted 5 lines

And the idea is not to
have a very efficient stats mechanism. When we have more cores engaged, IPsec performance is directly dependent on ability of RSS to split traffic to multiple cores. Such stats would come handy in understanding per core distribution of incoming traffic.

Quoted 1 line
#5 lore raw

PATCH v3 1/1 examples/ipsec-secgw: add per core packet stats

In reply to Anoob Joseph

Adding per core packet handling stats to analyze traffic distribution when multiple cores are engaged.

Since aggregating the packet stats across cores would affect performance, keeping the feature disabled using compile time flags.

---

v3:
* Added wrapper functions for updating rx, tx & dropped counts
* Updated free_pkts() to have stats updated internally
* Introduced similar free_pkt() function which updates stats and frees one packet
* Moved all inline functions and macros to ipsec-secgw.h
* Made STATS_INTERVAL macro to control the interval of the stats update. STATS_INTERVAL = 0 would disable the feature.

v2:
* Added lookup failure cases to drop count

 examples/ipsec-secgw/ipsec-secgw.c   | 113 ++++++++++++++++++++++++++++-------
 examples/ipsec-secgw/ipsec-secgw.h   |  68 +++++++++++++++++++++
 examples/ipsec-secgw/ipsec.c         |  20 +++----
 examples/ipsec-secgw/ipsec_process.c |  11 +---
 4 files changed, 171 insertions(+), 41 deletions(-)
examples/ipsec-secgw/ipsec-secgw.c +92 -21
index 6d02341..e97a4f8 100644
@@ -288,6 +288,59 @@ adjust_ipv6_pktlen(struct rte_mbuf *m, const struct rte_ipv6_hdr *iph,
 	}
 }
 
+#if (STATS_INTERVAL > 0)
+
+/* Print out statistics on packet distribution */
+static void
+print_stats(void)
+{
+	uint64_t total_packets_dropped, total_packets_tx, total_packets_rx;
+	unsigned int coreid;
+	float burst_percent;
+
+	total_packets_dropped = 0;
+	total_packets_tx = 0;
+	total_packets_rx = 0;
+
+	const char clr[] = { 27, '[', '2', 'J', '\0' };
+
+	/* Clear screen and move to top left */
+	printf("%s", clr);
+
+	printf("\nCore statistics ====================================");
+
+	for (coreid = 0; coreid < RTE_MAX_LCORE; coreid++) {
+		/* skip disabled cores */
+		if (rte_lcore_is_enabled(coreid) == 0)
+			continue;
+		burst_percent = (float)(core_statistics[coreid].burst_rx * 100)/
+					core_statistics[coreid].rx;
+		printf("\nStatistics for core %u ------------------------------"
+			   "\nPackets received: %20"PRIu64
+			   "\nPackets sent: %24"PRIu64
+			   "\nPackets dropped: %21"PRIu64
+			   "\nBurst percent: %23.2f",
+			   coreid,
+			   core_statistics[coreid].rx,
+			   core_statistics[coreid].tx,
+			   core_statistics[coreid].dropped,
+			   burst_percent);
+
+		total_packets_dropped += core_statistics[coreid].dropped;
+		total_packets_tx += core_statistics[coreid].tx;
+		total_packets_rx += core_statistics[coreid].rx;
+	}
+	printf("\nAggregate statistics ==============================="
+		   "\nTotal packets received: %14"PRIu64
+		   "\nTotal packets sent: %18"PRIu64
+		   "\nTotal packets dropped: %15"PRIu64,
+		   total_packets_rx,
+		   total_packets_tx,
+		   total_packets_dropped);
+	printf("\n====================================================\n");
+}
+#endif /* STATS_INTERVAL */
+
 static inline void
 prepare_one_packet(struct rte_mbuf *pkt, struct ipsec_traffic *t)
 {
@@ -333,7 +386,7 @@ prepare_one_packet(struct rte_mbuf *pkt, struct ipsec_traffic *t)
 
 		/* drop packet when IPv6 header exceeds first segment length */
 		if (unlikely(l3len > pkt->data_len)) {
-			rte_pktmbuf_free(pkt);
+			free_pkt(pkt);
 			return;
 		}
 
@@ -350,7 +403,7 @@ prepare_one_packet(struct rte_mbuf *pkt, struct ipsec_traffic *t)
 		/* Unknown/Unsupported type, drop the packet */
 		RTE_LOG(ERR, IPSEC, "Unsupported packet type 0x%x\n",
 			rte_be_to_cpu_16(eth->ether_type));
-		rte_pktmbuf_free(pkt);
+		free_pkt(pkt);
 		return;
 	}
 
@@ -477,9 +530,12 @@ send_burst(struct lcore_conf *qconf, uint16_t n, uint16_t port)
 	prepare_tx_burst(m_table, n, port, qconf);
 
 	ret = rte_eth_tx_burst(port, queueid, m_table, n);
+
+	core_stats_update_tx(ret);
+
 	if (unlikely(ret < n)) {
 		do {
-			rte_pktmbuf_free(m_table[ret]);
+			free_pkt(m_table[ret]);
 		} while (++ret < n);
 	}
 
@@ -525,7 +581,7 @@ send_fragment_packet(struct lcore_conf *qconf, struct rte_mbuf *m,
 			"error code: %d\n",
 			__func__, m->pkt_len, rte_errno);
 
-	rte_pktmbuf_free(m);
+	free_pkt(m);
 	return len;
 }
 
@@ -550,7 +606,7 @@ send_single_packet(struct rte_mbuf *m, uint16_t port, uint8_t proto)
 	} else if (frag_tbl_sz > 0)
 		len = send_fragment_packet(qconf, m, port, proto);
 	else
-		rte_pktmbuf_free(m);
+		free_pkt(m);
 
 	/* enough pkts to be sent */
 	if (unlikely(len == MAX_PKT_BURST)) {
@@ -584,19 +640,19 @@ inbound_sp_sa(struct sp_ctx *sp, struct sa_ctx *sa, struct traffic_type *ip,
 			continue;
 		}
 		if (res == DISCARD) {
-			rte_pktmbuf_free(m);
+			free_pkt(m);
 			continue;
 		}
 
 		/* Only check SPI match for processed IPSec packets */
 		if (i < lim && ((m->ol_flags & PKT_RX_SEC_OFFLOAD) == 0)) {
-			rte_pktmbuf_free(m);
+			free_pkt(m);
 			continue;
 		}
 
 		sa_idx = res - 1;
 		if (!inbound_sa_check(sa, m, sa_idx)) {
-			rte_pktmbuf_free(m);
+			free_pkt(m);
 			continue;
 		}
 		ip->pkts[j++] = m;
@@ -631,7 +687,7 @@ split46_traffic(struct ipsec_traffic *trf, struct rte_mbuf *mb[], uint32_t num)
 					offsetof(struct ip6_hdr, ip6_nxt));
 			n6++;
 		} else
-			rte_pktmbuf_free(m);
+			free_pkt(m);
 	}
 
 	trf->ip4.num = n4;
@@ -683,7 +739,7 @@ outbound_sp(struct sp_ctx *sp, struct traffic_type *ip,
 		m = ip->pkts[i];
 		sa_idx = ip->res[i] - 1;
 		if (ip->res[i] == DISCARD)
-			rte_pktmbuf_free(m);
+			free_pkt(m);
 		else if (ip->res[i] == BYPASS)
 			ip->pkts[j++] = m;
 		else {
@@ -702,8 +758,7 @@ process_pkts_outbound(struct ipsec_ctx *ipsec_ctx,
 	uint16_t idx, nb_pkts_out, i;
 
 	/* Drop any IPsec traffic from protected ports */
-	for (i = 0; i < traffic->ipsec.num; i++)
-		rte_pktmbuf_free(traffic->ipsec.pkts[i]);
+	free_pkts(traffic->ipsec.pkts, traffic->ipsec.num);
 
 	traffic->ipsec.num = 0;
 
@@ -743,14 +798,12 @@ process_pkts_inbound_nosp(struct ipsec_ctx *ipsec_ctx,
 	uint32_t nb_pkts_in, i, idx;
 
 	/* Drop any IPv4 traffic from unprotected ports */
-	for (i = 0; i < traffic->ip4.num; i++)
-		rte_pktmbuf_free(traffic->ip4.pkts[i]);
+	free_pkts(traffic->ip4.pkts, traffic->ip4.num);
 
 	traffic->ip4.num = 0;
 
 	/* Drop any IPv6 traffic from unprotected ports */
-	for (i = 0; i < traffic->ip6.num; i++)
-		rte_pktmbuf_free(traffic->ip6.pkts[i]);
+	free_pkts(traffic->ip6.pkts, traffic->ip6.num);
 
 	traffic->ip6.num = 0;
 
@@ -786,8 +839,7 @@ process_pkts_outbound_nosp(struct ipsec_ctx *ipsec_ctx,
 	struct ip *ip;
 
 	/* Drop any IPsec traffic from protected ports */
-	for (i = 0; i < traffic->ipsec.num; i++)
-		rte_pktmbuf_free(traffic->ipsec.pkts[i]);
+	free_pkts(traffic->ipsec.pkts, traffic->ipsec.num);
 
 	n = 0;
 
@@ -901,7 +953,7 @@ route4_pkts(struct rt_ctx *rt_ctx, struct rte_mbuf *pkts[], uint8_t nb_pkts)
 		}
 
 		if ((pkt_hop & RTE_LPM_LOOKUP_SUCCESS) == 0) {
-			rte_pktmbuf_free(pkts[i]);
+			free_pkt(pkts[i]);
 			continue;
 		}
 		send_single_packet(pkts[i], pkt_hop & 0xff, IPPROTO_IP);
@@ -953,7 +1005,7 @@ route6_pkts(struct rt_ctx *rt_ctx, struct rte_mbuf *pkts[], uint8_t nb_pkts)
 		}
 
 		if (pkt_hop == -1) {
-			rte_pktmbuf_free(pkts[i]);
+			free_pkt(pkts[i]);
 			continue;
 		}
 		send_single_packet(pkts[i], pkt_hop & 0xff, IPPROTO_IPV6);
@@ -1099,6 +1151,10 @@ ipsec_poll_mode_worker(void)
 	const uint64_t drain_tsc = (rte_get_tsc_hz() + US_PER_S - 1)
 			/ US_PER_S * BURST_TX_DRAIN_US;
 	struct lcore_rx_queue *rxql;
+#if (STATS_INTERVAL > 0)
+	const uint64_t timer_period = STATS_INTERVAL * rte_get_timer_hz();
+	uint64_t timer_tsc = 0;
+#endif /* STATS_INTERVAL */
 
 	prev_tsc = 0;
 	lcore_id = rte_lcore_id();
@@ -1159,6 +1215,19 @@ ipsec_poll_mode_worker(void)
 			drain_tx_buffers(qconf);
 			drain_crypto_buffers(qconf);
 			prev_tsc = cur_tsc;
+#if (STATS_INTERVAL > 0)
+			if (lcore_id == rte_get_master_lcore()) {
+				/* advance the timer */
+				timer_tsc += diff_tsc;
+
+				/* if timer has reached its timeout */
+				if (unlikely(timer_tsc >= timer_period)) {
+					print_stats();
+					/* reset the timer */
+					timer_tsc = 0;
+				}
+			}
+#endif /* STATS_INTERVAL */
 		}
 
 		for (i = 0; i < qconf->nb_rx_queue; ++i) {
@@ -1169,8 +1238,10 @@ ipsec_poll_mode_worker(void)
 			nb_rx = rte_eth_rx_burst(portid, queueid,
 					pkts, MAX_PKT_BURST);
 
-			if (nb_rx > 0)
+			if (nb_rx > 0) {
+				core_stats_update_rx(nb_rx);
 				process_pkts(qconf, pkts, nb_rx, portid);
+			}
 
 			/* dequeue and process completed crypto-ops */
 			if (is_unprotected_port(portid))
examples/ipsec-secgw/ipsec-secgw.h +68 -0

84 lines not on this page · raw message

examples/ipsec-secgw/ipsec.c +10 -10
index bf88d80..351f1f1 100644
@@ -500,7 +500,7 @@ enqueue_cop_burst(struct cdev_qp *cqp)
 			cqp->id, cqp->qp, ret, len);
 			/* drop packets that we fail to enqueue */
 			for (i = ret; i < len; i++)
-				rte_pktmbuf_free(cqp->buf[i]->sym->m_src);
+				free_pkt(cqp->buf[i]->sym->m_src);
 	}
 	cqp->in_flight += ret;
 	cqp->len = 0;
@@ -528,7 +528,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 
 	for (i = 0; i < nb_pkts; i++) {
 		if (unlikely(sas[i] == NULL)) {
-			rte_pktmbuf_free(pkts[i]);
+			free_pkt(pkts[i]);
 			continue;
 		}
 
@@ -549,7 +549,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 
 			if ((unlikely(ips->security.ses == NULL)) &&
 				create_lookaside_session(ipsec_ctx, sa, ips)) {
-				rte_pktmbuf_free(pkts[i]);
+				free_pkt(pkts[i]);
 				continue;
 			}
 
@@ -563,7 +563,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 		case RTE_SECURITY_ACTION_TYPE_CPU_CRYPTO:
 			RTE_LOG(ERR, IPSEC, "CPU crypto is not supported by the"
 					" legacy mode.");
-			rte_pktmbuf_free(pkts[i]);
+			free_pkt(pkts[i]);
 			continue;
 
 		case RTE_SECURITY_ACTION_TYPE_NONE:
@@ -575,7 +575,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 
 			if ((unlikely(ips->crypto.ses == NULL)) &&
 				create_lookaside_session(ipsec_ctx, sa, ips)) {
-				rte_pktmbuf_free(pkts[i]);
+				free_pkt(pkts[i]);
 				continue;
 			}
 
@@ -584,7 +584,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 
 			ret = xform_func(pkts[i], sa, &priv->cop);
 			if (unlikely(ret)) {
-				rte_pktmbuf_free(pkts[i]);
+				free_pkt(pkts[i]);
 				continue;
 			}
 			break;
@@ -608,7 +608,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 
 			ret = xform_func(pkts[i], sa, &priv->cop);
 			if (unlikely(ret)) {
-				rte_pktmbuf_free(pkts[i]);
+				free_pkt(pkts[i]);
 				continue;
 			}
 
@@ -643,7 +643,7 @@ ipsec_inline_dequeue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 		sa = priv->sa;
 		ret = xform_func(pkt, sa, &priv->cop);
 		if (unlikely(ret)) {
-			rte_pktmbuf_free(pkt);
+			free_pkt(pkt);
 			continue;
 		}
 		pkts[nb_pkts++] = pkt;
@@ -690,13 +690,13 @@ ipsec_dequeue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 				RTE_SECURITY_ACTION_TYPE_NONE) {
 				ret = xform_func(pkt, sa, cops[j]);
 				if (unlikely(ret)) {
-					rte_pktmbuf_free(pkt);
+					free_pkt(pkt);
 					continue;
 				}
 			} else if (ipsec_get_action_type(sa) ==
 				RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) {
 				if (cops[j]->status) {
-					rte_pktmbuf_free(pkt);
+					free_pkt(pkt);
 					continue;
 				}
 			}
examples/ipsec-secgw/ipsec_process.c +1 -10
index bb2f2b8..4748299 100644
@@ -12,22 +12,13 @@
 #include <rte_mbuf.h>
 
 #include "ipsec.h"
+#include "ipsec-secgw.h"
 
 #define SATP_OUT_IPV4(t)	\
 	((((t) & RTE_IPSEC_SATP_MODE_MASK) == RTE_IPSEC_SATP_MODE_TRANS && \
 	(((t) & RTE_IPSEC_SATP_IPV_MASK) == RTE_IPSEC_SATP_IPV4)) || \
 	((t) & RTE_IPSEC_SATP_MODE_MASK) == RTE_IPSEC_SATP_MODE_TUNLV4)
 
-/* helper routine to free bulk of packets */
-static inline void
-free_pkts(struct rte_mbuf *mb[], uint32_t n)
-{
-	uint32_t i;
-
-	for (i = 0; i != n; i++)
-		rte_pktmbuf_free(mb[i]);
-}
-
 /* helper routine to free bulk of crypto-ops and related packets */
 static inline void
 free_cops(struct rte_crypto_op *cop[], uint32_t n)
Signature · 1 line
2.7.4
#6 lore raw
Quoted 18 lines
examples/ipsec-secgw/ipsec-secgw.c +7 -0
@@ -1099,6 +1151,10 @@ ipsec_poll_mode_worker(void)
 	const uint64_t drain_tsc = (rte_get_tsc_hz() + US_PER_S - 1)
 			/ US_PER_S * BURST_TX_DRAIN_US;
 	struct lcore_rx_queue *rxql;
+#if (STATS_INTERVAL > 0)
+	const uint64_t timer_period = STATS_INTERVAL * rte_get_timer_hz();
+	uint64_t timer_tsc = 0;
+#endif /* STATS_INTERVAL */
 
 	prev_tsc = 0;
 	lcore_id = rte_lcore_id();
@@ -1159,6 +1215,19 @@ ipsec_poll_mode_worker(void)
 			drain_tx_buffers(qconf);
 			drain_crypto_buffers(qconf);
 			prev_tsc = cur_tsc;
+#if (STATS_INTERVAL > 0)
+			if (lcore_id == rte_get_master_lcore()) {
+				/* advance the timer */

examples/ipsec-secgw/ipsec-secgw.c ipsec_poll_mode_worker(void) · line 1230

Quoted 10 lines
examples/ipsec-secgw/ipsec-secgw.c +10 -0
+				timer_tsc += diff_tsc;
+
+				/* if timer has reached its timeout */
+				if (unlikely(timer_tsc >= timer_period)) {
+					print_stats();
+					/* reset the timer */
+					timer_tsc = 0;
+				}
+			}
+#endif /* STATS_INTERVAL */

I still don't understand why to do it in data-path thread. As I said in previous comments, in DPDK there is a control thread that can be used for such house-keeping tasks. Why not to use it (via rte_alarm or so) and keep data-path threads less affected.

Quoted 20 lines
examples/ipsec-secgw/ipsec-secgw.c +3 -1
 		}
 
 		for (i = 0; i < qconf->nb_rx_queue; ++i) {
@@ -1169,8 +1238,10 @@ ipsec_poll_mode_worker(void)
 			nb_rx = rte_eth_rx_burst(portid, queueid,
 					pkts, MAX_PKT_BURST);
 
-			if (nb_rx > 0)
+			if (nb_rx > 0) {
+				core_stats_update_rx(nb_rx);
 				process_pkts(qconf, pkts, nb_rx, portid);
+			}
 
 			/* dequeue and process completed crypto-ops */
 			if (is_unprotected_port(portid))
examples/ipsec-secgw/ipsec-secgw.h +0 -0
index 4b53cb5..5b3561f 100644

examples/ipsec-secgw/ipsec-secgw.h line 9

Quoted 4 lines
examples/ipsec-secgw/ipsec-secgw.h +1 -0
@@ -6,6 +6,8 @@
 
 #include <stdbool.h>
 
+#define STATS_INTERVAL 0

Shouldn't it be:
#ifndef STATS_INTERVAL
#define STATS_INTERVAL 0
#endif
?

To allow user specify statis interval via EXTRA_CFLAGS='-DSTATS_INTERVAL=10' or so.

Quoted 69 lines
examples/ipsec-secgw/ipsec-secgw.h +55 -0
+
 #define NB_SOCKETS 4
 
 #define MAX_PKT_BURST 32
@@ -69,6 +71,17 @@ struct ethaddr_info {
 	uint64_t src, dst;
 };
 
+#if (STATS_INTERVAL > 0)
+struct ipsec_core_statistics {
+	uint64_t tx;
+	uint64_t rx;
+	uint64_t dropped;
+	uint64_t burst_rx;
+} __rte_cache_aligned;
+
+struct ipsec_core_statistics core_statistics[RTE_MAX_LCORE];
+#endif /* STATS_INTERVAL */
+
 extern struct ethaddr_info ethaddr_tbl[RTE_MAX_ETHPORTS];
 
 /* Port mask to identify the unprotected ports */
@@ -85,4 +98,59 @@ is_unprotected_port(uint16_t port_id)
 	return unprotected_port_mask & (1 << port_id);
 }
 
+static inline void
+core_stats_update_rx(int n)
+{
+#if (STATS_INTERVAL > 0)
+	int lcore_id = rte_lcore_id();
+	core_statistics[lcore_id].rx += n;
+	if (n == MAX_PKT_BURST)
+		core_statistics[lcore_id].burst_rx += n;
+#else
+	RTE_SET_USED(n);
+#endif /* STATS_INTERVAL */
+}
+
+static inline void
+core_stats_update_tx(int n)
+{
+#if (STATS_INTERVAL > 0)
+	int lcore_id = rte_lcore_id();
+	core_statistics[lcore_id].tx += n;
+#else
+	RTE_SET_USED(n);
+#endif /* STATS_INTERVAL */
+}
+
+static inline void
+core_stats_update_drop(int n)
+{
+#if (STATS_INTERVAL > 0)
+	int lcore_id = rte_lcore_id();
+	core_statistics[lcore_id].dropped += n;
+#else
+	RTE_SET_USED(n);
+#endif /* STATS_INTERVAL */
+}
+
+/* helper routine to free bulk of packets */
+static inline void
+free_pkts(struct rte_mbuf *mb[], uint32_t n)
+{
+	uint32_t i;
+
+	for (i = 0; i != n; i++)
+		rte_pktmbuf_free(mb[i]);

examples/ipsec-secgw/ipsec-secgw.h is_unprotected_port(uint16_t port_id) · line 153

Quoted 10 lines
examples/ipsec-secgw/ipsec-secgw.h +10 -0
+
+	core_stats_update_drop(n);
+}
+
+/* helper routine to free single packet */
+static inline void
+free_pkt(struct rte_mbuf *mb)
+{
+	rte_pktmbuf_free(mb);
+	core_stats_update_drop(1);

Probably just:
free_pkts(&mb, 1);
?

Quoted 3 lines
examples/ipsec-secgw/ipsec-secgw.h +2 -0
+}
+
 #endif /* _IPSEC_SECGW_H_ */
#7 lore raw

Hi Konstantin,

Please see inline.

Thanks,
Anoob

Quoted 36 lines

-----Original Message-----
From: Ananyev, Konstantin <[email protected]>
Sent: Thursday, May 7, 2020 9:42 PM
To: Anoob Joseph <[email protected]>; Akhil Goyal <[email protected]>; Nicolau, Radu <[email protected]>
Cc: Narayana Prasad Raju Athreya <[email protected]>; [email protected]
Subject: [EXT] RE: [PATCH v3] examples/ipsec-secgw: add per core packet stats

External Email

----------------------------------------------------------------------

Quoted 25 lines
examples/ipsec-secgw/ipsec-secgw.c +14 -0
@@ -1099,6 +1151,10 @@ ipsec_poll_mode_worker(void)
 	const uint64_t drain_tsc = (rte_get_tsc_hz() + US_PER_S - 1)
 			/ US_PER_S * BURST_TX_DRAIN_US;
 	struct lcore_rx_queue *rxql;
+#if (STATS_INTERVAL > 0)
+	const uint64_t timer_period = STATS_INTERVAL * rte_get_timer_hz();
+	uint64_t timer_tsc = 0;
+#endif /* STATS_INTERVAL */
 
 	prev_tsc = 0;
 	lcore_id = rte_lcore_id();
@@ -1159,6 +1215,19 @@ ipsec_poll_mode_worker(void)
 			drain_tx_buffers(qconf);
 			drain_crypto_buffers(qconf);
 			prev_tsc = cur_tsc;
+#if (STATS_INTERVAL > 0)
+			if (lcore_id == rte_get_master_lcore()) {
+				/* advance the timer */
+				timer_tsc += diff_tsc;
+
+				/* if timer has reached its timeout */
+				if (unlikely(timer_tsc >= timer_period)) {
+					print_stats();
+					/* reset the timer */
+					timer_tsc = 0;
Quoted 8 lines

examples/ipsec-secgw/ipsec-secgw.c adjust_ipv6_pktlen(struct rte_mbuf *m, const struct rte_ipv6_hdr *iph, · line 342

Quoted 3 lines
examples/ipsec-secgw/ipsec-secgw.c +3 -0
+				}
+			}
+#endif /* STATS_INTERVAL */

I still don't understand why to do it in data-path thread.
As I said in previous comments, in DPDK there is a control thread that can be used for such house-keeping tasks.
Why not to use it (via rte_alarm or so) and keep data-path threads less affected.

[Anoob] From Marvell's estimates, this stats collection and reporting will be expensive and so cannot be enabled by default. This is required for analyzing the traffic distribution in cases where the performance isn't scaling as expected. And this patch achieves the desired feature. If Intel would like to improve the approach, that can be taken up as a separate patch.

Quoted 26 lines

Review ipsec_poll_mode_worker(void) · line 1233

Quoted 4 lines
(quoted diff) +0 -0
 		}
 
 		for (i = 0; i < qconf->nb_rx_queue; ++i) { @@ -1169,8 +1238,10

@@

Quoted 21 lines

ipsec_poll_mode_worker(void)

examples/ipsec-secgw/ipsec-secgw.c +3 -1
 			nb_rx = rte_eth_rx_burst(portid, queueid,
 					pkts, MAX_PKT_BURST);
 
-			if (nb_rx > 0)
+			if (nb_rx > 0) {
+				core_stats_update_rx(nb_rx);
 				process_pkts(qconf, pkts, nb_rx, portid);
+			}
 
 			/* dequeue and process completed crypto-ops */
 			if (is_unprotected_port(portid))
examples/ipsec-secgw/ipsec-secgw.h +0 -0
index 4b53cb5..5b3561f 100644
@@ -6,6 +6,8 @@
 
 #include <stdbool.h>
 
Quoted 7 lines

examples/ipsec-secgw/ipsec-secgw.h line 9

Quoted 1 line
examples/ipsec-secgw/ipsec-secgw.h +1 -0
+#define STATS_INTERVAL 0

Shouldn't it be:
#ifndef STATS_INTERVAL
#define STATS_INTERVAL 0
#endif
?

[Anoob] Will update in v4.

Quoted 72 lines

To allow user specify statis interval via EXTRA_CFLAGS='-DSTATS_INTERVAL=10' or so.

Quoted 68 lines
examples/ipsec-secgw/ipsec-secgw.h +12 -0
+
 #define NB_SOCKETS 4
 
 #define MAX_PKT_BURST 32
@@ -69,6 +71,17 @@ struct ethaddr_info {
 	uint64_t src, dst;
 };
 
+#if (STATS_INTERVAL > 0)
+struct ipsec_core_statistics {
+	uint64_t tx;
+	uint64_t rx;
+	uint64_t dropped;
+	uint64_t burst_rx;
+} __rte_cache_aligned;
+
+struct ipsec_core_statistics core_statistics[RTE_MAX_LCORE]; #endif
+/* STATS_INTERVAL */
+
 extern struct ethaddr_info ethaddr_tbl[RTE_MAX_ETHPORTS];
 
 /* Port mask to identify the unprotected ports */ @@ -85,4 +98,59 @@

is_unprotected_port(uint16_t port_id)

examples/ipsec-secgw/ipsec-secgw.h +43 -0
 	return unprotected_port_mask & (1 << port_id);  }
 
+static inline void
+core_stats_update_rx(int n)
+{
+#if (STATS_INTERVAL > 0)
+	int lcore_id = rte_lcore_id();
+	core_statistics[lcore_id].rx += n;
+	if (n == MAX_PKT_BURST)
+		core_statistics[lcore_id].burst_rx += n; #else
+	RTE_SET_USED(n);
+#endif /* STATS_INTERVAL */
+}
+
+static inline void
+core_stats_update_tx(int n)
+{
+#if (STATS_INTERVAL > 0)
+	int lcore_id = rte_lcore_id();
+	core_statistics[lcore_id].tx += n;
+#else
+	RTE_SET_USED(n);
+#endif /* STATS_INTERVAL */
+}
+
+static inline void
+core_stats_update_drop(int n)
+{
+#if (STATS_INTERVAL > 0)
+	int lcore_id = rte_lcore_id();
+	core_statistics[lcore_id].dropped += n; #else
+	RTE_SET_USED(n);
+#endif /* STATS_INTERVAL */
+}
+
+/* helper routine to free bulk of packets */ static inline void
+free_pkts(struct rte_mbuf *mb[], uint32_t n) {
+	uint32_t i;
+
+	for (i = 0; i != n; i++)
+		rte_pktmbuf_free(mb[i]);
+
+	core_stats_update_drop(n);
+}
+
Quoted 8 lines

examples/ipsec-secgw/ipsec-secgw.h is_unprotected_port(uint16_t port_id) · line 153

Quoted 4 lines
examples/ipsec-secgw/ipsec-secgw.h +4 -0
+/* helper routine to free single packet */ static inline void
+free_pkt(struct rte_mbuf *mb) {
+	rte_pktmbuf_free(mb);
+	core_stats_update_drop(1);

Probably just:
free_pkts(&mb, 1);
?

[Anoob] Will update in v4.

Quoted 4 lines
Quoted 4 lines
examples/ipsec-secgw/ipsec-secgw.h +2 -0
+}
+
 #endif /* _IPSEC_SECGW_H_ */
#8 lore raw
Quoted 35 lines
Quoted 26 lines
Quoted 26 lines
examples/ipsec-secgw/ipsec-secgw.c +14 -0
@@ -1099,6 +1151,10 @@ ipsec_poll_mode_worker(void)
 	const uint64_t drain_tsc = (rte_get_tsc_hz() + US_PER_S - 1)
 			/ US_PER_S * BURST_TX_DRAIN_US;
 	struct lcore_rx_queue *rxql;
+#if (STATS_INTERVAL > 0)
+	const uint64_t timer_period = STATS_INTERVAL * rte_get_timer_hz();
+	uint64_t timer_tsc = 0;
+#endif /* STATS_INTERVAL */
 
 	prev_tsc = 0;
 	lcore_id = rte_lcore_id();
@@ -1159,6 +1215,19 @@ ipsec_poll_mode_worker(void)
 			drain_tx_buffers(qconf);
 			drain_crypto_buffers(qconf);
 			prev_tsc = cur_tsc;
+#if (STATS_INTERVAL > 0)
+			if (lcore_id == rte_get_master_lcore()) {
+				/* advance the timer */
+				timer_tsc += diff_tsc;
+
+				/* if timer has reached its timeout */
+				if (unlikely(timer_tsc >= timer_period)) {
+					print_stats();
+					/* reset the timer */
+					timer_tsc = 0;
Quoted 8 lines

examples/ipsec-secgw/ipsec-secgw.c adjust_ipv6_pktlen(struct rte_mbuf *m, const struct rte_ipv6_hdr *iph, · line 342

Quoted 3 lines
examples/ipsec-secgw/ipsec-secgw.c +3 -0
+				}
+			}
+#endif /* STATS_INTERVAL */

I still don't understand why to do it in data-path thread.
As I said in previous comments, in DPDK there is a control thread that can be used for such house-keeping tasks.
Why not to use it (via rte_alarm or so) and keep data-path threads less affected.

Quoted 2 lines

[Anoob] From Marvell's estimates, this stats collection and reporting will be expensive and so cannot be enabled by default. This is required for analyzing the traffic distribution in cases where the performance isn't scaling as expected.

Understood.

Quoted 1 line

And this patch achieves the desired feature.

Ok, but why not to do it in control (house-keeping) thread? That would achieve desired goal and keep data-path unaffected.

Quoted 1 line

If Intel would like to improve the approach, that can be taken up as a separate patch.

This is not a vendor specific part.
You making changes in common data-path code that is used by all ipsec-secgw users. I think it is everyone benefit (and responsibility) to keep common data-path code clean, tidy and fast.
If we can avoid polluting it with extra code, I don't see a reason not to do it.

Quoted 124 lines
Quoted 27 lines

Review ipsec_poll_mode_worker(void) · line 1233

Quoted 5 lines
(quoted diff) +0 -0
 		}
 
 		for (i = 0; i < qconf->nb_rx_queue; ++i) { @@ -1169,8 +1238,10

@@

Quoted 21 lines

ipsec_poll_mode_worker(void)

examples/ipsec-secgw/ipsec-secgw.c +3 -1
 			nb_rx = rte_eth_rx_burst(portid, queueid,
 					pkts, MAX_PKT_BURST);
 
-			if (nb_rx > 0)
+			if (nb_rx > 0) {
+				core_stats_update_rx(nb_rx);
 				process_pkts(qconf, pkts, nb_rx, portid);
+			}
 
 			/* dequeue and process completed crypto-ops */
 			if (is_unprotected_port(portid))
examples/ipsec-secgw/ipsec-secgw.h +0 -0
index 4b53cb5..5b3561f 100644
@@ -6,6 +6,8 @@
 
 #include <stdbool.h>
 
Quoted 7 lines

examples/ipsec-secgw/ipsec-secgw.h line 9

Quoted 1 line
examples/ipsec-secgw/ipsec-secgw.h +1 -0
+#define STATS_INTERVAL 0

Shouldn't it be:
#ifndef STATS_INTERVAL
#define STATS_INTERVAL 0
#endif
?

[Anoob] Will update in v4.

Quoted 72 lines

To allow user specify statis interval via EXTRA_CFLAGS='-DSTATS_INTERVAL=10' or so.

Quoted 68 lines
examples/ipsec-secgw/ipsec-secgw.h +12 -0
+
 #define NB_SOCKETS 4
 
 #define MAX_PKT_BURST 32
@@ -69,6 +71,17 @@ struct ethaddr_info {
 	uint64_t src, dst;
 };
 
+#if (STATS_INTERVAL > 0)
+struct ipsec_core_statistics {
+	uint64_t tx;
+	uint64_t rx;
+	uint64_t dropped;
+	uint64_t burst_rx;
+} __rte_cache_aligned;
+
+struct ipsec_core_statistics core_statistics[RTE_MAX_LCORE]; #endif
+/* STATS_INTERVAL */
+
 extern struct ethaddr_info ethaddr_tbl[RTE_MAX_ETHPORTS];
 
 /* Port mask to identify the unprotected ports */ @@ -85,4 +98,59 @@

is_unprotected_port(uint16_t port_id)

examples/ipsec-secgw/ipsec-secgw.h +43 -0
 	return unprotected_port_mask & (1 << port_id);  }
 
+static inline void
+core_stats_update_rx(int n)
+{
+#if (STATS_INTERVAL > 0)
+	int lcore_id = rte_lcore_id();
+	core_statistics[lcore_id].rx += n;
+	if (n == MAX_PKT_BURST)
+		core_statistics[lcore_id].burst_rx += n; #else
+	RTE_SET_USED(n);
+#endif /* STATS_INTERVAL */
+}
+
+static inline void
+core_stats_update_tx(int n)
+{
+#if (STATS_INTERVAL > 0)
+	int lcore_id = rte_lcore_id();
+	core_statistics[lcore_id].tx += n;
+#else
+	RTE_SET_USED(n);
+#endif /* STATS_INTERVAL */
+}
+
+static inline void
+core_stats_update_drop(int n)
+{
+#if (STATS_INTERVAL > 0)
+	int lcore_id = rte_lcore_id();
+	core_statistics[lcore_id].dropped += n; #else
+	RTE_SET_USED(n);
+#endif /* STATS_INTERVAL */
+}
+
+/* helper routine to free bulk of packets */ static inline void
+free_pkts(struct rte_mbuf *mb[], uint32_t n) {
+	uint32_t i;
+
+	for (i = 0; i != n; i++)
+		rte_pktmbuf_free(mb[i]);
+
+	core_stats_update_drop(n);
+}
+
Quoted 8 lines

examples/ipsec-secgw/ipsec-secgw.h is_unprotected_port(uint16_t port_id) · line 153

Quoted 4 lines
examples/ipsec-secgw/ipsec-secgw.h +4 -0
+/* helper routine to free single packet */ static inline void
+free_pkt(struct rte_mbuf *mb) {
+	rte_pktmbuf_free(mb);
+	core_stats_update_drop(1);

Probably just:
free_pkts(&mb, 1);
?

[Anoob] Will update in v4.

Quoted 4 lines
Quoted 4 lines
examples/ipsec-secgw/ipsec-secgw.h +2 -0
+}
+
 #endif /* _IPSEC_SECGW_H_ */
#9 lore raw

Hi Konstantin,

Please see inline.

Thanks,
Anoob

Quoted 60 lines

-----Original Message-----
From: Ananyev, Konstantin <[email protected]>
Sent: Wednesday, May 13, 2020 6:13 PM
To: Anoob Joseph <[email protected]>; Akhil Goyal <[email protected]>; Nicolau, Radu <[email protected]>
Cc: Narayana Prasad Raju Athreya <[email protected]>; [email protected]
Subject: [EXT] RE: [PATCH v3] examples/ipsec-secgw: add per core packet stats

External Email

----------------------------------------------------------------------

Quoted 25 lines
Quoted 25 lines
Quoted 25 lines
examples/ipsec-secgw/ipsec-secgw.c +14 -0
@@ -1099,6 +1151,10 @@ ipsec_poll_mode_worker(void)
 	const uint64_t drain_tsc = (rte_get_tsc_hz() + US_PER_S - 1)
 			/ US_PER_S * BURST_TX_DRAIN_US;
 	struct lcore_rx_queue *rxql;
+#if (STATS_INTERVAL > 0)
+	const uint64_t timer_period = STATS_INTERVAL * rte_get_timer_hz();
+	uint64_t timer_tsc = 0;
+#endif /* STATS_INTERVAL */
 
 	prev_tsc = 0;
 	lcore_id = rte_lcore_id();
@@ -1159,6 +1215,19 @@ ipsec_poll_mode_worker(void)
 			drain_tx_buffers(qconf);
 			drain_crypto_buffers(qconf);
 			prev_tsc = cur_tsc;
+#if (STATS_INTERVAL > 0)
+			if (lcore_id == rte_get_master_lcore()) {
+				/* advance the timer */
+				timer_tsc += diff_tsc;
+
+				/* if timer has reached its timeout */
+				if (unlikely(timer_tsc >= timer_period)) {
+					print_stats();
+					/* reset the timer */
+					timer_tsc = 0;
Quoted 8 lines
Quoted 8 lines

examples/ipsec-secgw/ipsec-secgw.c adjust_ipv6_pktlen(struct rte_mbuf *m, const struct rte_ipv6_hdr *iph, · line 342

Quoted 3 lines
examples/ipsec-secgw/ipsec-secgw.c +3 -0
+				}
+			}
+#endif /* STATS_INTERVAL */

I still don't understand why to do it in data-path thread. As I said in previous comments, in DPDK there is a control thread that can be used for such house-keeping tasks.
Why not to use it (via rte_alarm or so) and keep data-path threads less

affected.

Quoted 3 lines

[Anoob] From Marvell's estimates, this stats collection and reporting will be expensive and so cannot be enabled by default. This is required for

analyzing the traffic distribution in cases where the performance isn't scaling as expected.

Understood.

Quoted 1 line

And this patch achieves the desired feature.

Ok, but why not to do it in control (house-keeping) thread? That would achieve desired goal and keep data-path unaffected.

Quoted 1 line

If Intel would like to improve the approach, that can be taken up as a separate

Quoted 8 lines

patch.

This is not a vendor specific part.
You making changes in common data-path code that is used by all ipsec-secgw users.
I think it is everyone benefit (and responsibility) to keep common data-path code clean, tidy and fast.
If we can avoid polluting it with extra code, I don't see a reason not to do it.

[Anoob] I cannot say I've fully understood what you have suggested. I've submitted v4 based on what I understood from your comments. Please have a look at it.

Quoted 126 lines
Quoted 36 lines
Quoted 28 lines

Review ipsec_poll_mode_worker(void) · line 1233

Quoted 6 lines
(quoted diff) +0 -0
 		}
 
 		for (i = 0; i < qconf->nb_rx_queue; ++i) { @@ -1169,8 +1238,10

@@

Quoted 21 lines

ipsec_poll_mode_worker(void)

examples/ipsec-secgw/ipsec-secgw.c +3 -1
 			nb_rx = rte_eth_rx_burst(portid, queueid,
 					pkts, MAX_PKT_BURST);
 
-			if (nb_rx > 0)
+			if (nb_rx > 0) {
+				core_stats_update_rx(nb_rx);
 				process_pkts(qconf, pkts, nb_rx, portid);
+			}
 
 			/* dequeue and process completed crypto-ops */
 			if (is_unprotected_port(portid)) diff --git

a/examples/ipsec-secgw/ipsec-secgw.h
b/examples/ipsec-secgw/ipsec-secgw.h
index 4b53cb5..5b3561f 100644

examples/ipsec-secgw/ipsec-secgw.h +0 -0
@@ -6,6 +6,8 @@
 
 #include <stdbool.h>
 
Quoted 7 lines

examples/ipsec-secgw/ipsec-secgw.h line 9

Quoted 1 line
examples/ipsec-secgw/ipsec-secgw.h +1 -0
+#define STATS_INTERVAL 0

Shouldn't it be:
#ifndef STATS_INTERVAL
#define STATS_INTERVAL 0
#endif
?

Quoted 4 lines

[Anoob] Will update in v4.

Quoted 2 lines

To allow user specify statis interval via EXTRA_CFLAGS='-

DSTATS_INTERVAL=10'

Quoted 85 lines
Quoted 70 lines

or so.

Quoted 68 lines
examples/ipsec-secgw/ipsec-secgw.h +13 -0
+
 #define NB_SOCKETS 4
 
 #define MAX_PKT_BURST 32
@@ -69,6 +71,17 @@ struct ethaddr_info {
 	uint64_t src, dst;
 };
 
+#if (STATS_INTERVAL > 0)
+struct ipsec_core_statistics {
+	uint64_t tx;
+	uint64_t rx;
+	uint64_t dropped;
+	uint64_t burst_rx;
+} __rte_cache_aligned;
+
+struct ipsec_core_statistics core_statistics[RTE_MAX_LCORE];
+#endif
+/* STATS_INTERVAL */
+
 extern struct ethaddr_info ethaddr_tbl[RTE_MAX_ETHPORTS];
 
 /* Port mask to identify the unprotected ports */ @@ -85,4 +98,59

@@ is_unprotected_port(uint16_t port_id)

examples/ipsec-secgw/ipsec-secgw.h +42 -0
 	return unprotected_port_mask & (1 << port_id);  }
 
+static inline void
+core_stats_update_rx(int n)
+{
+#if (STATS_INTERVAL > 0)
+	int lcore_id = rte_lcore_id();
+	core_statistics[lcore_id].rx += n;
+	if (n == MAX_PKT_BURST)
+		core_statistics[lcore_id].burst_rx += n; #else
+	RTE_SET_USED(n);
+#endif /* STATS_INTERVAL */
+}
+
+static inline void
+core_stats_update_tx(int n)
+{
+#if (STATS_INTERVAL > 0)
+	int lcore_id = rte_lcore_id();
+	core_statistics[lcore_id].tx += n; #else
+	RTE_SET_USED(n);
+#endif /* STATS_INTERVAL */
+}
+
+static inline void
+core_stats_update_drop(int n)
+{
+#if (STATS_INTERVAL > 0)
+	int lcore_id = rte_lcore_id();
+	core_statistics[lcore_id].dropped += n; #else
+	RTE_SET_USED(n);
+#endif /* STATS_INTERVAL */
+}
+
+/* helper routine to free bulk of packets */ static inline void
+free_pkts(struct rte_mbuf *mb[], uint32_t n) {
+	uint32_t i;
+
+	for (i = 0; i != n; i++)
+		rte_pktmbuf_free(mb[i]);
+
+	core_stats_update_drop(n);
+}
+
Quoted 8 lines

examples/ipsec-secgw/ipsec-secgw.h is_unprotected_port(uint16_t port_id) · line 153

Quoted 4 lines
examples/ipsec-secgw/ipsec-secgw.h +4 -0
+/* helper routine to free single packet */ static inline void
+free_pkt(struct rte_mbuf *mb) {
+	rte_pktmbuf_free(mb);
+	core_stats_update_drop(1);

Probably just:
free_pkts(&mb, 1);
?

[Anoob] Will update in v4.

Quoted 4 lines
Quoted 4 lines
examples/ipsec-secgw/ipsec-secgw.h +2 -0
+}
+
 #endif /* _IPSEC_SECGW_H_ */
#10 lore raw
In reply to Anoob Joseph
Quoted 29 lines
examples/ipsec-secgw/ipsec-secgw.c +27 -0
+#if (STATS_INTERVAL > 0)
+
+/* Print out statistics on packet distribution */
+static void
+print_stats(void)
+{
+	uint64_t total_packets_dropped, total_packets_tx, total_packets_rx;
+	unsigned int coreid;
+	float burst_percent;
+
+	total_packets_dropped = 0;
+	total_packets_tx = 0;
+	total_packets_rx = 0;
+
+	const char clr[] = { 27, '[', '2', 'J', '\0' };
+
+	/* Clear screen and move to top left */
+	printf("%s", clr);
+
+	printf("\nCore statistics ====================================");
+
+	for (coreid = 0; coreid < RTE_MAX_LCORE; coreid++) {
+		/* skip disabled cores */
+		if (rte_lcore_is_enabled(coreid) == 0)
+			continue;
+		burst_percent = (float)(core_statistics[coreid].burst_rx * 100)/
+					core_statistics[coreid].rx;

examples/ipsec-secgw/ipsec-secgw.c adjust_ipv6_pktlen(struct rte_mbuf *m, const struct rte_ipv6_hdr *iph, · line 327

Quoted 10 lines
examples/ipsec-secgw/ipsec-secgw.c +10 -0
+		printf("\nStatistics for core %u ------------------------------"
+			   "\nPackets received: %20"PRIu64
+			   "\nPackets sent: %24"PRIu64
+			   "\nPackets dropped: %21"PRIu64
+			   "\nBurst percent: %23.2f",
+			   coreid,
+			   core_statistics[coreid].rx,
+			   core_statistics[coreid].tx,
+			   core_statistics[coreid].dropped,
+			   burst_percent);

As one more comment:
Can we also collect number of calls and display average pkt/call (for both rx and tx)?

Quoted 421 lines
examples/ipsec-secgw/ipsec-secgw.c +55 -21
+
+		total_packets_dropped += core_statistics[coreid].dropped;
+		total_packets_tx += core_statistics[coreid].tx;
+		total_packets_rx += core_statistics[coreid].rx;
+	}
+	printf("\nAggregate statistics ==============================="
+		   "\nTotal packets received: %14"PRIu64
+		   "\nTotal packets sent: %18"PRIu64
+		   "\nTotal packets dropped: %15"PRIu64,
+		   total_packets_rx,
+		   total_packets_tx,
+		   total_packets_dropped);
+	printf("\n====================================================\n");
+}
+#endif /* STATS_INTERVAL */
+
 static inline void
 prepare_one_packet(struct rte_mbuf *pkt, struct ipsec_traffic *t)
 {
@@ -333,7 +386,7 @@ prepare_one_packet(struct rte_mbuf *pkt, struct ipsec_traffic *t)
 
 		/* drop packet when IPv6 header exceeds first segment length */
 		if (unlikely(l3len > pkt->data_len)) {
-			rte_pktmbuf_free(pkt);
+			free_pkt(pkt);
 			return;
 		}
 
@@ -350,7 +403,7 @@ prepare_one_packet(struct rte_mbuf *pkt, struct ipsec_traffic *t)
 		/* Unknown/Unsupported type, drop the packet */
 		RTE_LOG(ERR, IPSEC, "Unsupported packet type 0x%x\n",
 			rte_be_to_cpu_16(eth->ether_type));
-		rte_pktmbuf_free(pkt);
+		free_pkt(pkt);
 		return;
 	}
 
@@ -477,9 +530,12 @@ send_burst(struct lcore_conf *qconf, uint16_t n, uint16_t port)
 	prepare_tx_burst(m_table, n, port, qconf);
 
 	ret = rte_eth_tx_burst(port, queueid, m_table, n);
+
+	core_stats_update_tx(ret);
+
 	if (unlikely(ret < n)) {
 		do {
-			rte_pktmbuf_free(m_table[ret]);
+			free_pkt(m_table[ret]);
 		} while (++ret < n);
 	}
 
@@ -525,7 +581,7 @@ send_fragment_packet(struct lcore_conf *qconf, struct rte_mbuf *m,
 			"error code: %d\n",
 			__func__, m->pkt_len, rte_errno);
 
-	rte_pktmbuf_free(m);
+	free_pkt(m);
 	return len;
 }
 
@@ -550,7 +606,7 @@ send_single_packet(struct rte_mbuf *m, uint16_t port, uint8_t proto)
 	} else if (frag_tbl_sz > 0)
 		len = send_fragment_packet(qconf, m, port, proto);
 	else
-		rte_pktmbuf_free(m);
+		free_pkt(m);
 
 	/* enough pkts to be sent */
 	if (unlikely(len == MAX_PKT_BURST)) {
@@ -584,19 +640,19 @@ inbound_sp_sa(struct sp_ctx *sp, struct sa_ctx *sa, struct traffic_type *ip,
 			continue;
 		}
 		if (res == DISCARD) {
-			rte_pktmbuf_free(m);
+			free_pkt(m);
 			continue;
 		}
 
 		/* Only check SPI match for processed IPSec packets */
 		if (i < lim && ((m->ol_flags & PKT_RX_SEC_OFFLOAD) == 0)) {
-			rte_pktmbuf_free(m);
+			free_pkt(m);
 			continue;
 		}
 
 		sa_idx = res - 1;
 		if (!inbound_sa_check(sa, m, sa_idx)) {
-			rte_pktmbuf_free(m);
+			free_pkt(m);
 			continue;
 		}
 		ip->pkts[j++] = m;
@@ -631,7 +687,7 @@ split46_traffic(struct ipsec_traffic *trf, struct rte_mbuf *mb[], uint32_t num)
 					offsetof(struct ip6_hdr, ip6_nxt));
 			n6++;
 		} else
-			rte_pktmbuf_free(m);
+			free_pkt(m);
 	}
 
 	trf->ip4.num = n4;
@@ -683,7 +739,7 @@ outbound_sp(struct sp_ctx *sp, struct traffic_type *ip,
 		m = ip->pkts[i];
 		sa_idx = ip->res[i] - 1;
 		if (ip->res[i] == DISCARD)
-			rte_pktmbuf_free(m);
+			free_pkt(m);
 		else if (ip->res[i] == BYPASS)
 			ip->pkts[j++] = m;
 		else {
@@ -702,8 +758,7 @@ process_pkts_outbound(struct ipsec_ctx *ipsec_ctx,
 	uint16_t idx, nb_pkts_out, i;
 
 	/* Drop any IPsec traffic from protected ports */
-	for (i = 0; i < traffic->ipsec.num; i++)
-		rte_pktmbuf_free(traffic->ipsec.pkts[i]);
+	free_pkts(traffic->ipsec.pkts, traffic->ipsec.num);
 
 	traffic->ipsec.num = 0;
 
@@ -743,14 +798,12 @@ process_pkts_inbound_nosp(struct ipsec_ctx *ipsec_ctx,
 	uint32_t nb_pkts_in, i, idx;
 
 	/* Drop any IPv4 traffic from unprotected ports */
-	for (i = 0; i < traffic->ip4.num; i++)
-		rte_pktmbuf_free(traffic->ip4.pkts[i]);
+	free_pkts(traffic->ip4.pkts, traffic->ip4.num);
 
 	traffic->ip4.num = 0;
 
 	/* Drop any IPv6 traffic from unprotected ports */
-	for (i = 0; i < traffic->ip6.num; i++)
-		rte_pktmbuf_free(traffic->ip6.pkts[i]);
+	free_pkts(traffic->ip6.pkts, traffic->ip6.num);
 
 	traffic->ip6.num = 0;
 
@@ -786,8 +839,7 @@ process_pkts_outbound_nosp(struct ipsec_ctx *ipsec_ctx,
 	struct ip *ip;
 
 	/* Drop any IPsec traffic from protected ports */
-	for (i = 0; i < traffic->ipsec.num; i++)
-		rte_pktmbuf_free(traffic->ipsec.pkts[i]);
+	free_pkts(traffic->ipsec.pkts, traffic->ipsec.num);
 
 	n = 0;
 
@@ -901,7 +953,7 @@ route4_pkts(struct rt_ctx *rt_ctx, struct rte_mbuf *pkts[], uint8_t nb_pkts)
 		}
 
 		if ((pkt_hop & RTE_LPM_LOOKUP_SUCCESS) == 0) {
-			rte_pktmbuf_free(pkts[i]);
+			free_pkt(pkts[i]);
 			continue;
 		}
 		send_single_packet(pkts[i], pkt_hop & 0xff, IPPROTO_IP);
@@ -953,7 +1005,7 @@ route6_pkts(struct rt_ctx *rt_ctx, struct rte_mbuf *pkts[], uint8_t nb_pkts)
 		}
 
 		if (pkt_hop == -1) {
-			rte_pktmbuf_free(pkts[i]);
+			free_pkt(pkts[i]);
 			continue;
 		}
 		send_single_packet(pkts[i], pkt_hop & 0xff, IPPROTO_IPV6);
@@ -1099,6 +1151,10 @@ ipsec_poll_mode_worker(void)
 	const uint64_t drain_tsc = (rte_get_tsc_hz() + US_PER_S - 1)
 			/ US_PER_S * BURST_TX_DRAIN_US;
 	struct lcore_rx_queue *rxql;
+#if (STATS_INTERVAL > 0)
+	const uint64_t timer_period = STATS_INTERVAL * rte_get_timer_hz();
+	uint64_t timer_tsc = 0;
+#endif /* STATS_INTERVAL */
 
 	prev_tsc = 0;
 	lcore_id = rte_lcore_id();
@@ -1159,6 +1215,19 @@ ipsec_poll_mode_worker(void)
 			drain_tx_buffers(qconf);
 			drain_crypto_buffers(qconf);
 			prev_tsc = cur_tsc;
+#if (STATS_INTERVAL > 0)
+			if (lcore_id == rte_get_master_lcore()) {
+				/* advance the timer */
+				timer_tsc += diff_tsc;
+
+				/* if timer has reached its timeout */
+				if (unlikely(timer_tsc >= timer_period)) {
+					print_stats();
+					/* reset the timer */
+					timer_tsc = 0;
+				}
+			}
+#endif /* STATS_INTERVAL */
 		}
 
 		for (i = 0; i < qconf->nb_rx_queue; ++i) {
@@ -1169,8 +1238,10 @@ ipsec_poll_mode_worker(void)
 			nb_rx = rte_eth_rx_burst(portid, queueid,
 					pkts, MAX_PKT_BURST);
 
-			if (nb_rx > 0)
+			if (nb_rx > 0) {
+				core_stats_update_rx(nb_rx);
 				process_pkts(qconf, pkts, nb_rx, portid);
+			}
 
 			/* dequeue and process completed crypto-ops */
 			if (is_unprotected_port(portid))
examples/ipsec-secgw/ipsec-secgw.h +68 -0
index 4b53cb5..5b3561f 100644
@@ -6,6 +6,8 @@
 
 #include <stdbool.h>
 
+#define STATS_INTERVAL 0
+
 #define NB_SOCKETS 4
 
 #define MAX_PKT_BURST 32
@@ -69,6 +71,17 @@ struct ethaddr_info {
 	uint64_t src, dst;
 };
 
+#if (STATS_INTERVAL > 0)
+struct ipsec_core_statistics {
+	uint64_t tx;
+	uint64_t rx;
+	uint64_t dropped;
+	uint64_t burst_rx;
+} __rte_cache_aligned;
+
+struct ipsec_core_statistics core_statistics[RTE_MAX_LCORE];
+#endif /* STATS_INTERVAL */
+
 extern struct ethaddr_info ethaddr_tbl[RTE_MAX_ETHPORTS];
 
 /* Port mask to identify the unprotected ports */
@@ -85,4 +98,59 @@ is_unprotected_port(uint16_t port_id)
 	return unprotected_port_mask & (1 << port_id);
 }
 
+static inline void
+core_stats_update_rx(int n)
+{
+#if (STATS_INTERVAL > 0)
+	int lcore_id = rte_lcore_id();
+	core_statistics[lcore_id].rx += n;
+	if (n == MAX_PKT_BURST)
+		core_statistics[lcore_id].burst_rx += n;
+#else
+	RTE_SET_USED(n);
+#endif /* STATS_INTERVAL */
+}
+
+static inline void
+core_stats_update_tx(int n)
+{
+#if (STATS_INTERVAL > 0)
+	int lcore_id = rte_lcore_id();
+	core_statistics[lcore_id].tx += n;
+#else
+	RTE_SET_USED(n);
+#endif /* STATS_INTERVAL */
+}
+
+static inline void
+core_stats_update_drop(int n)
+{
+#if (STATS_INTERVAL > 0)
+	int lcore_id = rte_lcore_id();
+	core_statistics[lcore_id].dropped += n;
+#else
+	RTE_SET_USED(n);
+#endif /* STATS_INTERVAL */
+}
+
+/* helper routine to free bulk of packets */
+static inline void
+free_pkts(struct rte_mbuf *mb[], uint32_t n)
+{
+	uint32_t i;
+
+	for (i = 0; i != n; i++)
+		rte_pktmbuf_free(mb[i]);
+
+	core_stats_update_drop(n);
+}
+
+/* helper routine to free single packet */
+static inline void
+free_pkt(struct rte_mbuf *mb)
+{
+	rte_pktmbuf_free(mb);
+	core_stats_update_drop(1);
+}
+
 #endif /* _IPSEC_SECGW_H_ */
examples/ipsec-secgw/ipsec.c +10 -10
index bf88d80..351f1f1 100644
@@ -500,7 +500,7 @@ enqueue_cop_burst(struct cdev_qp *cqp)
 			cqp->id, cqp->qp, ret, len);
 			/* drop packets that we fail to enqueue */
 			for (i = ret; i < len; i++)
-				rte_pktmbuf_free(cqp->buf[i]->sym->m_src);
+				free_pkt(cqp->buf[i]->sym->m_src);
 	}
 	cqp->in_flight += ret;
 	cqp->len = 0;
@@ -528,7 +528,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 
 	for (i = 0; i < nb_pkts; i++) {
 		if (unlikely(sas[i] == NULL)) {
-			rte_pktmbuf_free(pkts[i]);
+			free_pkt(pkts[i]);
 			continue;
 		}
 
@@ -549,7 +549,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 
 			if ((unlikely(ips->security.ses == NULL)) &&
 				create_lookaside_session(ipsec_ctx, sa, ips)) {
-				rte_pktmbuf_free(pkts[i]);
+				free_pkt(pkts[i]);
 				continue;
 			}
 
@@ -563,7 +563,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 		case RTE_SECURITY_ACTION_TYPE_CPU_CRYPTO:
 			RTE_LOG(ERR, IPSEC, "CPU crypto is not supported by the"
 					" legacy mode.");
-			rte_pktmbuf_free(pkts[i]);
+			free_pkt(pkts[i]);
 			continue;
 
 		case RTE_SECURITY_ACTION_TYPE_NONE:
@@ -575,7 +575,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 
 			if ((unlikely(ips->crypto.ses == NULL)) &&
 				create_lookaside_session(ipsec_ctx, sa, ips)) {
-				rte_pktmbuf_free(pkts[i]);
+				free_pkt(pkts[i]);
 				continue;
 			}
 
@@ -584,7 +584,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 
 			ret = xform_func(pkts[i], sa, &priv->cop);
 			if (unlikely(ret)) {
-				rte_pktmbuf_free(pkts[i]);
+				free_pkt(pkts[i]);
 				continue;
 			}
 			break;
@@ -608,7 +608,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 
 			ret = xform_func(pkts[i], sa, &priv->cop);
 			if (unlikely(ret)) {
-				rte_pktmbuf_free(pkts[i]);
+				free_pkt(pkts[i]);
 				continue;
 			}
 
@@ -643,7 +643,7 @@ ipsec_inline_dequeue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 		sa = priv->sa;
 		ret = xform_func(pkt, sa, &priv->cop);
 		if (unlikely(ret)) {
-			rte_pktmbuf_free(pkt);
+			free_pkt(pkt);
 			continue;
 		}
 		pkts[nb_pkts++] = pkt;
@@ -690,13 +690,13 @@ ipsec_dequeue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 				RTE_SECURITY_ACTION_TYPE_NONE) {
 				ret = xform_func(pkt, sa, cops[j]);
 				if (unlikely(ret)) {
-					rte_pktmbuf_free(pkt);
+					free_pkt(pkt);
 					continue;
 				}
 			} else if (ipsec_get_action_type(sa) ==
 				RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) {
 				if (cops[j]->status) {
-					rte_pktmbuf_free(pkt);
+					free_pkt(pkt);
 					continue;
 				}
 			}
examples/ipsec-secgw/ipsec_process.c +1 -11
index bb2f2b8..4748299 100644
@@ -12,22 +12,13 @@
 #include <rte_mbuf.h>
 
 #include "ipsec.h"
+#include "ipsec-secgw.h"
 
 #define SATP_OUT_IPV4(t)	\
 	((((t) & RTE_IPSEC_SATP_MODE_MASK) == RTE_IPSEC_SATP_MODE_TRANS && \
 	(((t) & RTE_IPSEC_SATP_IPV_MASK) == RTE_IPSEC_SATP_IPV4)) || \
 	((t) & RTE_IPSEC_SATP_MODE_MASK) == RTE_IPSEC_SATP_MODE_TUNLV4)
 
-/* helper routine to free bulk of packets */
-static inline void
-free_pkts(struct rte_mbuf *mb[], uint32_t n)
-{
-	uint32_t i;
-
-	for (i = 0; i != n; i++)
-		rte_pktmbuf_free(mb[i]);
-}
-
 /* helper routine to free bulk of crypto-ops and related packets */
 static inline void
 free_cops(struct rte_crypto_op *cop[], uint32_t n)
--

2.7.4

#11 lore raw

Hi Konstantin,

Please see inline.

Thanks,
Anoob

Quoted 46 lines

-----Original Message-----
From: Ananyev, Konstantin <[email protected]>
Sent: Friday, May 8, 2020 1:39 PM
To: Anoob Joseph <[email protected]>; Akhil Goyal <[email protected]>; Nicolau, Radu <[email protected]>
Cc: Narayana Prasad Raju Athreya <[email protected]>; [email protected]
Subject: [EXT] RE: [PATCH v3] examples/ipsec-secgw: add per core packet stats

External Email

----------------------------------------------------------------------

Quoted 34 lines
examples/ipsec-secgw/ipsec-secgw.c +33 -0
+#if (STATS_INTERVAL > 0)
+
+/* Print out statistics on packet distribution */ static void
+print_stats(void)
+{
+	uint64_t total_packets_dropped, total_packets_tx, total_packets_rx;
+	unsigned int coreid;
+	float burst_percent;
+
+	total_packets_dropped = 0;
+	total_packets_tx = 0;
+	total_packets_rx = 0;
+
+	const char clr[] = { 27, '[', '2', 'J', '\0' };
+
+	/* Clear screen and move to top left */
+	printf("%s", clr);
+
+	printf("\nCore statistics ====================================");
+
+	for (coreid = 0; coreid < RTE_MAX_LCORE; coreid++) {
+		/* skip disabled cores */
+		if (rte_lcore_is_enabled(coreid) == 0)
+			continue;
+		burst_percent = (float)(core_statistics[coreid].burst_rx * 100)/
+					core_statistics[coreid].rx;
+		printf("\nStatistics for core %u ------------------------------"
+			   "\nPackets received: %20"PRIu64
+			   "\nPackets sent: %24"PRIu64
+			   "\nPackets dropped: %21"PRIu64
+			   "\nBurst percent: %23.2f",
+			   coreid,
+			   core_statistics[coreid].rx,
Quoted 9 lines

examples/ipsec-secgw/ipsec-secgw.c adjust_ipv6_pktlen(struct rte_mbuf *m, const struct rte_ipv6_hdr *iph, · line 327

Quoted 3 lines
examples/ipsec-secgw/ipsec-secgw.c +3 -0
+			   core_statistics[coreid].tx,
+			   core_statistics[coreid].dropped,
+			   burst_percent);

As one more comment:
Can we also collect number of calls and display average pkt/call (for both rx and tx)?

[Anoob] Can you describe which "call" you meant? We can capture more logs if required.

Quoted 440 lines

examples/ipsec-secgw/ipsec-secgw.c adjust_ipv6_pktlen(struct rte_mbuf *m, const struct rte_ipv6_hdr *iph, · line 343

Quoted 13 lines
examples/ipsec-secgw/ipsec-secgw.c +13 -0
+
+		total_packets_dropped += core_statistics[coreid].dropped;
+		total_packets_tx += core_statistics[coreid].tx;
+		total_packets_rx += core_statistics[coreid].rx;
+	}
+	printf("\nAggregate statistics ==============================="
+		   "\nTotal packets received: %14"PRIu64
+		   "\nTotal packets sent: %18"PRIu64
+		   "\nTotal packets dropped: %15"PRIu64,
+		   total_packets_rx,
+		   total_packets_tx,
+		   total_packets_dropped);
+
	printf("\n===================================================

=\n");

examples/ipsec-secgw/ipsec-secgw.c prepare_one_packet(struct rte_mbuf *pkt, struct · line 388

Quoted 9 lines
examples/ipsec-secgw/ipsec-secgw.c +3 -0
+}
+#endif /* STATS_INTERVAL */
+
 static inline void
 prepare_one_packet(struct rte_mbuf *pkt, struct ipsec_traffic *t)  {
@@ -333,7 +386,7 @@ prepare_one_packet(struct rte_mbuf *pkt, struct
 ipsec_traffic *t)
 
 		/* drop packet when IPv6 header exceeds first segment length

*/

examples/ipsec-secgw/ipsec-secgw.c prepare_one_packet(struct rte_mbuf *pkt, struct

Quoted 7 lines
examples/ipsec-secgw/ipsec-secgw.c +1 -1
 		if (unlikely(l3len > pkt->data_len)) {
-			rte_pktmbuf_free(pkt);
+			free_pkt(pkt);
 			return;
 		}
@@ -350,7 +403,7 @@ prepare_one_packet(struct rte_mbuf *pkt, struct

ipsec_traffic *t)

examples/ipsec-secgw/ipsec-secgw.c send_burst(struct lcore_conf *qconf, uint16_t n,

Quoted 9 lines
examples/ipsec-secgw/ipsec-secgw.c +1 -1
 		/* Unknown/Unsupported type, drop the packet */
 		RTE_LOG(ERR, IPSEC, "Unsupported packet type 0x%x\n",
 			rte_be_to_cpu_16(eth->ether_type));
-		rte_pktmbuf_free(pkt);
+		free_pkt(pkt);
 		return;
 	}
 
@@ -477,9 +530,12 @@ send_burst(struct lcore_conf *qconf, uint16_t n,

uint16_t port)

examples/ipsec-secgw/ipsec-secgw.c send_fragment_packet(struct lcore_conf *qconf,

Quoted 14 lines
examples/ipsec-secgw/ipsec-secgw.c +4 -1
 	prepare_tx_burst(m_table, n, port, qconf);
 
 	ret = rte_eth_tx_burst(port, queueid, m_table, n);
+
+	core_stats_update_tx(ret);
+
 	if (unlikely(ret < n)) {
 		do {
-			rte_pktmbuf_free(m_table[ret]);
+			free_pkt(m_table[ret]);
 		} while (++ret < n);
 	}
 
@@ -525,7 +581,7 @@ send_fragment_packet(struct lcore_conf *qconf,

struct rte_mbuf *m,

examples/ipsec-secgw/ipsec-secgw.c send_single_packet(struct rte_mbuf *m, uint16_t

Quoted 9 lines
examples/ipsec-secgw/ipsec-secgw.c +1 -1
 			"error code: %d\n",
 			__func__, m->pkt_len, rte_errno);
 
-	rte_pktmbuf_free(m);
+	free_pkt(m);
 	return len;
 }
 
@@ -550,7 +606,7 @@ send_single_packet(struct rte_mbuf *m, uint16_t

port, uint8_t proto)

examples/ipsec-secgw/ipsec-secgw.c split46_traffic(struct ipsec_traffic *trf, struct rte_mbuf

Quoted 32 lines
examples/ipsec-secgw/ipsec-secgw.c +1 -1
 	} else if (frag_tbl_sz > 0)
 		len = send_fragment_packet(qconf, m, port, proto);
 	else
-		rte_pktmbuf_free(m);
+		free_pkt(m);
 
 	/* enough pkts to be sent */
 	if (unlikely(len == MAX_PKT_BURST)) { @@ -584,19 +640,19 @@

inbound_sp_sa(struct sp_ctx *sp, struct sa_ctx *sa, struct traffic_type *ip,

examples/ipsec-secgw/ipsec-secgw.c +3 -3
 			continue;
 		}
 		if (res == DISCARD) {
-			rte_pktmbuf_free(m);
+			free_pkt(m);
 			continue;
 		}
 
 		/* Only check SPI match for processed IPSec packets */
 		if (i < lim && ((m->ol_flags & PKT_RX_SEC_OFFLOAD) == 0)) {
-			rte_pktmbuf_free(m);
+			free_pkt(m);
 			continue;
 		}
 
 		sa_idx = res - 1;
 		if (!inbound_sa_check(sa, m, sa_idx)) {
-			rte_pktmbuf_free(m);
+			free_pkt(m);
 			continue;
 		}
 		ip->pkts[j++] = m;
@@ -631,7 +687,7 @@ split46_traffic(struct ipsec_traffic *trf, struct rte_mbuf

*mb[], uint32_t num)

examples/ipsec-secgw/ipsec-secgw.c process_pkts_inbound_nosp(struct ipsec_ctx

Quoted 28 lines
examples/ipsec-secgw/ipsec-secgw.c +3 -4
 					offsetof(struct ip6_hdr, ip6_nxt));
 			n6++;
 		} else
-			rte_pktmbuf_free(m);
+			free_pkt(m);
 	}
 
 	trf->ip4.num = n4;
@@ -683,7 +739,7 @@ outbound_sp(struct sp_ctx *sp, struct traffic_type *ip,
 		m = ip->pkts[i];
 		sa_idx = ip->res[i] - 1;
 		if (ip->res[i] == DISCARD)
-			rte_pktmbuf_free(m);
+			free_pkt(m);
 		else if (ip->res[i] == BYPASS)
 			ip->pkts[j++] = m;
 		else {
@@ -702,8 +758,7 @@ process_pkts_outbound(struct ipsec_ctx *ipsec_ctx,
 	uint16_t idx, nb_pkts_out, i;
 
 	/* Drop any IPsec traffic from protected ports */
-	for (i = 0; i < traffic->ipsec.num; i++)
-		rte_pktmbuf_free(traffic->ipsec.pkts[i]);
+	free_pkts(traffic->ipsec.pkts, traffic->ipsec.num);
 
 	traffic->ipsec.num = 0;
 
@@ -743,14 +798,12 @@ process_pkts_inbound_nosp(struct ipsec_ctx

*ipsec_ctx,

examples/ipsec-secgw/ipsec-secgw.c process_pkts_outbound_nosp(struct ipsec_ctx

Quoted 17 lines
examples/ipsec-secgw/ipsec-secgw.c +2 -4
 	uint32_t nb_pkts_in, i, idx;
 
 	/* Drop any IPv4 traffic from unprotected ports */
-	for (i = 0; i < traffic->ip4.num; i++)
-		rte_pktmbuf_free(traffic->ip4.pkts[i]);
+	free_pkts(traffic->ip4.pkts, traffic->ip4.num);
 
 	traffic->ip4.num = 0;
 
 	/* Drop any IPv6 traffic from unprotected ports */
-	for (i = 0; i < traffic->ip6.num; i++)
-		rte_pktmbuf_free(traffic->ip6.pkts[i]);
+	free_pkts(traffic->ip6.pkts, traffic->ip6.num);
 
 	traffic->ip6.num = 0;
 
@@ -786,8 +839,7 @@ process_pkts_outbound_nosp(struct ipsec_ctx

*ipsec_ctx,

examples/ipsec-secgw/ipsec-secgw.c route4_pkts(struct rt_ctx *rt_ctx, struct rte_mbuf

Quoted 10 lines
examples/ipsec-secgw/ipsec-secgw.c +1 -2
 	struct ip *ip;
 
 	/* Drop any IPsec traffic from protected ports */
-	for (i = 0; i < traffic->ipsec.num; i++)
-		rte_pktmbuf_free(traffic->ipsec.pkts[i]);
+	free_pkts(traffic->ipsec.pkts, traffic->ipsec.num);
 
 	n = 0;
 
@@ -901,7 +953,7 @@ route4_pkts(struct rt_ctx *rt_ctx, struct rte_mbuf

*pkts[], uint8_t nb_pkts)

examples/ipsec-secgw/ipsec-secgw.c route4_pkts(struct rt_ctx *rt_ctx, struct rte_mbuf *pkts[], uint8_t nb_pkts)

Quoted 8 lines
examples/ipsec-secgw/ipsec-secgw.c +1 -1
 		}
 
 		if ((pkt_hop & RTE_LPM_LOOKUP_SUCCESS) == 0) {
-			rte_pktmbuf_free(pkts[i]);
+			free_pkt(pkts[i]);
 			continue;
 		}
 		send_single_packet(pkts[i], pkt_hop & 0xff, IPPROTO_IP); @@ -

953,7

Quoted 1 line

+1005,7 @@ route6_pkts(struct rt_ctx *rt_ctx, struct rte_mbuf *pkts[],

uint8_t nb_pkts)

examples/ipsec-secgw/ipsec-secgw.c route6_pkts(struct rt_ctx *rt_ctx, struct rte_mbuf *pkts[], uint8_t nb_pkts) · line 1011

Quoted 8 lines
examples/ipsec-secgw/ipsec-secgw.c +1 -1
 		}
 
 		if (pkt_hop == -1) {
-			rte_pktmbuf_free(pkts[i]);
+			free_pkt(pkts[i]);
 			continue;
 		}
 		send_single_packet(pkts[i], pkt_hop & 0xff, IPPROTO_IPV6);

@@

Quoted 21 lines

-1099,6 +1151,10 @@ ipsec_poll_mode_worker(void)

examples/ipsec-secgw/ipsec-secgw.c +10 -0
 	const uint64_t drain_tsc = (rte_get_tsc_hz() + US_PER_S - 1)
 			/ US_PER_S * BURST_TX_DRAIN_US;
 	struct lcore_rx_queue *rxql;
+#if (STATS_INTERVAL > 0)
+	const uint64_t timer_period = STATS_INTERVAL * rte_get_timer_hz();
+	uint64_t timer_tsc = 0;
+#endif /* STATS_INTERVAL */
 
 	prev_tsc = 0;
 	lcore_id = rte_lcore_id();
@@ -1159,6 +1215,19 @@ ipsec_poll_mode_worker(void)
 			drain_tx_buffers(qconf);
 			drain_crypto_buffers(qconf);
 			prev_tsc = cur_tsc;
+#if (STATS_INTERVAL > 0)
+			if (lcore_id == rte_get_master_lcore()) {
+				/* advance the timer */
+				timer_tsc += diff_tsc;
+
+				/* if timer has reached its timeout */

examples/ipsec-secgw/ipsec-secgw.c ipsec_poll_mode_worker(void) · line 1233

Quoted 10 lines
examples/ipsec-secgw/ipsec-secgw.c +7 -0
+				if (unlikely(timer_tsc >= timer_period)) {
+					print_stats();
+					/* reset the timer */
+					timer_tsc = 0;
+				}
+			}
+#endif /* STATS_INTERVAL */
 		}
 
 		for (i = 0; i < qconf->nb_rx_queue; ++i) { @@ -1169,8 +1238,10

@@

examples/ipsec-secgw/ipsec.c ipsec_enqueue(ipsec_xform_fn xform_func, struct

Quoted 131 lines

ipsec_poll_mode_worker(void)

examples/ipsec-secgw/ipsec-secgw.c +3 -1
 			nb_rx = rte_eth_rx_burst(portid, queueid,
 					pkts, MAX_PKT_BURST);
 
-			if (nb_rx > 0)
+			if (nb_rx > 0) {
+				core_stats_update_rx(nb_rx);
 				process_pkts(qconf, pkts, nb_rx, portid);
+			}
 
 			/* dequeue and process completed crypto-ops */
 			if (is_unprotected_port(portid))
examples/ipsec-secgw/ipsec-secgw.h +13 -0
index 4b53cb5..5b3561f 100644
@@ -6,6 +6,8 @@
 
 #include <stdbool.h>
 
+#define STATS_INTERVAL 0
+
 #define NB_SOCKETS 4
 
 #define MAX_PKT_BURST 32
@@ -69,6 +71,17 @@ struct ethaddr_info {
 	uint64_t src, dst;
 };
 
+#if (STATS_INTERVAL > 0)
+struct ipsec_core_statistics {
+	uint64_t tx;
+	uint64_t rx;
+	uint64_t dropped;
+	uint64_t burst_rx;
+} __rte_cache_aligned;
+
+struct ipsec_core_statistics core_statistics[RTE_MAX_LCORE]; #endif
+/* STATS_INTERVAL */
+
 extern struct ethaddr_info ethaddr_tbl[RTE_MAX_ETHPORTS];
 
 /* Port mask to identify the unprotected ports */ @@ -85,4 +98,59 @@

is_unprotected_port(uint16_t port_id)

examples/ipsec-secgw/ipsec-secgw.h +49 -0
 	return unprotected_port_mask & (1 << port_id);  }
 
+static inline void
+core_stats_update_rx(int n)
+{
+#if (STATS_INTERVAL > 0)
+	int lcore_id = rte_lcore_id();
+	core_statistics[lcore_id].rx += n;
+	if (n == MAX_PKT_BURST)
+		core_statistics[lcore_id].burst_rx += n; #else
+	RTE_SET_USED(n);
+#endif /* STATS_INTERVAL */
+}
+
+static inline void
+core_stats_update_tx(int n)
+{
+#if (STATS_INTERVAL > 0)
+	int lcore_id = rte_lcore_id();
+	core_statistics[lcore_id].tx += n;
+#else
+	RTE_SET_USED(n);
+#endif /* STATS_INTERVAL */
+}
+
+static inline void
+core_stats_update_drop(int n)
+{
+#if (STATS_INTERVAL > 0)
+	int lcore_id = rte_lcore_id();
+	core_statistics[lcore_id].dropped += n; #else
+	RTE_SET_USED(n);
+#endif /* STATS_INTERVAL */
+}
+
+/* helper routine to free bulk of packets */ static inline void
+free_pkts(struct rte_mbuf *mb[], uint32_t n) {
+	uint32_t i;
+
+	for (i = 0; i != n; i++)
+		rte_pktmbuf_free(mb[i]);
+
+	core_stats_update_drop(n);
+}
+
+/* helper routine to free single packet */ static inline void
+free_pkt(struct rte_mbuf *mb) {
+	rte_pktmbuf_free(mb);
+	core_stats_update_drop(1);
+}
+
 #endif /* _IPSEC_SECGW_H_ */
(quoted diff) +0 -0

b/examples/ipsec-secgw/ipsec.c index bf88d80..351f1f1 100644

examples/ipsec-secgw/ipsec.c +3 -3
@@ -500,7 +500,7 @@ enqueue_cop_burst(struct cdev_qp *cqp)
 			cqp->id, cqp->qp, ret, len);
 			/* drop packets that we fail to enqueue */
 			for (i = ret; i < len; i++)
-				rte_pktmbuf_free(cqp->buf[i]->sym->m_src);
+				free_pkt(cqp->buf[i]->sym->m_src);
 	}
 	cqp->in_flight += ret;
 	cqp->len = 0;
@@ -528,7 +528,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct
 ipsec_ctx *ipsec_ctx,
 
 	for (i = 0; i < nb_pkts; i++) {
 		if (unlikely(sas[i] == NULL)) {
-			rte_pktmbuf_free(pkts[i]);
+			free_pkt(pkts[i]);
 			continue;
 		}
@@ -549,7 +549,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct
 ipsec_ctx *ipsec_ctx,
 
 			if ((unlikely(ips->security.ses == NULL)) &&
 				create_lookaside_session(ipsec_ctx, sa, ips)) {
-				rte_pktmbuf_free(pkts[i]);
+				free_pkt(pkts[i]);
 				continue;
 			}
@@ -563,7 +563,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct

ipsec_ctx *ipsec_ctx,

examples/ipsec-secgw/ipsec.c ipsec_enqueue(ipsec_xform_fn xform_func, struct · line 564

Quoted 2 lines
examples/ipsec-secgw/ipsec.c +0 -0
 		case RTE_SECURITY_ACTION_TYPE_CPU_CRYPTO:
 			RTE_LOG(ERR, IPSEC, "CPU crypto is not supported by

the"

examples/ipsec-secgw/ipsec.c ipsec_inline_dequeue(ipsec_xform_fn xform_func,

Quoted 37 lines
examples/ipsec-secgw/ipsec.c +3 -3
 					" legacy mode.");
-			rte_pktmbuf_free(pkts[i]);
+			free_pkt(pkts[i]);
 			continue;
 
 		case RTE_SECURITY_ACTION_TYPE_NONE:
@@ -575,7 +575,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct
 ipsec_ctx *ipsec_ctx,
 
 			if ((unlikely(ips->crypto.ses == NULL)) &&
 				create_lookaside_session(ipsec_ctx, sa, ips)) {
-				rte_pktmbuf_free(pkts[i]);
+				free_pkt(pkts[i]);
 				continue;
 			}
@@ -584,7 +584,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct
 ipsec_ctx *ipsec_ctx,
 
 			ret = xform_func(pkts[i], sa, &priv->cop);
 			if (unlikely(ret)) {
-				rte_pktmbuf_free(pkts[i]);
+				free_pkt(pkts[i]);
 				continue;
 			}
examples/ipsec-secgw/ipsec.c +1 -1
 			break;
@@ -608,7 +608,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct
 ipsec_ctx *ipsec_ctx,
 
 			ret = xform_func(pkts[i], sa, &priv->cop);
 			if (unlikely(ret)) {
-				rte_pktmbuf_free(pkts[i]);
+				free_pkt(pkts[i]);
 				continue;
 			}
@@ -643,7 +643,7 @@ ipsec_inline_dequeue(ipsec_xform_fn xform_func,

struct ipsec_ctx *ipsec_ctx,

examples/ipsec-secgw/ipsec.c ipsec_dequeue(ipsec_xform_fn xform_func, struct

Quoted 9 lines
examples/ipsec-secgw/ipsec.c +1 -1
 		sa = priv->sa;
 		ret = xform_func(pkt, sa, &priv->cop);
 		if (unlikely(ret)) {
-			rte_pktmbuf_free(pkt);
+			free_pkt(pkt);
 			continue;
 		}
 		pkts[nb_pkts++] = pkt;
@@ -690,13 +690,13 @@ ipsec_dequeue(ipsec_xform_fn xform_func, struct

ipsec_ctx *ipsec_ctx,

examples/ipsec-secgw/ipsec.c ipsec_dequeue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,

Quoted 9 lines
examples/ipsec-secgw/ipsec.c +1 -1
 				RTE_SECURITY_ACTION_TYPE_NONE) {
 				ret = xform_func(pkt, sa, cops[j]);
 				if (unlikely(ret)) {
-					rte_pktmbuf_free(pkt);
+					free_pkt(pkt);
 					continue;
 				}
 			} else if (ipsec_get_action_type(sa) ==
 
	RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) {
Quoted 12 lines
examples/ipsec-secgw/ipsec.c +1 -1
 				if (cops[j]->status) {
-					rte_pktmbuf_free(pkt);
+					free_pkt(pkt);
 					continue;
 				}
 			}
examples/ipsec-secgw/ipsec_process.c +0 -0
index bb2f2b8..4748299 100644

examples/ipsec-secgw/ipsec_process.c line 18

Quoted 7 lines
examples/ipsec-secgw/ipsec_process.c +1 -0
@@ -12,22 +12,13 @@
 #include <rte_mbuf.h>
 
 #include "ipsec.h"
+#include "ipsec-secgw.h"
 
 #define SATP_OUT_IPV4(t)	\
 	((((t) & RTE_IPSEC_SATP_MODE_MASK) ==

RTE_IPSEC_SATP_MODE_TRANS && \

examples/ipsec-secgw/ipsec_process.c line 20

Quoted 2 lines
examples/ipsec-secgw/ipsec_process.c +0 -0
 	(((t) & RTE_IPSEC_SATP_IPV_MASK) == RTE_IPSEC_SATP_IPV4)) || \
 	((t) & RTE_IPSEC_SATP_MODE_MASK) ==

RTE_IPSEC_SATP_MODE_TUNLV4)

Quoted 13 lines
examples/ipsec-secgw/ipsec_process.c +0 -9
-/* helper routine to free bulk of packets */ -static inline void
-free_pkts(struct rte_mbuf *mb[], uint32_t n) -{
-	uint32_t i;
-
-	for (i = 0; i != n; i++)
-		rte_pktmbuf_free(mb[i]);
-}
-
 /* helper routine to free bulk of crypto-ops and related packets */
 static inline void  free_cops(struct rte_crypto_op *cop[], uint32_t n)
--

2.7.4

#12 lore raw
Quoted 44 lines
Quoted 34 lines
Quoted 34 lines
examples/ipsec-secgw/ipsec-secgw.c +33 -0
+#if (STATS_INTERVAL > 0)
+
+/* Print out statistics on packet distribution */ static void
+print_stats(void)
+{
+	uint64_t total_packets_dropped, total_packets_tx, total_packets_rx;
+	unsigned int coreid;
+	float burst_percent;
+
+	total_packets_dropped = 0;
+	total_packets_tx = 0;
+	total_packets_rx = 0;
+
+	const char clr[] = { 27, '[', '2', 'J', '\0' };
+
+	/* Clear screen and move to top left */
+	printf("%s", clr);
+
+	printf("\nCore statistics ====================================");
+
+	for (coreid = 0; coreid < RTE_MAX_LCORE; coreid++) {
+		/* skip disabled cores */
+		if (rte_lcore_is_enabled(coreid) == 0)
+			continue;
+		burst_percent = (float)(core_statistics[coreid].burst_rx * 100)/
+					core_statistics[coreid].rx;
+		printf("\nStatistics for core %u ------------------------------"
+			   "\nPackets received: %20"PRIu64
+			   "\nPackets sent: %24"PRIu64
+			   "\nPackets dropped: %21"PRIu64
+			   "\nBurst percent: %23.2f",
+			   coreid,
+			   core_statistics[coreid].rx,
Quoted 9 lines

examples/ipsec-secgw/ipsec-secgw.c adjust_ipv6_pktlen(struct rte_mbuf *m, const struct rte_ipv6_hdr *iph, · line 327

Quoted 3 lines
examples/ipsec-secgw/ipsec-secgw.c +3 -0
+			   core_statistics[coreid].tx,
+			   core_statistics[coreid].dropped,
+			   burst_percent);

As one more comment:
Can we also collect number of calls and display average pkt/call (for both rx and tx)?

Quoted 1 line

[Anoob] Can you describe which "call" you meant? We can capture more logs if required.

I meant something like that:
+static inline void
+core_stats_update_rx(int n)
+{
+#if (STATS_INTERVAL > 0)
+ int lcore_id = rte_lcore_id();
+ core_statistics[lcore_id].rx += n;
+ core_statistics[lcore_id].rx_call++;
+ if (n == MAX_PKT_BURST)
+ core_statistics[lcore_id].burst_rx += n;
+#else
+ RTE_SET_USED(n);
+#endif /* STATS_INTERVAL */
+}

And then in print_stats():
"\nPackets received: %20"PRIu64
",RX calls: %" PRIu64
"RX packets/call: %.2f"
core_statistics[coreid].rx, core_statistics[coreid].rx_call, (double)core_statistics[coreid].rx / core_statistics[coreid].rx_call

Quoted 441 lines
Quoted 441 lines
Quoted 4 lines
examples/ipsec-secgw/ipsec-secgw.c +3 -0
+
+		total_packets_dropped += core_statistics[coreid].dropped;
+		total_packets_tx += core_statistics[coreid].tx;

examples/ipsec-secgw/ipsec-secgw.c adjust_ipv6_pktlen(struct rte_mbuf *m, const struct rte_ipv6_hdr *iph, · line 343

Quoted 10 lines
examples/ipsec-secgw/ipsec-secgw.c +10 -0
+		total_packets_rx += core_statistics[coreid].rx;
+	}
+	printf("\nAggregate statistics ==============================="
+		   "\nTotal packets received: %14"PRIu64
+		   "\nTotal packets sent: %18"PRIu64
+		   "\nTotal packets dropped: %15"PRIu64,
+		   total_packets_rx,
+		   total_packets_tx,
+		   total_packets_dropped);
+
	printf("\n===================================================

=\n");

examples/ipsec-secgw/ipsec-secgw.c prepare_one_packet(struct rte_mbuf *pkt, struct · line 388

Quoted 9 lines
examples/ipsec-secgw/ipsec-secgw.c +3 -0
+}
+#endif /* STATS_INTERVAL */
+
 static inline void
 prepare_one_packet(struct rte_mbuf *pkt, struct ipsec_traffic *t)  {
@@ -333,7 +386,7 @@ prepare_one_packet(struct rte_mbuf *pkt, struct
 ipsec_traffic *t)
 
 		/* drop packet when IPv6 header exceeds first segment length

*/

examples/ipsec-secgw/ipsec-secgw.c prepare_one_packet(struct rte_mbuf *pkt, struct

Quoted 7 lines
examples/ipsec-secgw/ipsec-secgw.c +1 -1
 		if (unlikely(l3len > pkt->data_len)) {
-			rte_pktmbuf_free(pkt);
+			free_pkt(pkt);
 			return;
 		}
@@ -350,7 +403,7 @@ prepare_one_packet(struct rte_mbuf *pkt, struct

ipsec_traffic *t)

examples/ipsec-secgw/ipsec-secgw.c send_burst(struct lcore_conf *qconf, uint16_t n,

Quoted 9 lines
examples/ipsec-secgw/ipsec-secgw.c +1 -1
 		/* Unknown/Unsupported type, drop the packet */
 		RTE_LOG(ERR, IPSEC, "Unsupported packet type 0x%x\n",
 			rte_be_to_cpu_16(eth->ether_type));
-		rte_pktmbuf_free(pkt);
+		free_pkt(pkt);
 		return;
 	}
 
@@ -477,9 +530,12 @@ send_burst(struct lcore_conf *qconf, uint16_t n,

uint16_t port)

examples/ipsec-secgw/ipsec-secgw.c send_fragment_packet(struct lcore_conf *qconf,

Quoted 14 lines
examples/ipsec-secgw/ipsec-secgw.c +4 -1
 	prepare_tx_burst(m_table, n, port, qconf);
 
 	ret = rte_eth_tx_burst(port, queueid, m_table, n);
+
+	core_stats_update_tx(ret);
+
 	if (unlikely(ret < n)) {
 		do {
-			rte_pktmbuf_free(m_table[ret]);
+			free_pkt(m_table[ret]);
 		} while (++ret < n);
 	}
 
@@ -525,7 +581,7 @@ send_fragment_packet(struct lcore_conf *qconf,

struct rte_mbuf *m,

examples/ipsec-secgw/ipsec-secgw.c send_single_packet(struct rte_mbuf *m, uint16_t

Quoted 9 lines
examples/ipsec-secgw/ipsec-secgw.c +1 -1
 			"error code: %d\n",
 			__func__, m->pkt_len, rte_errno);
 
-	rte_pktmbuf_free(m);
+	free_pkt(m);
 	return len;
 }
 
@@ -550,7 +606,7 @@ send_single_packet(struct rte_mbuf *m, uint16_t

port, uint8_t proto)

examples/ipsec-secgw/ipsec-secgw.c split46_traffic(struct ipsec_traffic *trf, struct rte_mbuf

Quoted 32 lines
examples/ipsec-secgw/ipsec-secgw.c +1 -1
 	} else if (frag_tbl_sz > 0)
 		len = send_fragment_packet(qconf, m, port, proto);
 	else
-		rte_pktmbuf_free(m);
+		free_pkt(m);
 
 	/* enough pkts to be sent */
 	if (unlikely(len == MAX_PKT_BURST)) { @@ -584,19 +640,19 @@

inbound_sp_sa(struct sp_ctx *sp, struct sa_ctx *sa, struct traffic_type *ip,

examples/ipsec-secgw/ipsec-secgw.c +3 -3
 			continue;
 		}
 		if (res == DISCARD) {
-			rte_pktmbuf_free(m);
+			free_pkt(m);
 			continue;
 		}
 
 		/* Only check SPI match for processed IPSec packets */
 		if (i < lim && ((m->ol_flags & PKT_RX_SEC_OFFLOAD) == 0)) {
-			rte_pktmbuf_free(m);
+			free_pkt(m);
 			continue;
 		}
 
 		sa_idx = res - 1;
 		if (!inbound_sa_check(sa, m, sa_idx)) {
-			rte_pktmbuf_free(m);
+			free_pkt(m);
 			continue;
 		}
 		ip->pkts[j++] = m;
@@ -631,7 +687,7 @@ split46_traffic(struct ipsec_traffic *trf, struct rte_mbuf

*mb[], uint32_t num)

examples/ipsec-secgw/ipsec-secgw.c process_pkts_inbound_nosp(struct ipsec_ctx

Quoted 28 lines
examples/ipsec-secgw/ipsec-secgw.c +3 -4
 					offsetof(struct ip6_hdr, ip6_nxt));
 			n6++;
 		} else
-			rte_pktmbuf_free(m);
+			free_pkt(m);
 	}
 
 	trf->ip4.num = n4;
@@ -683,7 +739,7 @@ outbound_sp(struct sp_ctx *sp, struct traffic_type *ip,
 		m = ip->pkts[i];
 		sa_idx = ip->res[i] - 1;
 		if (ip->res[i] == DISCARD)
-			rte_pktmbuf_free(m);
+			free_pkt(m);
 		else if (ip->res[i] == BYPASS)
 			ip->pkts[j++] = m;
 		else {
@@ -702,8 +758,7 @@ process_pkts_outbound(struct ipsec_ctx *ipsec_ctx,
 	uint16_t idx, nb_pkts_out, i;
 
 	/* Drop any IPsec traffic from protected ports */
-	for (i = 0; i < traffic->ipsec.num; i++)
-		rte_pktmbuf_free(traffic->ipsec.pkts[i]);
+	free_pkts(traffic->ipsec.pkts, traffic->ipsec.num);
 
 	traffic->ipsec.num = 0;
 
@@ -743,14 +798,12 @@ process_pkts_inbound_nosp(struct ipsec_ctx

*ipsec_ctx,

examples/ipsec-secgw/ipsec-secgw.c process_pkts_outbound_nosp(struct ipsec_ctx

Quoted 17 lines
examples/ipsec-secgw/ipsec-secgw.c +2 -4
 	uint32_t nb_pkts_in, i, idx;
 
 	/* Drop any IPv4 traffic from unprotected ports */
-	for (i = 0; i < traffic->ip4.num; i++)
-		rte_pktmbuf_free(traffic->ip4.pkts[i]);
+	free_pkts(traffic->ip4.pkts, traffic->ip4.num);
 
 	traffic->ip4.num = 0;
 
 	/* Drop any IPv6 traffic from unprotected ports */
-	for (i = 0; i < traffic->ip6.num; i++)
-		rte_pktmbuf_free(traffic->ip6.pkts[i]);
+	free_pkts(traffic->ip6.pkts, traffic->ip6.num);
 
 	traffic->ip6.num = 0;
 
@@ -786,8 +839,7 @@ process_pkts_outbound_nosp(struct ipsec_ctx

*ipsec_ctx,

examples/ipsec-secgw/ipsec-secgw.c route4_pkts(struct rt_ctx *rt_ctx, struct rte_mbuf

Quoted 10 lines
examples/ipsec-secgw/ipsec-secgw.c +1 -2
 	struct ip *ip;
 
 	/* Drop any IPsec traffic from protected ports */
-	for (i = 0; i < traffic->ipsec.num; i++)
-		rte_pktmbuf_free(traffic->ipsec.pkts[i]);
+	free_pkts(traffic->ipsec.pkts, traffic->ipsec.num);
 
 	n = 0;
 
@@ -901,7 +953,7 @@ route4_pkts(struct rt_ctx *rt_ctx, struct rte_mbuf

*pkts[], uint8_t nb_pkts)

examples/ipsec-secgw/ipsec-secgw.c route4_pkts(struct rt_ctx *rt_ctx, struct rte_mbuf *pkts[], uint8_t nb_pkts)

Quoted 8 lines
examples/ipsec-secgw/ipsec-secgw.c +1 -1
 		}
 
 		if ((pkt_hop & RTE_LPM_LOOKUP_SUCCESS) == 0) {
-			rte_pktmbuf_free(pkts[i]);
+			free_pkt(pkts[i]);
 			continue;
 		}
 		send_single_packet(pkts[i], pkt_hop & 0xff, IPPROTO_IP); @@ -

953,7

Quoted 1 line

+1005,7 @@ route6_pkts(struct rt_ctx *rt_ctx, struct rte_mbuf *pkts[],

uint8_t nb_pkts)

examples/ipsec-secgw/ipsec-secgw.c route6_pkts(struct rt_ctx *rt_ctx, struct rte_mbuf *pkts[], uint8_t nb_pkts) · line 1011

Quoted 8 lines
examples/ipsec-secgw/ipsec-secgw.c +1 -1
 		}
 
 		if (pkt_hop == -1) {
-			rte_pktmbuf_free(pkts[i]);
+			free_pkt(pkts[i]);
 			continue;
 		}
 		send_single_packet(pkts[i], pkt_hop & 0xff, IPPROTO_IPV6);

@@

Quoted 21 lines

-1099,6 +1151,10 @@ ipsec_poll_mode_worker(void)

examples/ipsec-secgw/ipsec-secgw.c +10 -0
 	const uint64_t drain_tsc = (rte_get_tsc_hz() + US_PER_S - 1)
 			/ US_PER_S * BURST_TX_DRAIN_US;
 	struct lcore_rx_queue *rxql;
+#if (STATS_INTERVAL > 0)
+	const uint64_t timer_period = STATS_INTERVAL * rte_get_timer_hz();
+	uint64_t timer_tsc = 0;
+#endif /* STATS_INTERVAL */
 
 	prev_tsc = 0;
 	lcore_id = rte_lcore_id();
@@ -1159,6 +1215,19 @@ ipsec_poll_mode_worker(void)
 			drain_tx_buffers(qconf);
 			drain_crypto_buffers(qconf);
 			prev_tsc = cur_tsc;
+#if (STATS_INTERVAL > 0)
+			if (lcore_id == rte_get_master_lcore()) {
+				/* advance the timer */
+				timer_tsc += diff_tsc;
+
+				/* if timer has reached its timeout */

examples/ipsec-secgw/ipsec-secgw.c ipsec_poll_mode_worker(void) · line 1233

Quoted 10 lines
examples/ipsec-secgw/ipsec-secgw.c +7 -0
+				if (unlikely(timer_tsc >= timer_period)) {
+					print_stats();
+					/* reset the timer */
+					timer_tsc = 0;
+				}
+			}
+#endif /* STATS_INTERVAL */
 		}
 
 		for (i = 0; i < qconf->nb_rx_queue; ++i) { @@ -1169,8 +1238,10

@@

examples/ipsec-secgw/ipsec.c ipsec_enqueue(ipsec_xform_fn xform_func, struct

Quoted 131 lines

ipsec_poll_mode_worker(void)

examples/ipsec-secgw/ipsec-secgw.c +3 -1
 			nb_rx = rte_eth_rx_burst(portid, queueid,
 					pkts, MAX_PKT_BURST);
 
-			if (nb_rx > 0)
+			if (nb_rx > 0) {
+				core_stats_update_rx(nb_rx);
 				process_pkts(qconf, pkts, nb_rx, portid);
+			}
 
 			/* dequeue and process completed crypto-ops */
 			if (is_unprotected_port(portid))
examples/ipsec-secgw/ipsec-secgw.h +13 -0
index 4b53cb5..5b3561f 100644
@@ -6,6 +6,8 @@
 
 #include <stdbool.h>
 
+#define STATS_INTERVAL 0
+
 #define NB_SOCKETS 4
 
 #define MAX_PKT_BURST 32
@@ -69,6 +71,17 @@ struct ethaddr_info {
 	uint64_t src, dst;
 };
 
+#if (STATS_INTERVAL > 0)
+struct ipsec_core_statistics {
+	uint64_t tx;
+	uint64_t rx;
+	uint64_t dropped;
+	uint64_t burst_rx;
+} __rte_cache_aligned;
+
+struct ipsec_core_statistics core_statistics[RTE_MAX_LCORE]; #endif
+/* STATS_INTERVAL */
+
 extern struct ethaddr_info ethaddr_tbl[RTE_MAX_ETHPORTS];
 
 /* Port mask to identify the unprotected ports */ @@ -85,4 +98,59 @@

is_unprotected_port(uint16_t port_id)

examples/ipsec-secgw/ipsec-secgw.h +49 -0
 	return unprotected_port_mask & (1 << port_id);  }
 
+static inline void
+core_stats_update_rx(int n)
+{
+#if (STATS_INTERVAL > 0)
+	int lcore_id = rte_lcore_id();
+	core_statistics[lcore_id].rx += n;
+	if (n == MAX_PKT_BURST)
+		core_statistics[lcore_id].burst_rx += n; #else
+	RTE_SET_USED(n);
+#endif /* STATS_INTERVAL */
+}
+
+static inline void
+core_stats_update_tx(int n)
+{
+#if (STATS_INTERVAL > 0)
+	int lcore_id = rte_lcore_id();
+	core_statistics[lcore_id].tx += n;
+#else
+	RTE_SET_USED(n);
+#endif /* STATS_INTERVAL */
+}
+
+static inline void
+core_stats_update_drop(int n)
+{
+#if (STATS_INTERVAL > 0)
+	int lcore_id = rte_lcore_id();
+	core_statistics[lcore_id].dropped += n; #else
+	RTE_SET_USED(n);
+#endif /* STATS_INTERVAL */
+}
+
+/* helper routine to free bulk of packets */ static inline void
+free_pkts(struct rte_mbuf *mb[], uint32_t n) {
+	uint32_t i;
+
+	for (i = 0; i != n; i++)
+		rte_pktmbuf_free(mb[i]);
+
+	core_stats_update_drop(n);
+}
+
+/* helper routine to free single packet */ static inline void
+free_pkt(struct rte_mbuf *mb) {
+	rte_pktmbuf_free(mb);
+	core_stats_update_drop(1);
+}
+
 #endif /* _IPSEC_SECGW_H_ */
(quoted diff) +0 -0

b/examples/ipsec-secgw/ipsec.c index bf88d80..351f1f1 100644

examples/ipsec-secgw/ipsec.c +3 -3
@@ -500,7 +500,7 @@ enqueue_cop_burst(struct cdev_qp *cqp)
 			cqp->id, cqp->qp, ret, len);
 			/* drop packets that we fail to enqueue */
 			for (i = ret; i < len; i++)
-				rte_pktmbuf_free(cqp->buf[i]->sym->m_src);
+				free_pkt(cqp->buf[i]->sym->m_src);
 	}
 	cqp->in_flight += ret;
 	cqp->len = 0;
@@ -528,7 +528,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct
 ipsec_ctx *ipsec_ctx,
 
 	for (i = 0; i < nb_pkts; i++) {
 		if (unlikely(sas[i] == NULL)) {
-			rte_pktmbuf_free(pkts[i]);
+			free_pkt(pkts[i]);
 			continue;
 		}
@@ -549,7 +549,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct
 ipsec_ctx *ipsec_ctx,
 
 			if ((unlikely(ips->security.ses == NULL)) &&
 				create_lookaside_session(ipsec_ctx, sa, ips)) {
-				rte_pktmbuf_free(pkts[i]);
+				free_pkt(pkts[i]);
 				continue;
 			}
@@ -563,7 +563,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct

ipsec_ctx *ipsec_ctx,

examples/ipsec-secgw/ipsec.c ipsec_enqueue(ipsec_xform_fn xform_func, struct · line 564

Quoted 2 lines
examples/ipsec-secgw/ipsec.c +0 -0
 		case RTE_SECURITY_ACTION_TYPE_CPU_CRYPTO:
 			RTE_LOG(ERR, IPSEC, "CPU crypto is not supported by

the"

examples/ipsec-secgw/ipsec.c ipsec_inline_dequeue(ipsec_xform_fn xform_func,

Quoted 37 lines
examples/ipsec-secgw/ipsec.c +3 -3
 					" legacy mode.");
-			rte_pktmbuf_free(pkts[i]);
+			free_pkt(pkts[i]);
 			continue;
 
 		case RTE_SECURITY_ACTION_TYPE_NONE:
@@ -575,7 +575,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct
 ipsec_ctx *ipsec_ctx,
 
 			if ((unlikely(ips->crypto.ses == NULL)) &&
 				create_lookaside_session(ipsec_ctx, sa, ips)) {
-				rte_pktmbuf_free(pkts[i]);
+				free_pkt(pkts[i]);
 				continue;
 			}
@@ -584,7 +584,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct
 ipsec_ctx *ipsec_ctx,
 
 			ret = xform_func(pkts[i], sa, &priv->cop);
 			if (unlikely(ret)) {
-				rte_pktmbuf_free(pkts[i]);
+				free_pkt(pkts[i]);
 				continue;
 			}
examples/ipsec-secgw/ipsec.c +1 -1
 			break;
@@ -608,7 +608,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct
 ipsec_ctx *ipsec_ctx,
 
 			ret = xform_func(pkts[i], sa, &priv->cop);
 			if (unlikely(ret)) {
-				rte_pktmbuf_free(pkts[i]);
+				free_pkt(pkts[i]);
 				continue;
 			}
@@ -643,7 +643,7 @@ ipsec_inline_dequeue(ipsec_xform_fn xform_func,

struct ipsec_ctx *ipsec_ctx,

examples/ipsec-secgw/ipsec.c ipsec_dequeue(ipsec_xform_fn xform_func, struct

Quoted 9 lines
examples/ipsec-secgw/ipsec.c +1 -1
 		sa = priv->sa;
 		ret = xform_func(pkt, sa, &priv->cop);
 		if (unlikely(ret)) {
-			rte_pktmbuf_free(pkt);
+			free_pkt(pkt);
 			continue;
 		}
 		pkts[nb_pkts++] = pkt;
@@ -690,13 +690,13 @@ ipsec_dequeue(ipsec_xform_fn xform_func, struct

ipsec_ctx *ipsec_ctx,

examples/ipsec-secgw/ipsec.c ipsec_dequeue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,

Quoted 9 lines
examples/ipsec-secgw/ipsec.c +1 -1
 				RTE_SECURITY_ACTION_TYPE_NONE) {
 				ret = xform_func(pkt, sa, cops[j]);
 				if (unlikely(ret)) {
-					rte_pktmbuf_free(pkt);
+					free_pkt(pkt);
 					continue;
 				}
 			} else if (ipsec_get_action_type(sa) ==
 
	RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) {
Quoted 12 lines
examples/ipsec-secgw/ipsec.c +1 -1
 				if (cops[j]->status) {
-					rte_pktmbuf_free(pkt);
+					free_pkt(pkt);
 					continue;
 				}
 			}
examples/ipsec-secgw/ipsec_process.c +0 -0
index bb2f2b8..4748299 100644

examples/ipsec-secgw/ipsec_process.c line 18

Quoted 7 lines
examples/ipsec-secgw/ipsec_process.c +1 -0
@@ -12,22 +12,13 @@
 #include <rte_mbuf.h>
 
 #include "ipsec.h"
+#include "ipsec-secgw.h"
 
 #define SATP_OUT_IPV4(t)	\
 	((((t) & RTE_IPSEC_SATP_MODE_MASK) ==

RTE_IPSEC_SATP_MODE_TRANS && \

examples/ipsec-secgw/ipsec_process.c line 20

Quoted 2 lines
examples/ipsec-secgw/ipsec_process.c +0 -0
 	(((t) & RTE_IPSEC_SATP_IPV_MASK) == RTE_IPSEC_SATP_IPV4)) || \
 	((t) & RTE_IPSEC_SATP_MODE_MASK) ==

RTE_IPSEC_SATP_MODE_TUNLV4)

Quoted 13 lines
examples/ipsec-secgw/ipsec_process.c +0 -9
-/* helper routine to free bulk of packets */ -static inline void
-free_pkts(struct rte_mbuf *mb[], uint32_t n) -{
-	uint32_t i;
-
-	for (i = 0; i != n; i++)
-		rte_pktmbuf_free(mb[i]);
-}
-
 /* helper routine to free bulk of crypto-ops and related packets */
 static inline void  free_cops(struct rte_crypto_op *cop[], uint32_t n)
--

2.7.4

#13 lore raw

PATCH v4 1/1 examples/ipsec-secgw: add per core packet stats

In reply to Anoob Joseph

Adding per core packet handling stats to analyze traffic distribution when multiple cores are engaged.

Since aggregating the packet stats across cores would affect performance, keeping the feature disabled using compile time flags.

---
v4:
* Moved print stats routine to control thread
* Added stats for rx/tx pkts per call
* Replaced free_pkt(m) with free_pkts(&m, 1)

v3:
* Added wrapper functions for updating rx, tx & dropped counts
* Updated free_pkts() so that stats is updated internally
* Introduced similar free_pkt() function which updates stats and frees the packet
* Moved all inline functions and macros to ipsec-secgw.h
* Made STATS_INTERVAL macro to control the interval of the stats update. STATS_INTERVAL = 0 would disable the feature.

v2:
* Added lookup failure cases to drop count

 examples/ipsec-secgw/ipsec-secgw.c   | 114 ++++++++++++++++++++++++++++-------
 examples/ipsec-secgw/ipsec-secgw.h   |  66 ++++++++++++++++++++
 examples/ipsec-secgw/ipsec.c         |  20 +++---
 examples/ipsec-secgw/ipsec_process.c |  11 +---
 4 files changed, 170 insertions(+), 41 deletions(-)
examples/ipsec-secgw/ipsec-secgw.c +93 -21
index f777ce2..d2b5b2c 100644
@@ -47,6 +47,7 @@
 #include <rte_eventdev.h>
 #include <rte_ip.h>
 #include <rte_ip_frag.h>
+#include <rte_alarm.h>
 
 #include "event_helper.h"
 #include "ipsec.h"
@@ -287,6 +288,70 @@ adjust_ipv6_pktlen(struct rte_mbuf *m, const struct rte_ipv6_hdr *iph,
 	}
 }
 
+#if (STATS_INTERVAL > 0)
+
+/* Print out statistics on packet distribution */
+static void
+print_stats_cb(__rte_unused void *param)
+{
+	uint64_t total_packets_dropped, total_packets_tx, total_packets_rx;
+	float burst_percent, rx_per_call, tx_per_call;
+	unsigned int coreid;
+
+	total_packets_dropped = 0;
+	total_packets_tx = 0;
+	total_packets_rx = 0;
+
+	const char clr[] = { 27, '[', '2', 'J', '\0' };
+	const char topLeft[] = { 27, '[', '1', ';', '1', 'H', '\0' };
+
+	/* Clear screen and move to top left */
+	printf("%s%s", clr, topLeft);
+
+	printf("\nCore statistics ====================================");
+
+	for (coreid = 0; coreid < RTE_MAX_LCORE; coreid++) {
+		/* skip disabled cores */
+		if (rte_lcore_is_enabled(coreid) == 0)
+			continue;
+		burst_percent = (float)(core_statistics[coreid].burst_rx * 100)/
+					core_statistics[coreid].rx;
+		rx_per_call =  (float)(core_statistics[coreid].rx)/
+				       core_statistics[coreid].rx_call;
+		tx_per_call =  (float)(core_statistics[coreid].tx)/
+				       core_statistics[coreid].tx_call;
+		printf("\nStatistics for core %u ------------------------------"
+			   "\nPackets received: %20"PRIu64
+			   "\nPackets sent: %24"PRIu64
+			   "\nPackets dropped: %21"PRIu64
+			   "\nBurst percent: %23.2f"
+			   "\nPackets per Rx call: %17.2f"
+			   "\nPackets per Tx call: %17.2f",
+			   coreid,
+			   core_statistics[coreid].rx,
+			   core_statistics[coreid].tx,
+			   core_statistics[coreid].dropped,
+			   burst_percent,
+			   rx_per_call,
+			   tx_per_call);
+
+		total_packets_dropped += core_statistics[coreid].dropped;
+		total_packets_tx += core_statistics[coreid].tx;
+		total_packets_rx += core_statistics[coreid].rx;
+	}
+	printf("\nAggregate statistics ==============================="
+		   "\nTotal packets received: %14"PRIu64
+		   "\nTotal packets sent: %18"PRIu64
+		   "\nTotal packets dropped: %15"PRIu64,
+		   total_packets_rx,
+		   total_packets_tx,
+		   total_packets_dropped);
+	printf("\n====================================================\n");
+
+	rte_eal_alarm_set(STATS_INTERVAL * US_PER_S, print_stats_cb, NULL);
+}
+#endif /* STATS_INTERVAL */
+
 static inline void
 prepare_one_packet(struct rte_mbuf *pkt, struct ipsec_traffic *t)
 {
@@ -332,7 +397,7 @@ prepare_one_packet(struct rte_mbuf *pkt, struct ipsec_traffic *t)
 
 		/* drop packet when IPv6 header exceeds first segment length */
 		if (unlikely(l3len > pkt->data_len)) {
-			rte_pktmbuf_free(pkt);
+			free_pkts(&pkt, 1);
 			return;
 		}
 
@@ -349,7 +414,7 @@ prepare_one_packet(struct rte_mbuf *pkt, struct ipsec_traffic *t)
 		/* Unknown/Unsupported type, drop the packet */
 		RTE_LOG(ERR, IPSEC, "Unsupported packet type 0x%x\n",
 			rte_be_to_cpu_16(eth->ether_type));
-		rte_pktmbuf_free(pkt);
+		free_pkts(&pkt, 1);
 		return;
 	}
 
@@ -476,9 +541,12 @@ send_burst(struct lcore_conf *qconf, uint16_t n, uint16_t port)
 	prepare_tx_burst(m_table, n, port, qconf);
 
 	ret = rte_eth_tx_burst(port, queueid, m_table, n);
+
+	core_stats_update_tx(ret);
+
 	if (unlikely(ret < n)) {
 		do {
-			rte_pktmbuf_free(m_table[ret]);
+			free_pkts(&m_table[ret], 1);
 		} while (++ret < n);
 	}
 
@@ -524,7 +592,7 @@ send_fragment_packet(struct lcore_conf *qconf, struct rte_mbuf *m,
 			"error code: %d\n",
 			__func__, m->pkt_len, rte_errno);
 
-	rte_pktmbuf_free(m);
+	free_pkts(&m, 1);
 	return len;
 }
 
@@ -549,7 +617,7 @@ send_single_packet(struct rte_mbuf *m, uint16_t port, uint8_t proto)
 	} else if (frag_tbl_sz > 0)
 		len = send_fragment_packet(qconf, m, port, proto);
 	else
-		rte_pktmbuf_free(m);
+		free_pkts(&m, 1);
 
 	/* enough pkts to be sent */
 	if (unlikely(len == MAX_PKT_BURST)) {
@@ -583,19 +651,19 @@ inbound_sp_sa(struct sp_ctx *sp, struct sa_ctx *sa, struct traffic_type *ip,
 			continue;
 		}
 		if (res == DISCARD) {
-			rte_pktmbuf_free(m);
+			free_pkts(&m, 1);
 			continue;
 		}
 
 		/* Only check SPI match for processed IPSec packets */
 		if (i < lim && ((m->ol_flags & PKT_RX_SEC_OFFLOAD) == 0)) {
-			rte_pktmbuf_free(m);
+			free_pkts(&m, 1);
 			continue;
 		}
 
 		sa_idx = res - 1;
 		if (!inbound_sa_check(sa, m, sa_idx)) {
-			rte_pktmbuf_free(m);
+			free_pkts(&m, 1);
 			continue;
 		}
 		ip->pkts[j++] = m;
@@ -630,7 +698,7 @@ split46_traffic(struct ipsec_traffic *trf, struct rte_mbuf *mb[], uint32_t num)
 					offsetof(struct ip6_hdr, ip6_nxt));
 			n6++;
 		} else
-			rte_pktmbuf_free(m);
+			free_pkts(&m, 1);
 	}
 
 	trf->ip4.num = n4;
@@ -682,7 +750,7 @@ outbound_sp(struct sp_ctx *sp, struct traffic_type *ip,
 		m = ip->pkts[i];
 		sa_idx = ip->res[i] - 1;
 		if (ip->res[i] == DISCARD)
-			rte_pktmbuf_free(m);
+			free_pkts(&m, 1);
 		else if (ip->res[i] == BYPASS)
 			ip->pkts[j++] = m;
 		else {
@@ -701,8 +769,7 @@ process_pkts_outbound(struct ipsec_ctx *ipsec_ctx,
 	uint16_t idx, nb_pkts_out, i;
 
 	/* Drop any IPsec traffic from protected ports */
-	for (i = 0; i < traffic->ipsec.num; i++)
-		rte_pktmbuf_free(traffic->ipsec.pkts[i]);
+	free_pkts(traffic->ipsec.pkts, traffic->ipsec.num);
 
 	traffic->ipsec.num = 0;
 
@@ -742,14 +809,12 @@ process_pkts_inbound_nosp(struct ipsec_ctx *ipsec_ctx,
 	uint32_t nb_pkts_in, i, idx;
 
 	/* Drop any IPv4 traffic from unprotected ports */
-	for (i = 0; i < traffic->ip4.num; i++)
-		rte_pktmbuf_free(traffic->ip4.pkts[i]);
+	free_pkts(traffic->ip4.pkts, traffic->ip4.num);
 
 	traffic->ip4.num = 0;
 
 	/* Drop any IPv6 traffic from unprotected ports */
-	for (i = 0; i < traffic->ip6.num; i++)
-		rte_pktmbuf_free(traffic->ip6.pkts[i]);
+	free_pkts(traffic->ip6.pkts, traffic->ip6.num);
 
 	traffic->ip6.num = 0;
 
@@ -785,8 +850,7 @@ process_pkts_outbound_nosp(struct ipsec_ctx *ipsec_ctx,
 	struct ip *ip;
 
 	/* Drop any IPsec traffic from protected ports */
-	for (i = 0; i < traffic->ipsec.num; i++)
-		rte_pktmbuf_free(traffic->ipsec.pkts[i]);
+	free_pkts(traffic->ipsec.pkts, traffic->ipsec.num);
 
 	n = 0;
 
@@ -900,7 +964,7 @@ route4_pkts(struct rt_ctx *rt_ctx, struct rte_mbuf *pkts[], uint8_t nb_pkts)
 		}
 
 		if ((pkt_hop & RTE_LPM_LOOKUP_SUCCESS) == 0) {
-			rte_pktmbuf_free(pkts[i]);
+			free_pkts(&pkts[i], 1);
 			continue;
 		}
 		send_single_packet(pkts[i], pkt_hop & 0xff, IPPROTO_IP);
@@ -952,7 +1016,7 @@ route6_pkts(struct rt_ctx *rt_ctx, struct rte_mbuf *pkts[], uint8_t nb_pkts)
 		}
 
 		if (pkt_hop == -1) {
-			rte_pktmbuf_free(pkts[i]);
+			free_pkts(&pkts[i], 1);
 			continue;
 		}
 		send_single_packet(pkts[i], pkt_hop & 0xff, IPPROTO_IPV6);
@@ -1168,8 +1232,10 @@ ipsec_poll_mode_worker(void)
 			nb_rx = rte_eth_rx_burst(portid, queueid,
 					pkts, MAX_PKT_BURST);
 
-			if (nb_rx > 0)
+			if (nb_rx > 0) {
+				core_stats_update_rx(nb_rx);
 				process_pkts(qconf, pkts, nb_rx, portid);
+			}
 
 			/* dequeue and process completed crypto-ops */
 			if (is_unprotected_port(portid))
@@ -2916,6 +2982,12 @@ main(int32_t argc, char **argv)
 
 	check_all_ports_link_status(enabled_port_mask);
 
+#if (STATS_INTERVAL > 0)
+	rte_eal_alarm_set(STATS_INTERVAL * US_PER_S, print_stats_cb, NULL);
+#else
+	RTE_LOG(INFO, IPSEC, "Stats display disabled\n");
+#endif /* STATS_INTERVAL */
+
 	/* launch per-lcore init on every lcore */
 	rte_eal_mp_remote_launch(ipsec_launch_one_lcore, eh_conf, CALL_MASTER);
 	RTE_LCORE_FOREACH_SLAVE(lcore_id) {
examples/ipsec-secgw/ipsec-secgw.h +66 -0

82 lines not on this page · raw message

examples/ipsec-secgw/ipsec.c +10 -10
index bf88d80..01faa7a 100644
@@ -500,7 +500,7 @@ enqueue_cop_burst(struct cdev_qp *cqp)
 			cqp->id, cqp->qp, ret, len);
 			/* drop packets that we fail to enqueue */
 			for (i = ret; i < len; i++)
-				rte_pktmbuf_free(cqp->buf[i]->sym->m_src);
+				free_pkts(&cqp->buf[i]->sym->m_src, 1);
 	}
 	cqp->in_flight += ret;
 	cqp->len = 0;
@@ -528,7 +528,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 
 	for (i = 0; i < nb_pkts; i++) {
 		if (unlikely(sas[i] == NULL)) {
-			rte_pktmbuf_free(pkts[i]);
+			free_pkts(&pkts[i], 1);
 			continue;
 		}
 
@@ -549,7 +549,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 
 			if ((unlikely(ips->security.ses == NULL)) &&
 				create_lookaside_session(ipsec_ctx, sa, ips)) {
-				rte_pktmbuf_free(pkts[i]);
+				free_pkts(&pkts[i], 1);
 				continue;
 			}
 
@@ -563,7 +563,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 		case RTE_SECURITY_ACTION_TYPE_CPU_CRYPTO:
 			RTE_LOG(ERR, IPSEC, "CPU crypto is not supported by the"
 					" legacy mode.");
-			rte_pktmbuf_free(pkts[i]);
+			free_pkts(&pkts[i], 1);
 			continue;
 
 		case RTE_SECURITY_ACTION_TYPE_NONE:
@@ -575,7 +575,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 
 			if ((unlikely(ips->crypto.ses == NULL)) &&
 				create_lookaside_session(ipsec_ctx, sa, ips)) {
-				rte_pktmbuf_free(pkts[i]);
+				free_pkts(&pkts[i], 1);
 				continue;
 			}
 
@@ -584,7 +584,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 
 			ret = xform_func(pkts[i], sa, &priv->cop);
 			if (unlikely(ret)) {
-				rte_pktmbuf_free(pkts[i]);
+				free_pkts(&pkts[i], 1);
 				continue;
 			}
 			break;
@@ -608,7 +608,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 
 			ret = xform_func(pkts[i], sa, &priv->cop);
 			if (unlikely(ret)) {
-				rte_pktmbuf_free(pkts[i]);
+				free_pkts(&pkts[i], 1);
 				continue;
 			}
 
@@ -643,7 +643,7 @@ ipsec_inline_dequeue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 		sa = priv->sa;
 		ret = xform_func(pkt, sa, &priv->cop);
 		if (unlikely(ret)) {
-			rte_pktmbuf_free(pkt);
+			free_pkts(&pkt, 1);
 			continue;
 		}
 		pkts[nb_pkts++] = pkt;
@@ -690,13 +690,13 @@ ipsec_dequeue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 				RTE_SECURITY_ACTION_TYPE_NONE) {
 				ret = xform_func(pkt, sa, cops[j]);
 				if (unlikely(ret)) {
-					rte_pktmbuf_free(pkt);
+					free_pkts(&pkt, 1);
 					continue;
 				}
 			} else if (ipsec_get_action_type(sa) ==
 				RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) {
 				if (cops[j]->status) {
-					rte_pktmbuf_free(pkt);
+					free_pkts(&pkt, 1);
 					continue;
 				}
 			}
examples/ipsec-secgw/ipsec_process.c +1 -10
index 6d3a3c9..5012e1a 100644
@@ -12,22 +12,13 @@
 #include <rte_mbuf.h>
 
 #include "ipsec.h"
+#include "ipsec-secgw.h"
 
 #define SATP_OUT_IPV4(t)	\
 	((((t) & RTE_IPSEC_SATP_MODE_MASK) == RTE_IPSEC_SATP_MODE_TRANS && \
 	(((t) & RTE_IPSEC_SATP_IPV_MASK) == RTE_IPSEC_SATP_IPV4)) || \
 	((t) & RTE_IPSEC_SATP_MODE_MASK) == RTE_IPSEC_SATP_MODE_TUNLV4)
 
-/* helper routine to free bulk of packets */
-static inline void
-free_pkts(struct rte_mbuf *mb[], uint32_t n)
-{
-	uint32_t i;
-
-	for (i = 0; i != n; i++)
-		rte_pktmbuf_free(mb[i]);
-}
-
 /* helper routine to free bulk of crypto-ops and related packets */
 static inline void
 free_cops(struct rte_crypto_op *cop[], uint32_t n)
Signature · 1 line
2.7.4
Acked-by on v4 1/1
#14 lore raw
Quoted 67 lines

Adding per core packet handling stats to analyze traffic distribution when multiple cores are engaged.

Since aggregating the packet stats across cores would affect performance, keeping the feature disabled using compile time flags.

---
v4:
* Moved print stats routine to control thread
* Added stats for rx/tx pkts per call
* Replaced free_pkt(m) with free_pkts(&m, 1)

v3:
* Added wrapper functions for updating rx, tx & dropped counts
* Updated free_pkts() so that stats is updated internally
* Introduced similar free_pkt() function which updates stats and frees the packet
* Moved all inline functions and macros to ipsec-secgw.h
* Made STATS_INTERVAL macro to control the interval of the stats update. STATS_INTERVAL = 0 would disable the feature.

v2:
* Added lookup failure cases to drop count

 examples/ipsec-secgw/ipsec-secgw.c   | 114 ++++++++++++++++++++++++++++-------
 examples/ipsec-secgw/ipsec-secgw.h   |  66 ++++++++++++++++++++
 examples/ipsec-secgw/ipsec.c         |  20 +++---
 examples/ipsec-secgw/ipsec_process.c |  11 +---
 4 files changed, 170 insertions(+), 41 deletions(-)
examples/ipsec-secgw/ipsec-secgw.c +21 -0
index f777ce2..d2b5b2c 100644
@@ -47,6 +47,7 @@
 #include <rte_eventdev.h>
 #include <rte_ip.h>
 #include <rte_ip_frag.h>
+#include <rte_alarm.h>
 
 #include "event_helper.h"
 #include "ipsec.h"
@@ -287,6 +288,70 @@ adjust_ipv6_pktlen(struct rte_mbuf *m, const struct rte_ipv6_hdr *iph,
 	}
 }
 
+#if (STATS_INTERVAL > 0)
+
+/* Print out statistics on packet distribution */
+static void
+print_stats_cb(__rte_unused void *param)
+{
+	uint64_t total_packets_dropped, total_packets_tx, total_packets_rx;
+	float burst_percent, rx_per_call, tx_per_call;
+	unsigned int coreid;
+
+	total_packets_dropped = 0;
+	total_packets_tx = 0;
+	total_packets_rx = 0;
+
+	const char clr[] = { 27, '[', '2', 'J', '\0' };
+	const char topLeft[] = { 27, '[', '1', ';', '1', 'H', '\0' };
+
+	/* Clear screen and move to top left */
+	printf("%s%s", clr, topLeft);
+

examples/ipsec-secgw/ipsec-secgw.c adjust_ipv6_pktlen(struct rte_mbuf *m, const struct rte_ipv6_hdr *iph, · line 320

Quoted 10 lines
examples/ipsec-secgw/ipsec-secgw.c +10 -0
+	printf("\nCore statistics ====================================");
+
+	for (coreid = 0; coreid < RTE_MAX_LCORE; coreid++) {
+		/* skip disabled cores */
+		if (rte_lcore_is_enabled(coreid) == 0)
+			continue;
+		burst_percent = (float)(core_statistics[coreid].burst_rx * 100)/
+					core_statistics[coreid].rx;
+		rx_per_call =  (float)(core_statistics[coreid].rx)/
+				       core_statistics[coreid].rx_call;

As a nit - probably better to use double, and check that divisors are not zero. Apart from that - LGTM.

Quoted 34 lines
examples/ipsec-secgw/ipsec-secgw.c +34 -0
+		tx_per_call =  (float)(core_statistics[coreid].tx)/
+				       core_statistics[coreid].tx_call;
+		printf("\nStatistics for core %u ------------------------------"
+			   "\nPackets received: %20"PRIu64
+			   "\nPackets sent: %24"PRIu64
+			   "\nPackets dropped: %21"PRIu64
+			   "\nBurst percent: %23.2f"
+			   "\nPackets per Rx call: %17.2f"
+			   "\nPackets per Tx call: %17.2f",
+			   coreid,
+			   core_statistics[coreid].rx,
+			   core_statistics[coreid].tx,
+			   core_statistics[coreid].dropped,
+			   burst_percent,
+			   rx_per_call,
+			   tx_per_call);
+
+		total_packets_dropped += core_statistics[coreid].dropped;
+		total_packets_tx += core_statistics[coreid].tx;
+		total_packets_rx += core_statistics[coreid].rx;
+	}
+	printf("\nAggregate statistics ==============================="
+		   "\nTotal packets received: %14"PRIu64
+		   "\nTotal packets sent: %18"PRIu64
+		   "\nTotal packets dropped: %15"PRIu64,
+		   total_packets_rx,
+		   total_packets_tx,
+		   total_packets_dropped);
+	printf("\n====================================================\n");
+
+	rte_eal_alarm_set(STATS_INTERVAL * US_PER_S, print_stats_cb, NULL);
+}
+#endif /* STATS_INTERVAL */
+
#15 lore raw

Hi Anoob/Konstantin,

Quoted 9 lines
Quoted 9 lines

Adding per core packet handling stats to analyze traffic distribution when multiple cores are engaged.

Since aggregating the packet stats across cores would affect performance, keeping the feature disabled using compile time flags.

---

Since this patch got acked pretty late in the release cycle, we need to defer This patch for next release.
We cannot have these kind of patches in RC3.

Regards,
Akhil

#16 lore raw

Hi Akhil,

Quoted 11 lines

----------------------------------------------------------------------
Hi Anoob/Konstantin,

Quoted 7 lines
Quoted 7 lines

Adding per core packet handling stats to analyze traffic distribution when multiple cores are engaged.

Since aggregating the packet stats across cores would affect performance, keeping the feature disabled using compile time flags.

Quoted 7 lines
Quoted 2 lines
Quoted 2 lines

---

Since this patch got acked pretty late in the release cycle, we need to defer This patch for next release.
We cannot have these kind of patches in RC3.

Not a problem. Do you want me to resubmit this patch for next release? Or will you be able to merge it early in the next cycle?

Thanks,
Anoob

#17 lore raw
Quoted 16 lines

Hi Akhil,

Quoted 14 lines

----------------------------------------------------------------------
Hi Anoob/Konstantin,

Quoted 9 lines
Quoted 9 lines

Adding per core packet handling stats to analyze traffic distribution when multiple cores are engaged.

Since aggregating the packet stats across cores would affect performance, keeping the feature disabled using compile time flags.

---

Quoted 9 lines
Quoted 1 line

Since this patch got acked pretty late in the release cycle, we need to defer

This

Quoted 3 lines

patch for next release.
We cannot have these kind of patches in RC3.

Not a problem. Do you want me to resubmit this patch for next release? Or will you be able to merge it early in the next cycle?

No need to send it again as of now. I will merge it early in the next cycle.

Acked-by Applied
#18 lore raw
Quoted 24 lines
Quoted 24 lines

Hi Akhil,

Quoted 8 lines

----------------------------------------------------------------------
Hi Anoob/Konstantin,

Quoted 4 lines
Quoted 4 lines

Adding per core packet handling stats to analyze traffic distribution when multiple cores are engaged.

Quoted 7 lines
Quoted 5 lines
Quoted 5 lines

Since aggregating the packet stats across cores would affect performance, keeping the feature disabled using compile time flags.

---

Since this patch got acked pretty late in the release cycle, we need to defer

This

Quoted 3 lines

patch for next release.
We cannot have these kind of patches in RC3.

Quoted 5 lines
Quoted 1 line

Not a problem. Do you want me to resubmit this patch for next release? Or

will

Quoted 2 lines

you be able to merge it early in the next cycle?

No need to send it again as of now. I will merge it early in the next cycle.

Applied to dpdk-next-crypto

Thanks.