examples/ipsec-secgw: add per core packet stats
PATCH 1/1 examples/ipsec-secgw: add per core packet stats
Adding per core packet handling stats to analyze traffic distribution when multiple cores are engaged.
Since aggregating the packet stats across cores would affect performance, keeping the feature disabled using compile time flags.
--- examples/ipsec-secgw/ipsec-secgw.c | 112 +++++++++++++++++++++++++++++++++-- examples/ipsec-secgw/ipsec-secgw.h | 2 + examples/ipsec-secgw/ipsec.h | 22 +++++++ examples/ipsec-secgw/ipsec_process.c | 5 ++ 4 files changed, 137 insertions(+), 4 deletions(-)
examples/ipsec-secgw/ipsec-secgw.c +108 -4
@@ -288,6 +288,61 @@ adjust_ipv6_pktlen(struct rte_mbuf *m, const struct rte_ipv6_hdr *iph, } } +#ifdef ENABLE_STATS +static uint64_t timer_period =10 ;/* default period is 10 seconds */ + +/* Print out statistics on packet distribution */ +static void +print_stats (void ) +{ +uint64_t total_packets_dropped, total_packets_tx, total_packets_rx; +unsigned int coreid; +float burst_percent; + + total_packets_dropped =0 ; + total_packets_tx =0 ; + total_packets_rx =0 ; + +const char clr[] = {27 ,'[' ,'2' ,'J' ,'\0' }; +const char topLeft[] = {27 ,'[' ,'1' ,';' ,'1' ,'H' ,'\0' }; + +/* Clear screen and move to top left */ +printf ("%s%s" , clr, topLeft); + +printf ("\nCore statistics ====================================" ); + +for (coreid =0 ; coreid <RTE_MAX_LCORE ; coreid++) { +/* skip disabled cores */ +if (rte_lcore_is_enabled (coreid) ==0 ) +continue ; + burst_percent = (float )(core_statistics[coreid].burst_rx *100 )/ + core_statistics[coreid].rx; +printf ("\nStatistics for core %u ------------------------------" +"\nPackets received: %20" PRIu64 +"\nPackets sent: %24" PRIu64 +"\nPackets dropped: %21" PRIu64 +"\nBurst percent: %23.2f" , + coreid, + core_statistics[coreid].rx, + core_statistics[coreid].tx, + core_statistics[coreid].dropped, + burst_percent); + + total_packets_dropped += core_statistics[coreid].dropped; + total_packets_tx += core_statistics[coreid].tx; + total_packets_rx += core_statistics[coreid].rx; + } +printf ("\nAggregate statistics ===============================" +"\nTotal packets received: %14" PRIu64 +"\nTotal packets sent: %18" PRIu64 +"\nTotal packets dropped: %15" PRIu64, + total_packets_rx, + total_packets_tx, + total_packets_dropped); +printf ("\n====================================================\n" ); +} +#endif /* ENABLE_STATS */ +static inline void prepare_one_packet (struct rte_mbuf *pkt,struct ipsec_traffic *t) {
@@ -351,6 +406,7 @@ prepare_one_packet(struct rte_mbuf *pkt, struct ipsec_traffic *t)RTE_LOG (ERR ,IPSEC ,"Unsupported packet type 0x%x\n" ,rte_be_to_cpu_16 (eth->ether_type));rte_pktmbuf_free (pkt); +core_stats_update_drop (1 );return ; }
@@ -471,6 +527,11 @@ send_burst(struct lcore_conf *qconf, uint16_t n, uint16_t port)int32_t ret;uint16_t queueid; +#ifdef ENABLE_STATS +int lcore_id =rte_lcore_id (); + core_statistics[lcore_id].tx += n; +#endif /* ENABLE_STATS */ + queueid = qconf->tx_queue_id[port]; m_table = (struct rte_mbuf **)qconf->tx_mbufs[port].m_table;
@@ -478,6 +539,9 @@ send_burst(struct lcore_conf *qconf, uint16_t n, uint16_t port) ret =rte_eth_tx_burst (port, queueid, m_table, n);if (unlikely (ret < n)) { +#ifdef ENABLE_STATS + core_statistics[lcore_id].dropped += n-ret; +#endif /* ENABLE_STATS */ do {rte_pktmbuf_free (m_table[ret]); }while (++ret < n);
@@ -584,18 +648,21 @@ inbound_sp_sa(struct sp_ctx *sp, struct sa_ctx *sa, struct traffic_type *ip,continue ; }if (res ==DISCARD ) { +core_stats_update_drop (1 );rte_pktmbuf_free (m);continue ; }/* Only check SPI match for processed IPSec packets */ if (i < lim && ((m->ol_flags &PKT_RX_SEC_OFFLOAD ) ==0 )) { +core_stats_update_drop (1 );rte_pktmbuf_free (m);continue ; } sa_idx = res -1 ;if (!inbound_sa_check (sa, m, sa_idx)) { +core_stats_update_drop (1 );rte_pktmbuf_free (m);continue ; }
@@ -630,8 +697,10 @@ split46_traffic(struct ipsec_traffic *trf, struct rte_mbuf *mb[], uint32_t num) uint8_t *, offsetof(struct ip6_hdr, ip6_nxt)); n6++; - }else + }else { +core_stats_update_drop (1 );rte_pktmbuf_free (m); + } } trf->ip4.num = n4;
@@ -682,11 +751,12 @@ outbound_sp(struct sp_ctx *sp, struct traffic_type *ip,for (i =0 ; i < ip->num; i++) { m = ip->pkts[i]; sa_idx = ip->res[i] -1 ; -if (ip->res[i] ==DISCARD ) +if (ip->res[i] ==DISCARD ) { +core_stats_update_drop (1 );rte_pktmbuf_free (m); -else if (ip->res[i] ==BYPASS ) + }else if (ip->res[i] ==BYPASS ) { ip->pkts[j++] = m; -else { + }else { ipsec->res[ipsec->num] = sa_idx; ipsec->pkts[ipsec->num++] = m; }
@@ -705,6 +775,8 @@ process_pkts_outbound(struct ipsec_ctx *ipsec_ctx,for (i =0 ; i < traffic->ipsec.num; i++)rte_pktmbuf_free (traffic->ipsec.pkts[i]); +core_stats_update_drop (traffic->ipsec.num); + traffic->ipsec.num =0 ;outbound_sp (ipsec_ctx->sp4_ctx, &traffic->ip4, &traffic->ipsec);
@@ -745,12 +817,14 @@ process_pkts_inbound_nosp(struct ipsec_ctx *ipsec_ctx,/* Drop any IPv4 traffic from unprotected ports */ for (i =0 ; i < traffic->ip4.num; i++)rte_pktmbuf_free (traffic->ip4.pkts[i]); +core_stats_update_drop (traffic->ip4.num); traffic->ip4.num =0 ;/* Drop any IPv6 traffic from unprotected ports */ for (i =0 ; i < traffic->ip6.num; i++)rte_pktmbuf_free (traffic->ip6.pkts[i]); +core_stats_update_drop (traffic->ip6.num); traffic->ip6.num =0 ;
@@ -788,6 +862,7 @@ process_pkts_outbound_nosp(struct ipsec_ctx *ipsec_ctx,/* Drop any IPsec traffic from protected ports */ for (i =0 ; i < traffic->ipsec.num; i++)rte_pktmbuf_free (traffic->ipsec.pkts[i]); +core_stats_update_drop (traffic->ipsec.num); n =0 ;
@@ -901,6 +976,7 @@ route4_pkts(struct rt_ctx *rt_ctx, struct rte_mbuf *pkts[], uint8_t nb_pkts) }if ((pkt_hop &RTE_LPM_LOOKUP_SUCCESS ) ==0 ) { +core_stats_update_drop (1 );rte_pktmbuf_free (pkts[i]);continue ; }
@@ -953,6 +1029,7 @@ route6_pkts(struct rt_ctx *rt_ctx, struct rte_mbuf *pkts[], uint8_t nb_pkts) }if (pkt_hop ==-1 ) { +core_stats_update_drop (1 );rte_pktmbuf_free (pkts[i]);continue ; }
@@ -1099,6 +1176,9 @@ ipsec_poll_mode_worker(void)const uint64_t drain_tsc = (rte_get_tsc_hz () +US_PER_S -1 ) /US_PER_S *BURST_TX_DRAIN_US ;struct lcore_rx_queue *rxql; +#ifdef ENABLE_STATS +uint64_t timer_tsc =0 ; +#endif /* ENABLE_STATS */ prev_tsc =0 ; lcore_id =rte_lcore_id ();
@@ -1159,6 +1239,19 @@ ipsec_poll_mode_worker(void)drain_tx_buffers (qconf);drain_crypto_buffers (qconf); prev_tsc = cur_tsc; +#ifdef ENABLE_STATS +if (lcore_id ==rte_get_master_lcore ()) { +/* advance the timer */ + timer_tsc += diff_tsc; + +/* if timer has reached its timeout */ +if (unlikely (timer_tsc >= timer_period)) { +print_stats (); +/* reset the timer */ + timer_tsc =0 ; + } + } +#endif /* ENABLE_STATS */ }for (i =0 ; i < qconf->nb_rx_queue; ++i) {
@@ -1169,6 +1262,12 @@ ipsec_poll_mode_worker(void) nb_rx =rte_eth_rx_burst (portid, queueid, pkts,MAX_PKT_BURST ); +#ifdef ENABLE_STATS + core_statistics[lcore_id].rx += nb_rx; +if (nb_rx ==MAX_PKT_BURST ) + core_statistics[lcore_id].burst_rx += nb_rx; +#endif /* ENABLE_STATS */ +if (nb_rx >0 )process_pkts (qconf, pkts, nb_rx, portid);
@@ -2747,6 +2846,11 @@ main(int32_t argc, char **argv)signal (SIGINT , signal_handler);signal (SIGTERM , signal_handler); +#ifdef ENABLE_STATS +/* convert to number of cycles */ + timer_period *=rte_get_timer_hz (); +#endif /* ENABLE_STATS */ +/* initialize event helper configuration */ eh_conf =eh_conf_init ();if (eh_conf ==NULL )
examples/ipsec-secgw/ipsec-secgw.h +2 -0
@@ -6,6 +6,8 @@#include <stdbool.h> +//#define ENABLE_STATS +#define NB_SOCKETS 4#define MAX_PKT_BURST 32
examples/ipsec-secgw/ipsec.h +22 -0
@@ -46,6 +46,17 @@#define IP6_VERSION (6) +#ifdef ENABLE_STATS +struct ipsec_core_statistics { +uint64_t tx; +uint64_t rx; +uint64_t dropped; +uint64_t burst_rx; +} __rte_cache_aligned; + +struct ipsec_core_statistics core_statistics[RTE_MAX_ETHPORTS ]; +#endif /* ENABLE_STATS */ +struct rte_crypto_xform ;struct ipsec_xform ;struct rte_mbuf ;
@@ -416,4 +427,15 @@ check_flow_params(uint16_t fdir_portid, uint8_t fdir_qid);int create_ipsec_esp_flow (struct ipsec_sa *sa); +static inline void +core_stats_update_drop (int n) +{ +#ifdef ENABLE_STATS +int lcore_id =rte_lcore_id (); + core_statistics[lcore_id].dropped += n; +#else +RTE_SET_USED (n); +#endif /* ENABLE_STATS */ +} +#endif /* __IPSEC_H__ */
examples/ipsec-secgw/ipsec_process.c +5 -0
@@ -24,6 +24,11 @@ free_pkts(struct rte_mbuf *mb[], uint32_t n) {uint32_t i; +#ifdef ENABLE_STATS +int lcore_id =rte_lcore_id (); + core_statistics[lcore_id].dropped += n; +#endif /* ENABLE_STATS */ +for (i =0 ; i != n; i++)rte_pktmbuf_free (mb[i]); }
Signature · 1 line
2.7.4
PATCH v2 1/1 examples/ipsec-secgw: add per core packet stats
Adding per core packet handling stats to analyze traffic distribution when multiple cores are engaged.
Since aggregating the packet stats across cores would affect performance, keeping the feature disabled using compile time flags.
---
v2:
* Added lookup failure cases to drop count
examples/ipsec-secgw/ipsec-secgw.c | 118 +++++++++++++++++++++++++++++++++-- examples/ipsec-secgw/ipsec-secgw.h | 2 + examples/ipsec-secgw/ipsec.c | 13 +++- examples/ipsec-secgw/ipsec.h | 22 +++++++ examples/ipsec-secgw/ipsec_process.c | 5 ++ 5 files changed, 154 insertions(+), 6 deletions(-)
examples/ipsec-secgw/ipsec-secgw.c +113 -5
@@ -288,6 +288,61 @@ adjust_ipv6_pktlen(struct rte_mbuf *m, const struct rte_ipv6_hdr *iph, } } +#ifdef ENABLE_STATS +static uint64_t timer_period =10 ;/* default period is 10 seconds */ + +/* Print out statistics on packet distribution */ +static void +print_stats (void ) +{ +uint64_t total_packets_dropped, total_packets_tx, total_packets_rx; +unsigned int coreid; +float burst_percent; + + total_packets_dropped =0 ; + total_packets_tx =0 ; + total_packets_rx =0 ; + +const char clr[] = {27 ,'[' ,'2' ,'J' ,'\0' }; +const char topLeft[] = {27 ,'[' ,'1' ,';' ,'1' ,'H' ,'\0' }; + +/* Clear screen and move to top left */ +printf ("%s%s" , clr, topLeft); + +printf ("\nCore statistics ====================================" ); + +for (coreid =0 ; coreid <RTE_MAX_LCORE ; coreid++) { +/* skip disabled cores */ +if (rte_lcore_is_enabled (coreid) ==0 ) +continue ; + burst_percent = (float )(core_statistics[coreid].burst_rx *100 )/ + core_statistics[coreid].rx; +printf ("\nStatistics for core %u ------------------------------" +"\nPackets received: %20" PRIu64 +"\nPackets sent: %24" PRIu64 +"\nPackets dropped: %21" PRIu64 +"\nBurst percent: %23.2f" , + coreid, + core_statistics[coreid].rx, + core_statistics[coreid].tx, + core_statistics[coreid].dropped, + burst_percent); + + total_packets_dropped += core_statistics[coreid].dropped; + total_packets_tx += core_statistics[coreid].tx; + total_packets_rx += core_statistics[coreid].rx; + } +printf ("\nAggregate statistics ===============================" +"\nTotal packets received: %14" PRIu64 +"\nTotal packets sent: %18" PRIu64 +"\nTotal packets dropped: %15" PRIu64, + total_packets_rx, + total_packets_tx, + total_packets_dropped); +printf ("\n====================================================\n" ); +} +#endif /* ENABLE_STATS */ +static inline void prepare_one_packet (struct rte_mbuf *pkt,struct ipsec_traffic *t) {
@@ -333,6 +388,7 @@ prepare_one_packet(struct rte_mbuf *pkt, struct ipsec_traffic *t)/* drop packet when IPv6 header exceeds first segment length */ if (unlikely (l3len > pkt->data_len)) { +core_stats_update_drop (1 );rte_pktmbuf_free (pkt);return ; }
@@ -350,6 +406,7 @@ prepare_one_packet(struct rte_mbuf *pkt, struct ipsec_traffic *t)/* Unknown/Unsupported type, drop the packet */ RTE_LOG (ERR ,IPSEC ,"Unsupported packet type 0x%x\n" ,rte_be_to_cpu_16 (eth->ether_type)); +core_stats_update_drop (1 );rte_pktmbuf_free (pkt);return ; }
@@ -471,6 +528,11 @@ send_burst(struct lcore_conf *qconf, uint16_t n, uint16_t port)int32_t ret;uint16_t queueid; +#ifdef ENABLE_STATS +int lcore_id =rte_lcore_id (); + core_statistics[lcore_id].tx += n; +#endif /* ENABLE_STATS */ + queueid = qconf->tx_queue_id[port]; m_table = (struct rte_mbuf **)qconf->tx_mbufs[port].m_table;
@@ -478,6 +540,9 @@ send_burst(struct lcore_conf *qconf, uint16_t n, uint16_t port) ret =rte_eth_tx_burst (port, queueid, m_table, n);if (unlikely (ret < n)) { +#ifdef ENABLE_STATS + core_statistics[lcore_id].dropped += n-ret; +#endif /* ENABLE_STATS */ do {rte_pktmbuf_free (m_table[ret]); }while (++ret < n);
@@ -525,6 +590,7 @@ send_fragment_packet(struct lcore_conf *qconf, struct rte_mbuf *m,"error code: %d\n" , __func__, m->pkt_len, rte_errno); +core_stats_update_drop (1 );rte_pktmbuf_free (m);return len; }
@@ -549,8 +615,10 @@ send_single_packet(struct rte_mbuf *m, uint16_t port, uint8_t proto)/* need to fragment the packet */ }else if (frag_tbl_sz >0 ) len =send_fragment_packet (qconf, m, port, proto); -else +else { +core_stats_update_drop (1 );rte_pktmbuf_free (m); + }/* enough pkts to be sent */ if (unlikely (len ==MAX_PKT_BURST )) {
@@ -584,18 +652,21 @@ inbound_sp_sa(struct sp_ctx *sp, struct sa_ctx *sa, struct traffic_type *ip,continue ; }if (res ==DISCARD ) { +core_stats_update_drop (1 );rte_pktmbuf_free (m);continue ; }/* Only check SPI match for processed IPSec packets */ if (i < lim && ((m->ol_flags &PKT_RX_SEC_OFFLOAD ) ==0 )) { +core_stats_update_drop (1 );rte_pktmbuf_free (m);continue ; } sa_idx = res -1 ;if (!inbound_sa_check (sa, m, sa_idx)) { +core_stats_update_drop (1 );rte_pktmbuf_free (m);continue ; }
@@ -630,8 +701,10 @@ split46_traffic(struct ipsec_traffic *trf, struct rte_mbuf *mb[], uint32_t num) uint8_t *, offsetof(struct ip6_hdr, ip6_nxt)); n6++; - }else + }else { +core_stats_update_drop (1 );rte_pktmbuf_free (m); + } } trf->ip4.num = n4;
@@ -682,11 +755,12 @@ outbound_sp(struct sp_ctx *sp, struct traffic_type *ip,for (i =0 ; i < ip->num; i++) { m = ip->pkts[i]; sa_idx = ip->res[i] -1 ; -if (ip->res[i] ==DISCARD ) +if (ip->res[i] ==DISCARD ) { +core_stats_update_drop (1 );rte_pktmbuf_free (m); -else if (ip->res[i] ==BYPASS ) + }else if (ip->res[i] ==BYPASS ) { ip->pkts[j++] = m; -else { + }else { ipsec->res[ipsec->num] = sa_idx; ipsec->pkts[ipsec->num++] = m; }
@@ -705,6 +779,8 @@ process_pkts_outbound(struct ipsec_ctx *ipsec_ctx,for (i =0 ; i < traffic->ipsec.num; i++)rte_pktmbuf_free (traffic->ipsec.pkts[i]); +core_stats_update_drop (traffic->ipsec.num); + traffic->ipsec.num =0 ;outbound_sp (ipsec_ctx->sp4_ctx, &traffic->ip4, &traffic->ipsec);
@@ -745,12 +821,14 @@ process_pkts_inbound_nosp(struct ipsec_ctx *ipsec_ctx,/* Drop any IPv4 traffic from unprotected ports */ for (i =0 ; i < traffic->ip4.num; i++)rte_pktmbuf_free (traffic->ip4.pkts[i]); +core_stats_update_drop (traffic->ip4.num); traffic->ip4.num =0 ;/* Drop any IPv6 traffic from unprotected ports */ for (i =0 ; i < traffic->ip6.num; i++)rte_pktmbuf_free (traffic->ip6.pkts[i]); +core_stats_update_drop (traffic->ip6.num); traffic->ip6.num =0 ;
@@ -788,6 +866,7 @@ process_pkts_outbound_nosp(struct ipsec_ctx *ipsec_ctx,/* Drop any IPsec traffic from protected ports */ for (i =0 ; i < traffic->ipsec.num; i++)rte_pktmbuf_free (traffic->ipsec.pkts[i]); +core_stats_update_drop (traffic->ipsec.num); n =0 ;
@@ -901,6 +980,7 @@ route4_pkts(struct rt_ctx *rt_ctx, struct rte_mbuf *pkts[], uint8_t nb_pkts) }if ((pkt_hop &RTE_LPM_LOOKUP_SUCCESS ) ==0 ) { +core_stats_update_drop (1 );rte_pktmbuf_free (pkts[i]);continue ; }
@@ -953,6 +1033,7 @@ route6_pkts(struct rt_ctx *rt_ctx, struct rte_mbuf *pkts[], uint8_t nb_pkts) }if (pkt_hop ==-1 ) { +core_stats_update_drop (1 );rte_pktmbuf_free (pkts[i]);continue ; }
@@ -1099,6 +1180,9 @@ ipsec_poll_mode_worker(void)const uint64_t drain_tsc = (rte_get_tsc_hz () +US_PER_S -1 ) /US_PER_S *BURST_TX_DRAIN_US ;struct lcore_rx_queue *rxql; +#ifdef ENABLE_STATS +uint64_t timer_tsc =0 ; +#endif /* ENABLE_STATS */ prev_tsc =0 ; lcore_id =rte_lcore_id ();
@@ -1159,6 +1243,19 @@ ipsec_poll_mode_worker(void)drain_tx_buffers (qconf);drain_crypto_buffers (qconf); prev_tsc = cur_tsc; +#ifdef ENABLE_STATS +if (lcore_id ==rte_get_master_lcore ()) { +/* advance the timer */ + timer_tsc += diff_tsc; + +/* if timer has reached its timeout */ +if (unlikely (timer_tsc >= timer_period)) { +print_stats (); +/* reset the timer */ + timer_tsc =0 ; + } + } +#endif /* ENABLE_STATS */ }for (i =0 ; i < qconf->nb_rx_queue; ++i) {
@@ -1169,6 +1266,12 @@ ipsec_poll_mode_worker(void) nb_rx =rte_eth_rx_burst (portid, queueid, pkts,MAX_PKT_BURST ); +#ifdef ENABLE_STATS + core_statistics[lcore_id].rx += nb_rx; +if (nb_rx ==MAX_PKT_BURST ) + core_statistics[lcore_id].burst_rx += nb_rx; +#endif /* ENABLE_STATS */ +if (nb_rx >0 )process_pkts (qconf, pkts, nb_rx, portid);
@@ -2747,6 +2850,11 @@ main(int32_t argc, char **argv)signal (SIGINT , signal_handler);signal (SIGTERM , signal_handler); +#ifdef ENABLE_STATS +/* convert to number of cycles */ + timer_period *=rte_get_timer_hz (); +#endif /* ENABLE_STATS */ +/* initialize event helper configuration */ eh_conf =eh_conf_init ();if (eh_conf ==NULL )
examples/ipsec-secgw/ipsec-secgw.h +2 -0
@@ -6,6 +6,8 @@#include <stdbool.h> +//#define ENABLE_STATS +#define NB_SOCKETS 4#define MAX_PKT_BURST 32
examples/ipsec-secgw/ipsec.c +12 -1
@@ -499,8 +499,10 @@ enqueue_cop_burst(struct cdev_qp *cqp)" enqueued %u crypto ops out of %u\n" , cqp->id, cqp->qp, ret, len);/* drop packets that we fail to enqueue */ -for (i = ret; i < len; i++) +for (i = ret; i < len; i++) { +core_stats_update_drop (1 );rte_pktmbuf_free (cqp->buf[i]->sym->m_src); + } } cqp->in_flight += ret; cqp->len =0 ;
@@ -528,6 +530,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,for (i =0 ; i < nb_pkts; i++) {if (unlikely (sas[i] ==NULL )) { +core_stats_update_drop (1 );rte_pktmbuf_free (pkts[i]);continue ; }
@@ -549,6 +552,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,if ((unlikely (ips->security.ses ==NULL )) &&create_lookaside_session (ipsec_ctx, sa, ips)) { +core_stats_update_drop (1 );rte_pktmbuf_free (pkts[i]);continue ; }
@@ -563,6 +567,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,case RTE_SECURITY_ACTION_TYPE_CPU_CRYPTO :RTE_LOG (ERR ,IPSEC ,"CPU crypto is not supported by the" " legacy mode." ); +core_stats_update_drop (1 );rte_pktmbuf_free (pkts[i]);continue ;
@@ -575,6 +580,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,if ((unlikely (ips->crypto.ses ==NULL )) &&create_lookaside_session (ipsec_ctx, sa, ips)) { +core_stats_update_drop (1 );rte_pktmbuf_free (pkts[i]);continue ; }
@@ -584,6 +590,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx, ret =xform_func (pkts[i], sa, &priv->cop);if (unlikely (ret)) { +core_stats_update_drop (1 );rte_pktmbuf_free (pkts[i]);continue ; }
@@ -608,6 +615,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx, ret =xform_func (pkts[i], sa, &priv->cop);if (unlikely (ret)) { +core_stats_update_drop (1 );rte_pktmbuf_free (pkts[i]);continue ; }
@@ -643,6 +651,7 @@ ipsec_inline_dequeue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx, sa = priv->sa; ret =xform_func (pkt, sa, &priv->cop);if (unlikely (ret)) { +core_stats_update_drop (1 );rte_pktmbuf_free (pkt);continue ; }
@@ -690,12 +699,14 @@ ipsec_dequeue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx, RTE_SECURITY_ACTION_TYPE_NONE) { ret =xform_func (pkt, sa, cops[j]);if (unlikely (ret)) { +core_stats_update_drop (1 );rte_pktmbuf_free (pkt);continue ; } } else if (ipsec_get_action_type(sa) ==RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL ) {if (cops[j]->status) { +core_stats_update_drop (1 );rte_pktmbuf_free (pkt);continue ; }
examples/ipsec-secgw/ipsec.h +22 -0
@@ -46,6 +46,17 @@#define IP6_VERSION (6) +#ifdef ENABLE_STATS +struct ipsec_core_statistics { +uint64_t tx; +uint64_t rx; +uint64_t dropped; +uint64_t burst_rx; +} __rte_cache_aligned; + +struct ipsec_core_statistics core_statistics[RTE_MAX_ETHPORTS ]; +#endif /* ENABLE_STATS */ +struct rte_crypto_xform ;struct ipsec_xform ;struct rte_mbuf ;
@@ -416,4 +427,15 @@ check_flow_params(uint16_t fdir_portid, uint8_t fdir_qid);int create_ipsec_esp_flow (struct ipsec_sa *sa); +static inline void +core_stats_update_drop (int n) +{ +#ifdef ENABLE_STATS +int lcore_id =rte_lcore_id (); + core_statistics[lcore_id].dropped += n; +#else +RTE_SET_USED (n); +#endif /* ENABLE_STATS */ +} +#endif /* __IPSEC_H__ */
examples/ipsec-secgw/ipsec_process.c +5 -0
@@ -24,6 +24,11 @@ free_pkts(struct rte_mbuf *mb[], uint32_t n) {uint32_t i; +#ifdef ENABLE_STATS +int lcore_id =rte_lcore_id (); + core_statistics[lcore_id].dropped += n; +#endif /* ENABLE_STATS */ +for (i =0 ; i != n; i++)rte_pktmbuf_free (mb[i]); }
Signature · 1 line
2.7.4
Quoted 25 lines
Adding per core packet handling stats to analyze traffic distribution when multiple cores are engaged.
Since aggregating the packet stats across cores would affect performance, keeping the feature disabled using compile time flags.
Signed-off-by Anoob Joseph <[email protected]>---
v2:
* Added lookup failure cases to drop countexamples/ipsec-secgw/ipsec-secgw.c | 118 +++++++++++++++++++++++++++++++++-- examples/ipsec-secgw/ipsec-secgw.h | 2 + examples/ipsec-secgw/ipsec.c | 13 +++- examples/ipsec-secgw/ipsec.h | 22 +++++++ examples/ipsec-secgw/ipsec_process.c | 5 ++ 5 files changed, 154 insertions(+), 6 deletions(-)examples/ipsec-secgw/ipsec-secgw.c +0
-0
examples/ipsec-secgw/ipsec-secgw.c adjust_ipv6_pktlen(struct rte_mbuf *m, const struct rte_ipv6_hdr *iph, · line 292
Quoted 5 lines
examples/ipsec-secgw/ipsec-secgw.c +2
-0@@ -288,6 +288,61 @@ adjust_ipv6_pktlen(struct rte_mbuf *m, const struct rte_ipv6_hdr *iph, } } +#ifdef ENABLE_STATS +static uint64_t timer_period =10 ;/* default period is 10 seconds */
I think it is better to add user ability to control stats period. Either runtime-option, or just compile time:
replace ENABLE_STATS with STATS_PERIOD (0 would mean stats disabled).
Quoted 8 lines
examples/ipsec-secgw/ipsec-secgw.c +8
-0+ +/* Print out statistics on packet distribution */ +static void +print_stats (void ) +{ +uint64_t total_packets_dropped, total_packets_tx, total_packets_rx; +unsigned int coreid; +float burst_percent;
examples/ipsec-secgw/ipsec-secgw.c adjust_ipv6_pktlen(struct rte_mbuf *m, const struct rte_ipv6_hdr *iph, · line 310
Quoted 10 lines
examples/ipsec-secgw/ipsec-secgw.c +10
-0+ + total_packets_dropped =0 ; + total_packets_tx =0 ; + total_packets_rx =0 ; + +const char clr[] = {27 ,'[' ,'2' ,'J' ,'\0' }; +const char topLeft[] = {27 ,'[' ,'1' ,';' ,'1' ,'H' ,'\0' }; + +/* Clear screen and move to top left */ +printf ("%s%s" , clr, topLeft);
Is that really needed?
examples/ipsec-secgw/ipsec-secgw.c adjust_ipv6_pktlen(struct rte_mbuf *m, const struct rte_ipv6_hdr *iph, · line 319
Quoted 9 lines
examples/ipsec-secgw/ipsec-secgw.c +9
-0+ +printf ("\nCore statistics ====================================" ); + +for (coreid =0 ; coreid <RTE_MAX_LCORE ; coreid++) { +/* skip disabled cores */ +if (rte_lcore_is_enabled (coreid) ==0 ) +continue ; + burst_percent = (float )(core_statistics[coreid].burst_rx *100 )/ + core_statistics[coreid].rx;
Would float be always enough here? Might better long double?
Quoted 46 lines
examples/ipsec-secgw/ipsec-secgw.c +28
-0+printf ("\nStatistics for core %u ------------------------------" +"\nPackets received: %20" PRIu64 +"\nPackets sent: %24" PRIu64 +"\nPackets dropped: %21" PRIu64 +"\nBurst percent: %23.2f" , + coreid, + core_statistics[coreid].rx, + core_statistics[coreid].tx, + core_statistics[coreid].dropped, + burst_percent); + + total_packets_dropped += core_statistics[coreid].dropped; + total_packets_tx += core_statistics[coreid].tx; + total_packets_rx += core_statistics[coreid].rx; + } +printf ("\nAggregate statistics ===============================" +"\nTotal packets received: %14" PRIu64 +"\nTotal packets sent: %18" PRIu64 +"\nTotal packets dropped: %15" PRIu64, + total_packets_rx, + total_packets_tx, + total_packets_dropped); +printf ("\n====================================================\n" ); +} +#endif /* ENABLE_STATS */ +static inline void prepare_one_packet (struct rte_mbuf *pkt,struct ipsec_traffic *t) {@@ -333,6 +388,7 @@ prepare_one_packet(struct rte_mbuf *pkt, struct ipsec_traffic *t)/* drop packet when IPv6 header exceeds first segment length */ if (unlikely (l3len > pkt->data_len)) { +core_stats_update_drop (1 );rte_pktmbuf_free (pkt);return ; }@@ -350,6 +406,7 @@ prepare_one_packet(struct rte_mbuf *pkt, struct ipsec_traffic *t)/* Unknown/Unsupported type, drop the packet */ RTE_LOG (ERR ,IPSEC ,"Unsupported packet type 0x%x\n" ,rte_be_to_cpu_16 (eth->ether_type)); +core_stats_update_drop (1 );rte_pktmbuf_free (pkt);return ; }
examples/ipsec-secgw/ipsec-secgw.c send_burst(struct lcore_conf *qconf, uint16_t n, uint16_t port) · line 534
Quoted 7 lines
examples/ipsec-secgw/ipsec-secgw.c +4
-0@@ -471,6 +528,11 @@ send_burst(struct lcore_conf *qconf, uint16_t n, uint16_t port)int32_t ret;uint16_t queueid; +#ifdef ENABLE_STATS +int lcore_id =rte_lcore_id (); + core_statistics[lcore_id].tx += n; +#endif /* ENABLE_STATS */
Instead of polluting genric code with ifdefs, why not
to introduce 2 new functions: core_stats_update_rx(), core_stats_update_tx(), as you did for core_stats_drop()?
examples/ipsec-secgw/ipsec-secgw.c send_burst(struct lcore_conf *qconf, uint16_t n, uint16_t port) · line 545
Quoted 11 lines
examples/ipsec-secgw/ipsec-secgw.c +4
-0+ queueid = qconf->tx_queue_id[port]; m_table = (struct rte_mbuf **)qconf->tx_mbufs[port].m_table;@@ -478,6 +540,9 @@ send_burst(struct lcore_conf *qconf, uint16_t n, uint16_t port) ret =rte_eth_tx_burst (port, queueid, m_table, n);if (unlikely (ret < n)) { +#ifdef ENABLE_STATS + core_statistics[lcore_id].dropped += n-ret; +#endif /* ENABLE_STATS */
You have core_stats_update_drop() for that - use it.
Quoted 12 lines
examples/ipsec-secgw/ipsec-secgw.c +1
-0do {rte_pktmbuf_free (m_table[ret]); }while (++ret < n);@@ -525,6 +590,7 @@ send_fragment_packet(struct lcore_conf *qconf, struct rte_mbuf *m,"error code: %d\n" , __func__, m->pkt_len, rte_errno); +core_stats_update_drop (1 );rte_pktmbuf_free (m);return len; }
examples/ipsec-secgw/ipsec-secgw.c send_single_packet(struct rte_mbuf *m, uint16_t port, uint8_t proto) · line 620
Quoted 7 lines
examples/ipsec-secgw/ipsec-secgw.c +2
-1@@ -549,8 +615,10 @@ send_single_packet(struct rte_mbuf *m, uint16_t port, uint8_t proto)/* need to fragment the packet */ }else if (frag_tbl_sz >0 ) len =send_fragment_packet (qconf, m, port, proto); -else +else { +core_stats_update_drop (1 );rte_pktmbuf_free (m);
It looks like a lot of such places...
Would it be worth to unite core_stats_update_drop() and rte_pktmbuf_free(m) Into some inline function: ipsec_secgw_packet_drop(struct rte_mbuf *m[], uint32_t n) and use it all over such places.
Quoted 39 lines
examples/ipsec-secgw/ipsec-secgw.c +7
-1+ }/* enough pkts to be sent */ if (unlikely (len ==MAX_PKT_BURST )) {@@ -584,18 +652,21 @@ inbound_sp_sa(struct sp_ctx *sp, struct sa_ctx *sa, struct traffic_type *ip,continue ; }if (res ==DISCARD ) { +core_stats_update_drop (1 );rte_pktmbuf_free (m);continue ; }/* Only check SPI match for processed IPSec packets */ if (i < lim && ((m->ol_flags &PKT_RX_SEC_OFFLOAD ) ==0 )) { +core_stats_update_drop (1 );rte_pktmbuf_free (m);continue ; } sa_idx = res -1 ;if (!inbound_sa_check (sa, m, sa_idx)) { +core_stats_update_drop (1 );rte_pktmbuf_free (m);continue ; }@@ -630,8 +701,10 @@ split46_traffic(struct ipsec_traffic *trf, struct rte_mbuf *mb[], uint32_t num) uint8_t *, offsetof(struct ip6_hdr, ip6_nxt)); n6++; - }else + }else { +core_stats_update_drop (1 );rte_pktmbuf_free (m); + } } trf->ip4.num = n4;
examples/ipsec-secgw/ipsec-secgw.c outbound_sp(struct sp_ctx *sp, struct traffic_type *ip, · line 761
Quoted 9 lines
examples/ipsec-secgw/ipsec-secgw.c +3
-2@@ -682,11 +755,12 @@ outbound_sp(struct sp_ctx *sp, struct traffic_type *ip,for (i =0 ; i < ip->num; i++) { m = ip->pkts[i]; sa_idx = ip->res[i] -1 ; -if (ip->res[i] ==DISCARD ) +if (ip->res[i] ==DISCARD ) { +core_stats_update_drop (1 );rte_pktmbuf_free (m); -else if (ip->res[i] ==BYPASS ) + }else if (ip->res[i] ==BYPASS ) {
Looks unnecessary.
Quoted 55 lines
examples/ipsec-secgw/ipsec-secgw.c +8
-1ip->pkts[j++] = m; - else { + }else { ipsec->res[ipsec->num] = sa_idx; ipsec->pkts[ipsec->num++] = m; }@@ -705,6 +779,8 @@ process_pkts_outbound(struct ipsec_ctx *ipsec_ctx,for (i =0 ; i < traffic->ipsec.num; i++)rte_pktmbuf_free (traffic->ipsec.pkts[i]); +core_stats_update_drop (traffic->ipsec.num); + traffic->ipsec.num =0 ;outbound_sp (ipsec_ctx->sp4_ctx, &traffic->ip4, &traffic->ipsec);@@ -745,12 +821,14 @@ process_pkts_inbound_nosp(struct ipsec_ctx *ipsec_ctx,/* Drop any IPv4 traffic from unprotected ports */ for (i =0 ; i < traffic->ip4.num; i++)rte_pktmbuf_free (traffic->ip4.pkts[i]); +core_stats_update_drop (traffic->ip4.num); traffic->ip4.num =0 ;/* Drop any IPv6 traffic from unprotected ports */ for (i =0 ; i < traffic->ip6.num; i++)rte_pktmbuf_free (traffic->ip6.pkts[i]); +core_stats_update_drop (traffic->ip6.num); traffic->ip6.num =0 ;@@ -788,6 +866,7 @@ process_pkts_outbound_nosp(struct ipsec_ctx *ipsec_ctx,/* Drop any IPsec traffic from protected ports */ for (i =0 ; i < traffic->ipsec.num; i++)rte_pktmbuf_free (traffic->ipsec.pkts[i]); +core_stats_update_drop (traffic->ipsec.num); n =0 ;@@ -901,6 +980,7 @@ route4_pkts(struct rt_ctx *rt_ctx, struct rte_mbuf *pkts[], uint8_t nb_pkts) }if ((pkt_hop &RTE_LPM_LOOKUP_SUCCESS ) ==0 ) { +core_stats_update_drop (1 );rte_pktmbuf_free (pkts[i]);continue ; }@@ -953,6 +1033,7 @@ route6_pkts(struct rt_ctx *rt_ctx, struct rte_mbuf *pkts[], uint8_t nb_pkts) }if (pkt_hop ==-1 ) { +core_stats_update_drop (1 );rte_pktmbuf_free (pkts[i]);continue ; }
examples/ipsec-secgw/ipsec-secgw.c ipsec_poll_mode_worker(void) · line 1185
Quoted 6 lines
examples/ipsec-secgw/ipsec-secgw.c +3
-0@@ -1099,6 +1180,9 @@ ipsec_poll_mode_worker(void)const uint64_t drain_tsc = (rte_get_tsc_hz () +US_PER_S -1 ) /US_PER_S *BURST_TX_DRAIN_US ;struct lcore_rx_queue *rxql; +#ifdef ENABLE_STATS +uint64_t timer_tsc =0 ; +#endif /* ENABLE_STATS */
Probably better just RTE_SET_USED(timer_tsc);
Quoted 10 lines
examples/ipsec-secgw/ipsec-secgw.c +3
-0prev_tsc =0 ; lcore_id =rte_lcore_id ();@@ -1159,6 +1243,19 @@ ipsec_poll_mode_worker(void)drain_tx_buffers (qconf);drain_crypto_buffers (qconf); prev_tsc = cur_tsc; +#ifdef ENABLE_STATS +if (lcore_id ==rte_get_master_lcore ()) { +/* advance the timer */
examples/ipsec-secgw/ipsec-secgw.c ipsec_poll_mode_worker(void) · line 1258
Quoted 10 lines
examples/ipsec-secgw/ipsec-secgw.c +10
-0+ timer_tsc += diff_tsc; + +/* if timer has reached its timeout */ +if (unlikely (timer_tsc >= timer_period)) { +print_stats (); +/* reset the timer */ + timer_tsc =0 ; + } + } +#endif /* ENABLE_STATS */
Why to do stats collection/display inside data-path?
Why not use rte_timer/rte_alarm and make it happen in control thread? Another option - make stats printing at some signal to the process. In that case we don't need to bother with time period at all - it will be user to decide.
Again if we remove that print_stats() from data-path it might become cheap enough to always collect it, and we will not need ENABLE_STATS macro at all.
examples/ipsec-secgw/ipsec-secgw.c ipsec_poll_mode_worker(void) · line 1274
Quoted 13 lines
examples/ipsec-secgw/ipsec-secgw.c +6
-0}for (i =0 ; i < qconf->nb_rx_queue; ++i) {@@ -1169,6 +1266,12 @@ ipsec_poll_mode_worker(void) nb_rx =rte_eth_rx_burst (portid, queueid, pkts,MAX_PKT_BURST ); +#ifdef ENABLE_STATS + core_statistics[lcore_id].rx += nb_rx; +if (nb_rx ==MAX_PKT_BURST ) + core_statistics[lcore_id].burst_rx += nb_rx; +#endif /* ENABLE_STATS */ +
Same for above for TX: no need to pollute the code with ifdefs. Better to introduce new function: stats_update_rx() or so.
Quoted 20 lines
examples/ipsec-secgw/ipsec-secgw.c +5
-0if (nb_rx >0 )process_pkts (qconf, pkts, nb_rx, portid);@@ -2747,6 +2850,11 @@ main(int32_t argc, char **argv)signal (SIGINT , signal_handler);signal (SIGTERM , signal_handler); +#ifdef ENABLE_STATS +/* convert to number of cycles */ + timer_period *=rte_get_timer_hz (); +#endif /* ENABLE_STATS */ +/* initialize event helper configuration */ eh_conf =eh_conf_init ();if (eh_conf ==NULL )examples/ipsec-secgw/ipsec-secgw.h +0
-0
examples/ipsec-secgw/ipsec-secgw.h line 10
Quoted 5 lines
examples/ipsec-secgw/ipsec-secgw.h +2
-0@@ -6,6 +6,8 @@#include <stdbool.h> +//#define ENABLE_STATS +
Should be removed I think.
Quoted 95 lines
examples/ipsec-secgw/ipsec-secgw.h +0
-0#define NB_SOCKETS 4#define MAX_PKT_BURST 32examples/ipsec-secgw/ipsec.c +12
-1@@ -499,8 +499,10 @@ enqueue_cop_burst(struct cdev_qp *cqp)" enqueued %u crypto ops out of %u\n" , cqp->id, cqp->qp, ret, len);/* drop packets that we fail to enqueue */ -for (i = ret; i < len; i++) +for (i = ret; i < len; i++) { +core_stats_update_drop (1 );rte_pktmbuf_free (cqp->buf[i]->sym->m_src); + } } cqp->in_flight += ret; cqp->len =0 ;@@ -528,6 +530,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,for (i =0 ; i < nb_pkts; i++) {if (unlikely (sas[i] ==NULL )) { +core_stats_update_drop (1 );rte_pktmbuf_free (pkts[i]);continue ; }@@ -549,6 +552,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,if ((unlikely (ips->security.ses ==NULL )) &&create_lookaside_session (ipsec_ctx, sa, ips)) { +core_stats_update_drop (1 );rte_pktmbuf_free (pkts[i]);continue ; }@@ -563,6 +567,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,case RTE_SECURITY_ACTION_TYPE_CPU_CRYPTO :RTE_LOG (ERR ,IPSEC ,"CPU crypto is not supported by the" " legacy mode." ); +core_stats_update_drop (1 );rte_pktmbuf_free (pkts[i]);continue ;@@ -575,6 +580,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,if ((unlikely (ips->crypto.ses ==NULL )) &&create_lookaside_session (ipsec_ctx, sa, ips)) { +core_stats_update_drop (1 );rte_pktmbuf_free (pkts[i]);continue ; }@@ -584,6 +590,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx, ret =xform_func (pkts[i], sa, &priv->cop);if (unlikely (ret)) { +core_stats_update_drop (1 );rte_pktmbuf_free (pkts[i]);continue ; }@@ -608,6 +615,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx, ret =xform_func (pkts[i], sa, &priv->cop);if (unlikely (ret)) { +core_stats_update_drop (1 );rte_pktmbuf_free (pkts[i]);continue ; }@@ -643,6 +651,7 @@ ipsec_inline_dequeue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx, sa = priv->sa; ret =xform_func (pkt, sa, &priv->cop);if (unlikely (ret)) { +core_stats_update_drop (1 );rte_pktmbuf_free (pkt);continue ; }@@ -690,12 +699,14 @@ ipsec_dequeue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx, RTE_SECURITY_ACTION_TYPE_NONE) { ret =xform_func (pkt, sa, cops[j]);if (unlikely (ret)) { +core_stats_update_drop (1 );rte_pktmbuf_free (pkt);continue ; } } else if (ipsec_get_action_type(sa) ==RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL ) {if (cops[j]->status) { +core_stats_update_drop (1 );rte_pktmbuf_free (pkt);continue ; }examples/ipsec-secgw/ipsec.h +0
-0
examples/ipsec-secgw/ipsec.h line 54
Quoted 9 lines
examples/ipsec-secgw/ipsec.h +6
-0@@ -46,6 +46,17 @@#define IP6_VERSION (6) +#ifdef ENABLE_STATS +struct ipsec_core_statistics { + uint64_t tx; + uint64_t rx; + uint64_t dropped; + uint64_t burst_rx;
A bit strange to have burst_rx and no similar counterpart for tx. BTW, do you need burst_rx?
Might be better:nb_calls_rx, nb_calls_tx;
and then: rx/nb_calls_rx will give you average burst size.
examples/ipsec-secgw/ipsec.h line 57
Quoted 3 lines
examples/ipsec-secgw/ipsec.h +3
-0+} __rte_cache_aligned; + +struct ipsec_core_statistics core_statistics[RTE_MAX_ETHPORTS ];
Should be RTE_MAX_LCORES, I think.
Quoted 26 lines
examples/ipsec-secgw/ipsec.h +13
-0+#endif /* ENABLE_STATS */ +struct rte_crypto_xform ;struct ipsec_xform ;struct rte_mbuf ;@@ -416,4 +427,15 @@ check_flow_params(uint16_t fdir_portid, uint8_t fdir_qid);int create_ipsec_esp_flow (struct ipsec_sa *sa); +static inline void +core_stats_update_drop (int n) +{ +#ifdef ENABLE_STATS +int lcore_id =rte_lcore_id (); + core_statistics[lcore_id].dropped += n; +#else +RTE_SET_USED (n); +#endif /* ENABLE_STATS */ +} +#endif /* __IPSEC_H__ */ examples/ipsec-secgw/ipsec_process.c +0
-0
examples/ipsec-secgw/ipsec_process.c free_pkts(struct rte_mbuf *mb[], uint32_t n) · line 31
Quoted 8 lines
examples/ipsec-secgw/ipsec_process.c +5
-0@@ -24,6 +24,11 @@ free_pkts(struct rte_mbuf *mb[], uint32_t n) {uint32_t i; +#ifdef ENABLE_STATS +int lcore_id =rte_lcore_id (); + core_statistics[lcore_id].dropped += n; +#endif /* ENABLE_STATS */ +
Same as above - why not use stats_update_drop() here?
Quoted 5 lines
examples/ipsec-secgw/ipsec_process.c +0
-1for (i =0 ; i != n; i++)rte_pktmbuf_free (mb[i]); } --2.7.4
Hi Konstantin,
Thanks for the review. Please see inline.
@Akhil, do you have any comments?
Thanks,
Anoob
Quoted 43 lines
-----Original Message-----
From: Ananyev, Konstantin <[email protected]>
Sent: Friday, April 24, 2020 4:44 PM
To: Anoob Joseph <[email protected]>; Akhil Goyal <[email protected]>; Nicolau, Radu <[email protected]>
Cc: Narayana Prasad Raju Athreya <[email protected]>; [email protected]
Subject: [EXT] RE: [dpdk-dev] [PATCH v2] examples/ipsec-secgw: add per core packet statsExternal Email
----------------------------------------------------------------------
Quoted 6 lines
Adding per core packet handling stats to analyze traffic distribution when multiple cores are engaged.
Since aggregating the packet stats across cores would affect performance, keeping the feature disabled using compile time flags.
Quoted 7 lines
Signed-off-by Anoob Joseph <[email protected]>---
v2:
* Added lookup failure cases to drop countexamples/ipsec-secgw/ipsec-secgw.c | 118+++++++++++++++++++++++++++++++++--
examples/ipsec-secgw/ipsec-secgw.c adjust_ipv6_pktlen(struct rte_mbuf *m, const struct
Quoted 12 lines
examples/ipsec-secgw/ipsec-secgw.h | 2 + examples/ipsec-secgw/ipsec.c | 13 +++- examples/ipsec-secgw/ipsec.h | 22 +++++++ examples/ipsec-secgw/ipsec_process.c | 5 ++ 5 files changed, 154 insertions(+), 6 deletions(-)examples/ipsec-secgw/ipsec-secgw.c +0
-0@@ -288,6 +288,61 @@ adjust_ipv6_pktlen(struct rte_mbuf *m, const struct
rte_ipv6_hdr*iph,Quoted 3 lines
examples/ipsec-secgw/ipsec-secgw.c +0
-0} }
Quoted 6 lines
examples/ipsec-secgw/ipsec-secgw.c adjust_ipv6_pktlen(struct rte_mbuf *m, const struct rte_ipv6_hdr *iph, · line 292
Quoted 2 lines
examples/ipsec-secgw/ipsec-secgw.c +2
-0+#ifdef ENABLE_STATS +static uint64_t timer_period =10 ;/* default period is 10 seconds */ I think it is better to add user ability to control stats period. Either runtime-option, or just compile time:
replaceENABLE_STATSwithSTATS_PERIOD(0 would mean stats disabled).
[Anoob] That's a good suggestion. Will make this change as part of v3.
Quoted 12 lines
Quoted 12 lines
examples/ipsec-secgw/ipsec-secgw.c +11
-0+ +/* Print out statistics on packet distribution */ static void +print_stats (void ) +{ +uint64_t total_packets_dropped, total_packets_tx, total_packets_rx; +unsigned int coreid; +float burst_percent; + + total_packets_dropped =0 ; + total_packets_tx =0 ; + total_packets_rx =0 ;
Quoted 8 lines
examples/ipsec-secgw/ipsec-secgw.c adjust_ipv6_pktlen(struct rte_mbuf *m, const struct rte_ipv6_hdr *iph, · line 310
Quoted 6 lines
examples/ipsec-secgw/ipsec-secgw.c +6
-0+ +const char clr[] = {27 ,'[' ,'2' ,'J' ,'\0' }; +const char topLeft[] = {27 ,'[' ,'1' ,';' ,'1' ,'H' ,'\0' }; + +/* Clear screen and move to top left */ +printf ("%s%s" , clr, topLeft);Is that really needed?
[Anoob] I had just copied from l2fwd and worked from that. But it seems like, 'topleft' is not needed. Will remove that. Without a clearscreen, the prints would just crowd the screen. I would like to retain clearscreen.
Quoted 12 lines
examples/ipsec-secgw/ipsec-secgw.c adjust_ipv6_pktlen(struct rte_mbuf *m, const struct rte_ipv6_hdr *iph, · line 319
Quoted 10 lines
examples/ipsec-secgw/ipsec-secgw.c +9
-0+ +printf ("\nCore statistics ====================================" ); + +for (coreid =0 ; coreid <RTE_MAX_LCORE ; coreid++) { +/* skip disabled cores */ +if (rte_lcore_is_enabled (coreid) ==0 ) +continue ; + burst_percent = (float )(core_statistics[coreid].burst_rx *100 )/ + core_statistics[coreid].rx;Would float be always enough here? Might better long double?
[Anoob] This is just to get the percentage of traffic doing bursts. Since it's percentage float seemed enough. Do you think we need double instead? I can change if you think so.
Quoted 55 lines
Quoted 14 lines
examples/ipsec-secgw/ipsec-secgw.c +13
-0+printf ("\nStatistics for core %u ------------------------------" +"\nPackets received: %20" PRIu64 +"\nPackets sent: %24" PRIu64 +"\nPackets dropped: %21" PRIu64 +"\nBurst percent: %23.2f" , + coreid, + core_statistics[coreid].rx, + core_statistics[coreid].tx, + core_statistics[coreid].dropped, + burst_percent); + + total_packets_dropped += core_statistics[coreid].dropped; + total_packets_tx += core_statistics[coreid].tx;examples/ipsec-secgw/ipsec-secgw.c adjust_ipv6_pktlen(struct rte_mbuf *m, const struct rte_ipv6_hdr *iph, · line 345
Quoted 10 lines
examples/ipsec-secgw/ipsec-secgw.c +10
-0+ total_packets_rx += core_statistics[coreid].rx; + } +printf ("\nAggregate statistics ===============================" +"\nTotal packets received: %14" PRIu64 +"\nTotal packets sent: %18" PRIu64 +"\nTotal packets dropped: %15" PRIu64, + total_packets_rx, + total_packets_tx, + total_packets_dropped); +printf("\n====================================================\n");
examples/ipsec-secgw/ipsec-secgw.c prepare_one_packet(struct rte_mbuf *pkt, struct · line 390
Quoted 9 lines
examples/ipsec-secgw/ipsec-secgw.c +3
-0+} +#endif /* ENABLE_STATS */ +static inline void prepare_one_packet (struct rte_mbuf *pkt,struct ipsec_traffic *t) {@@ -333,6 +388,7 @@ prepare_one_packet(struct rte_mbuf *pkt, struct ipsec_traffic *t)/* drop packet when IPv6 header exceeds first segment length */
examples/ipsec-secgw/ipsec-secgw.c prepare_one_packet(struct rte_mbuf *pkt, struct
Quoted 6 lines
examples/ipsec-secgw/ipsec-secgw.c +1
-0if (unlikely (l3len > pkt->data_len)) { +core_stats_update_drop (1 );rte_pktmbuf_free (pkt);return ;examples/ipsec-secgw/ipsec-secgw.c +0
-0}@@ -350,6 +406,7 @@ prepare_one_packet(struct rte_mbuf *pkt, struct
ipsec_traffic*t)examples/ipsec-secgw/ipsec-secgw.c send_burst(struct lcore_conf *qconf, uint16_t n,
Quoted 8 lines
examples/ipsec-secgw/ipsec-secgw.c +1
-0/* Unknown/Unsupported type, drop the packet */ RTE_LOG (ERR ,IPSEC ,"Unsupported packet type 0x%x\n" ,rte_be_to_cpu_16 (eth->ether_type)); +core_stats_update_drop (1 );rte_pktmbuf_free (pkt);return ; }@@ -471,6 +528,11 @@ send_burst(struct lcore_conf *qconf, uint16_t n,
uint16_tport)Quoted 3 lines
examples/ipsec-secgw/ipsec-secgw.c +0
-0int32_t ret;uint16_t queueid;
Quoted 8 lines
examples/ipsec-secgw/ipsec-secgw.c send_burst(struct lcore_conf *qconf, uint16_t n, uint16_t port) · line 534
Quoted 4 lines
examples/ipsec-secgw/ipsec-secgw.c +4
-0+#ifdef ENABLE_STATS +int lcore_id =rte_lcore_id (); + core_statistics[lcore_id].tx += n; +#endif /* ENABLE_STATS */ Instead of polluting genric code with ifdefs, why not to introduce 2 new
functions:core_stats_update_rx(),core_stats_update_tx(), as you did forcore_stats_drop()?
[Anoob] The drop would happen from multiple places while Rx & Tx would happen only from one place each. I wasn't sure which would be the right approach. If you agree to introduce rx, tx & drop stats functions, I will update so.
Quoted 8 lines
Quoted 8 lines
examples/ipsec-secgw/ipsec-secgw.c +1
-0+ queueid = qconf->tx_queue_id[port]; m_table = (struct rte_mbuf **)qconf->tx_mbufs[port].m_table;@@ -478,6 +540,9 @@ send_burst(struct lcore_conf *qconf, uint16_t n, uint16_t port)
Quoted 8 lines
examples/ipsec-secgw/ipsec-secgw.c send_burst(struct lcore_conf *qconf, uint16_t n, uint16_t port)
Quoted 4 lines
examples/ipsec-secgw/ipsec-secgw.c +2
-0ret =rte_eth_tx_burst (port, queueid, m_table, n);if (unlikely (ret < n)) { +#ifdef ENABLE_STATS + core_statistics[lcore_id].dropped += n-ret;#endif /*
ENABLE_STATSexamples/ipsec-secgw/ipsec-secgw.c send_burst(struct lcore_conf *qconf, uint16_t n, · line 546
Quoted 1 line
examples/ipsec-secgw/ipsec-secgw.c +1
-0+*/ You have
core_stats_update_drop()for that - use it.
[Anoob] lcore_id is already available as local variable. Will add the call instead.
Quoted 15 lines
examples/ipsec-secgw/ipsec-secgw.c send_fragment_packet(struct lcore_conf *qconf,
Quoted 5 lines
examples/ipsec-secgw/ipsec-secgw.c +0
-0do { rte_pktmbuf_free(m_table[ret]);examples/ipsec-secgw/ipsec-secgw.c +0
-0}while (++ret < n);@@ -525,6 +590,7 @@ send_fragment_packet(struct lcore_conf *qconf,struct
rte_mbuf*m,examples/ipsec-secgw/ipsec-secgw.c send_single_packet(struct rte_mbuf *m, uint16_t
Quoted 8 lines
examples/ipsec-secgw/ipsec-secgw.c +1
-0"error code: %d\n" , __func__, m->pkt_len, rte_errno); +core_stats_update_drop (1 );rte_pktmbuf_free (m);return len; }@@ -549,8 +615,10 @@ send_single_packet(struct rte_mbuf *m, uint16_tport,
uint8_tproto)
Quoted 12 lines
examples/ipsec-secgw/ipsec-secgw.c send_single_packet(struct rte_mbuf *m, uint16_t port, uint8_t proto) · line 620
Quoted 7 lines
examples/ipsec-secgw/ipsec-secgw.c +2
-1/* need to fragment the packet */ }else if (frag_tbl_sz >0 ) len =send_fragment_packet (qconf, m, port, proto); -else +else { +core_stats_update_drop (1 );rte_pktmbuf_free (m);It looks like a lot of such places...
Would it be worth to unitecore_stats_update_drop()andrte_pktmbuf_free(m)Into some inline function:ipsec_secgw_packet_drop(struct rte_mbuf *m[], uint32_t n)and use it all over such places.
[Anoob] Ack
Quoted 106 lines
examples/ipsec-secgw/ipsec-secgw.c split46_traffic(struct ipsec_traffic *trf, struct
Quoted 28 lines
examples/ipsec-secgw/ipsec-secgw.c +1
-0+ }/* enough pkts to be sent */ if (unlikely(len == MAX_PKT_BURST)) { @@ -584,18 +652,21 @@inbound_sp_sa(struct
sp_ctx*sp, structsa_ctx*sa, structtraffic_type*ip,examples/ipsec-secgw/ipsec-secgw.c +3
-0continue ; }if (res ==DISCARD ) { +core_stats_update_drop (1 );rte_pktmbuf_free (m);continue ; }/* Only check SPI match for processed IPSec packets */ if (i < lim && ((m->ol_flags &PKT_RX_SEC_OFFLOAD ) ==0 )) { +core_stats_update_drop (1 );rte_pktmbuf_free (m);continue ; } sa_idx = res -1 ;if (!inbound_sa_check (sa, m, sa_idx)) { +core_stats_update_drop (1 );rte_pktmbuf_free (m);continue ; }@@ -630,8 +701,10 @@ split46_traffic(struct ipsec_traffic *trf, struct
rte_mbuf*mb[],uint32_tnum)examples/ipsec-secgw/ipsec-secgw.c outbound_sp(struct sp_ctx *sp, struct traffic_type
Quoted 12 lines
examples/ipsec-secgw/ipsec-secgw.c +3
-1uint8_t *, offsetof(struct ip6_hdr, ip6_nxt)); n6++; - }else + }else { +core_stats_update_drop (1 );rte_pktmbuf_free (m); + } } trf->ip4.num = n4;@@ -682,11 +755,12 @@ outbound_sp(struct sp_ctx *sp, struct traffic_type*ip,
examples/ipsec-secgw/ipsec-secgw.c outbound_sp(struct sp_ctx *sp, struct traffic_type *ip, · line 761
Quoted 9 lines
examples/ipsec-secgw/ipsec-secgw.c +3
-2for (i =0 ; i < ip->num; i++) { m = ip->pkts[i]; sa_idx = ip->res[i] -1 ; -if (ip->res[i] ==DISCARD ) +if (ip->res[i] ==DISCARD ) { +core_stats_update_drop (1 );rte_pktmbuf_free (m); -else if (ip->res[i] ==BYPASS ) + }else if (ip->res[i] ==BYPASS ) {Looks unnecessary.
examples/ipsec-secgw/ipsec-secgw.c process_pkts_outbound_nosp(struct ipsec_ctx
Quoted 31 lines
examples/ipsec-secgw/ipsec-secgw.c +5
-2ip->pkts[j++] = m; - else { + }else { ipsec->res[ipsec->num] = sa_idx; ipsec->pkts[ipsec->num++] = m; }@@ -705,6 +779,8 @@ process_pkts_outbound(struct ipsec_ctx *ipsec_ctx,for (i =0 ; i < traffic->ipsec.num; i++)rte_pktmbuf_free (traffic->ipsec.pkts[i]); +core_stats_update_drop (traffic->ipsec.num); + traffic->ipsec.num =0 ; outbound_sp(ipsec_ctx->sp4_ctx, &traffic->ip4, &traffic->ipsec); @@ -745,12 +821,14 @@ process_pkts_inbound_nosp(struct ipsec_ctx *ipsec_ctx,/* Drop any IPv4 traffic from unprotected ports */ for (i =0 ; i < traffic->ip4.num; i++)rte_pktmbuf_free (traffic->ip4.pkts[i]); +core_stats_update_drop (traffic->ip4.num); traffic->ip4.num =0 ;/* Drop any IPv6 traffic from unprotected ports */ for (i =0 ; i < traffic->ip6.num; i++)rte_pktmbuf_free (traffic->ip6.pkts[i]); +core_stats_update_drop (traffic->ip6.num); traffic->ip6.num =0 ;@@ -788,6 +866,7 @@ process_pkts_outbound_nosp(struct ipsec_ctx
*ipsec_ctx,examples/ipsec-secgw/ipsec-secgw.c route4_pkts(struct rt_ctx *rt_ctx, struct rte_mbuf
Quoted 8 lines
examples/ipsec-secgw/ipsec-secgw.c +1
-0/* Drop any IPsec traffic from protected ports */ for (i =0 ; i < traffic->ipsec.num; i++)rte_pktmbuf_free (traffic->ipsec.pkts[i]); +core_stats_update_drop (traffic->ipsec.num); n =0 ;@@ -901,6 +980,7 @@ route4_pkts(struct rt_ctx *rt_ctx, struct rte_mbuf*pkts[],
uint8_tnb_pkts)examples/ipsec-secgw/ipsec-secgw.c route6_pkts(struct rt_ctx *rt_ctx, struct rte_mbuf
Quoted 8 lines
examples/ipsec-secgw/ipsec-secgw.c +1
-0}if ((pkt_hop &RTE_LPM_LOOKUP_SUCCESS ) ==0 ) { +core_stats_update_drop (1 );rte_pktmbuf_free (pkts[i]);continue ; }@@ -953,6 +1033,7 @@ route6_pkts(struct rt_ctx *rt_ctx, struct rte_mbuf*pkts[],
uint8_tnb_pkts)Quoted 2 lines
examples/ipsec-secgw/ipsec-secgw.c +0
-0}
Quoted 14 lines
examples/ipsec-secgw/ipsec-secgw.c ipsec_poll_mode_worker(void) · line 1185
Quoted 12 lines
examples/ipsec-secgw/ipsec-secgw.c +4
-0if (pkt_hop ==-1 ) { +core_stats_update_drop (1 );rte_pktmbuf_free (pkts[i]);continue ; }@@ -1099,6 +1180,9 @@ ipsec_poll_mode_worker(void)const uint64_t drain_tsc = (rte_get_tsc_hz () +US_PER_S -1 ) /US_PER_S *BURST_TX_DRAIN_US ;struct lcore_rx_queue *rxql; +#ifdef ENABLE_STATS +uint64_t timer_tsc =0 ; +#endif /* ENABLE_STATS */ Probably better just
RTE_SET_USED(timer_tsc);
[Anoob] Isn't it better to keep all the code related an optional feature under on flag? Either way is fine.
Quoted 23 lines
Quoted 11 lines
examples/ipsec-secgw/ipsec-secgw.c +3
-0prev_tsc =0 ; lcore_id =rte_lcore_id ();@@ -1159,6 +1243,19 @@ ipsec_poll_mode_worker(void)drain_tx_buffers (qconf);drain_crypto_buffers (qconf); prev_tsc = cur_tsc; +#ifdef ENABLE_STATS +if (lcore_id ==rte_get_master_lcore ()) { +/* advance the timer */ examples/ipsec-secgw/ipsec-secgw.c ipsec_poll_mode_worker(void) · line 1258
Quoted 10 lines
examples/ipsec-secgw/ipsec-secgw.c +10
-0+ timer_tsc += diff_tsc; + +/* if timer has reached its timeout */ +if (unlikely (timer_tsc >= timer_period)) { +print_stats (); +/* reset the timer */ + timer_tsc =0 ; + } + } +#endif /* ENABLE_STATS */
Quoted 7 lines
Why to do stats collection/display inside data-path?
Why not use rte_timer/rte_alarm and make it happen in control thread? Another option - make stats printing at some signal to the process. In that case we don't need to bother with time period at all - it will be user to decide.
Again if we remove thatprint_stats()from data-path it might become cheap enough to always collect it, and we will not needENABLE_STATSmacro at all.
[Anoob] The stats collection will not be cheap because we are updating a common structure from multiple cores. And the idea is not to have a very efficient stats mechanism. When we have more cores engaged, IPsec performance is directly dependent on ability of RSS to split traffic to multiple cores. Such stats would come handy in understanding per core distribution of incoming traffic.
Quoted 14 lines
examples/ipsec-secgw/ipsec-secgw.c ipsec_poll_mode_worker(void) · line 1261
Quoted 4 lines
examples/ipsec-secgw/ipsec-secgw.c +0
-0} for (i = 0; i < qconf->nb_rx_queue; ++i) { @@ -1169,6 +1266,12@@
examples/ipsec-secgw/ipsec-secgw.c ipsec_poll_mode_worker(void) · line 1272
Quoted 8 lines
ipsec_poll_mode_worker(void)examples/ipsec-secgw/ipsec-secgw.c +4
-0nb_rx =rte_eth_rx_burst (portid, queueid, pkts,MAX_PKT_BURST ); +#ifdef ENABLE_STATS + core_statistics[lcore_id].rx += nb_rx; +if (nb_rx ==MAX_PKT_BURST ) + core_statistics[lcore_id].burst_rx += nb_rx;#endif /*
Quoted 5 lines
examples/ipsec-secgw/ipsec-secgw.c
Quoted 2 lines
examples/ipsec-secgw/ipsec-secgw.c +2
-0+ENABLE_STATS */ +Same for above for TX: no need to pollute the code with ifdefs. Better to introduce new function:
stats_update_rx()or so.
[Anoob] Ack
Quoted 139 lines
Quoted 23 lines
examples/ipsec-secgw/ipsec-secgw.c +5
-0if (nb_rx >0 )process_pkts (qconf, pkts, nb_rx, portid);@@ -2747,6 +2850,11 @@ main(int32_t argc, char **argv)signal (SIGINT , signal_handler);signal (SIGTERM , signal_handler); +#ifdef ENABLE_STATS +/* convert to number of cycles */ + timer_period *=rte_get_timer_hz (); +#endif /* ENABLE_STATS */ +/* initialize event helper configuration */ eh_conf =eh_conf_init ();if (eh_conf ==NULL )examples/ipsec-secgw/ipsec-secgw.h +0
-0examples/ipsec-secgw/ipsec-secgw.h line 10
Quoted 5 lines
examples/ipsec-secgw/ipsec-secgw.h +2
-0@@ -6,6 +6,8 @@#include <stdbool.h> +//#define ENABLE_STATS +Should be removed I think.
examples/ipsec-secgw/ipsec.c ipsec_enqueue(ipsec_xform_fn xform_func, struct
Quoted 38 lines
examples/ipsec-secgw/ipsec-secgw.h +0
-0#define NB_SOCKETS 4#define MAX_PKT_BURST 32(quoted diff) +0
-0
b/examples/ipsec-secgw/ipsec.cindex bf88d80..dcb9312 100644examples/ipsec-secgw/ipsec.c +4
-1@@ -499,8 +499,10 @@ enqueue_cop_burst(struct cdev_qp *cqp)" enqueued %u crypto ops out of %u\n" , cqp->id, cqp->qp, ret, len);/* drop packets that we fail to enqueue */ -for (i = ret; i < len; i++) +for (i = ret; i < len; i++) { +core_stats_update_drop (1 );rte_pktmbuf_free (cqp->buf[i]->sym->m_src); + } } cqp->in_flight += ret; cqp->len =0 ;@@ -528,6 +530,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx, for (i = 0; i < nb_pkts; i++) {if (unlikely (sas[i] ==NULL )) { +core_stats_update_drop (1 );rte_pktmbuf_free (pkts[i]);continue ;examples/ipsec-secgw/ipsec.c +1
-0}@@ -549,6 +552,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx, if ((unlikely(ips->security.ses == NULL)) && create_lookaside_session(ipsec_ctx, sa, ips)) { +core_stats_update_drop (1 );rte_pktmbuf_free (pkts[i]);continue ;examples/ipsec-secgw/ipsec.c +0
-0}@@ -563,6 +567,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct
ipsec_ctx*ipsec_ctx,examples/ipsec-secgw/ipsec.c ipsec_enqueue(ipsec_xform_fn xform_func, struct · line 568
Quoted 2 lines
examples/ipsec-secgw/ipsec.c +0
-0case RTE_SECURITY_ACTION_TYPE_CPU_CRYPTO : RTE_LOG(ERR, IPSEC, "CPU crypto is not supported bythe"
examples/ipsec-secgw/ipsec.c ipsec_inline_dequeue(ipsec_xform_fn xform_func,
Quoted 33 lines
examples/ipsec-secgw/ipsec.c +2
-0" legacy mode."); +core_stats_update_drop (1 );rte_pktmbuf_free (pkts[i]);continue ;@@ -575,6 +580,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx, if ((unlikely(ips->crypto.ses == NULL)) && create_lookaside_session(ipsec_ctx, sa, ips)) { +core_stats_update_drop (1 );rte_pktmbuf_free (pkts[i]);continue ;examples/ipsec-secgw/ipsec.c +1
-0}@@ -584,6 +590,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, structipsec_ctx *ipsec_ctx, ret =xform_func (pkts[i], sa, &priv->cop);if (unlikely (ret)) { +core_stats_update_drop (1 );rte_pktmbuf_free (pkts[i]);continue ;examples/ipsec-secgw/ipsec.c +1
-0}@@ -608,6 +615,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, structipsec_ctx *ipsec_ctx, ret =xform_func (pkts[i], sa, &priv->cop);if (unlikely (ret)) { +core_stats_update_drop (1 );rte_pktmbuf_free (pkts[i]);continue ;examples/ipsec-secgw/ipsec.c +0
-0}@@ -643,6 +651,7 @@ ipsec_inline_dequeue(ipsec_xform_fn xform_func,struct
ipsec_ctx*ipsec_ctx,examples/ipsec-secgw/ipsec.c ipsec_dequeue(ipsec_xform_fn xform_func, struct
Quoted 8 lines
examples/ipsec-secgw/ipsec.c +1
-0sa = priv->sa; ret =xform_func (pkt, sa, &priv->cop);if (unlikely (ret)) { +core_stats_update_drop (1 );rte_pktmbuf_free (pkt);continue ; }@@ -690,12 +699,14 @@ ipsec_dequeue(ipsec_xform_fn xform_func, struct
ipsec_ctx*ipsec_ctx,examples/ipsec-secgw/ipsec.c ipsec_dequeue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
Quoted 9 lines
examples/ipsec-secgw/ipsec.c +1
-0RTE_SECURITY_ACTION_TYPE_NONE) { ret =xform_func (pkt, sa, cops[j]);if (unlikely (ret)) { +core_stats_update_drop (1 );rte_pktmbuf_free (pkt);continue ; } } else if (ipsec_get_action_type(sa) ==RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) {Quoted 13 lines
examples/ipsec-secgw/ipsec.c +1
-0if (cops[j]->status) { +core_stats_update_drop (1 );rte_pktmbuf_free (pkt);continue ; }(quoted diff) +0
-0
b/examples/ipsec-secgw/ipsec.hindex 1e642d1..8519eab 100644examples/ipsec-secgw/ipsec.h +0
-0@@ -46,6 +46,17 @@#define IP6_VERSION (6)
Quoted 12 lines
examples/ipsec-secgw/ipsec.h line 54
Quoted 6 lines
examples/ipsec-secgw/ipsec.h +6
-0+#ifdef ENABLE_STATS +struct ipsec_core_statistics { + uint64_t tx; + uint64_t rx; + uint64_t dropped; + uint64_t burst_rx;A bit strange to have
burst_rxand no similar counterpart for tx. BTW, do you needburst_rx?
Might be better:nb_calls_rx,nb_calls_tx;
and then: rx/nb_calls_rx will give you average burst size.
[Anoob] After the packets are received in the app, further IPsec processing is done, which is more efficient when done in batches. For example, SA lookup is better when done in a batch, as SIMD instructions would be used. Ability of traffic to be in burst or ability of ethdev to give burst will have an effect on the IPsec performance per core. Tx burst percentage isn't that important, since it is the app which submits for Tx.
Rx/nb_calls_rx would also give a similar kind of stat, but the difference is whether 1 packet cases and 31 packet cases are treated equal. I would prefer to keep it the current way, but what you suggested also would work.
Quoted 6 lines
examples/ipsec-secgw/ipsec.h line 57
Quoted 4 lines
examples/ipsec-secgw/ipsec.h +3
-0+} __rte_cache_aligned; + +struct ipsec_core_statistics core_statistics[RTE_MAX_ETHPORTS ];Should be
RTE_MAX_LCORES, I think.
[Anoob] Ack
Quoted 40 lines
Quoted 26 lines
examples/ipsec-secgw/ipsec.h +12
-0+#endif /* ENABLE_STATS */ +struct rte_crypto_xform ;struct ipsec_xform ;struct rte_mbuf ;@@ -416,4 +427,15 @@ check_flow_params(uint16_t fdir_portid, uint8_t fdir_qid); int create_ipsec_esp_flow(struct ipsec_sa *sa); +static inline void +core_stats_update_drop (int n) +{ +#ifdef ENABLE_STATS +int lcore_id =rte_lcore_id (); + core_statistics[lcore_id].dropped += n;#else +RTE_SET_USED (n); +#endif /* ENABLE_STATS */ +} +#endif /* __IPSEC_H__ */ examples/ipsec-secgw/ipsec_process.c +0
-0examples/ipsec-secgw/ipsec_process.c free_pkts(struct rte_mbuf *mb[], uint32_t n) { · line 29
Quoted 6 lines
examples/ipsec-secgw/ipsec_process.c +4
-0@@ -24,6 +24,11 @@ free_pkts(struct rte_mbuf *mb[], uint32_t n) {uint32_t i; +#ifdef ENABLE_STATS +int lcore_id =rte_lcore_id (); + core_statistics[lcore_id].dropped += n;#endif /* ENABLE_STATS */ +Same as above - why not use
stats_update_drop()here?Quoted 5 lines
examples/ipsec-secgw/ipsec_process.c +0
-1for (i =0 ; i != n; i++)rte_pktmbuf_free (mb[i]); } --2.7.4
Quoted 28 lines
Quoted 20 lines
Quoted 8 lines
examples/ipsec-secgw/ipsec-secgw.c +3
-0@@ -1159,6 +1243,19 @@ ipsec_poll_mode_worker(void)drain_tx_buffers (qconf);drain_crypto_buffers (qconf); prev_tsc = cur_tsc; +#ifdef ENABLE_STATS +if (lcore_id ==rte_get_master_lcore ()) { +/* advance the timer */ examples/ipsec-secgw/ipsec-secgw.c ipsec_poll_mode_worker(void) · line 1258
Quoted 10 lines
examples/ipsec-secgw/ipsec-secgw.c +10
-0+ timer_tsc += diff_tsc; + +/* if timer has reached its timeout */ +if (unlikely (timer_tsc >= timer_period)) { +print_stats (); +/* reset the timer */ + timer_tsc =0 ; + } + } +#endif /* ENABLE_STATS */ Quoted 7 lines
Why to do stats collection/display inside data-path?
Why not use rte_timer/rte_alarm and make it happen in control thread? Another option - make stats printing at some signal to the process. In that case we don't need to bother with time period at all - it will be user to decide.
Again if we remove thatprint_stats()from data-path it might become cheap enough to always collect it, and we will not needENABLE_STATSmacro at all.
Quoted 1 line
[Anoob] The stats collection will not be cheap because we are updating a common structure from multiple cores.
What I am saying - per-lcore stats collection (updating lcore stats by lcore itself) - should be cheap enough and can stay in data-path (probably even always, not only when particular compile flag is enabled). Global stats collection and display is quite expensive operation,
so we shouldn't keep it in data-path.
We have a control thread within dpdk (for alarms, signals, timers, etc.),
why not to call print_stats from there?
Quoted 5 lines
And the idea is not to
have a very efficient stats mechanism. When we have more cores engaged, IPsec performance is directly dependent on ability of RSS to split traffic to multiple cores. Such stats would come handy in understanding per core distribution of incoming traffic.Quoted 1 line
PATCH v3 1/1 examples/ipsec-secgw: add per core packet stats
In reply to Anoob JosephAdding per core packet handling stats to analyze traffic distribution when multiple cores are engaged.
Since aggregating the packet stats across cores would affect performance, keeping the feature disabled using compile time flags.
---
v3:
* Added wrapper functions for updating rx, tx & dropped counts
* Updated free_pkts() to have stats updated internally
* Introduced similar free_pkt() function which updates stats and frees one packet
* Moved all inline functions and macros to ipsec-secgw.h
* Made STATS_INTERVAL macro to control the interval of the stats update. STATS_INTERVAL = 0 would disable the feature.
v2:
* Added lookup failure cases to drop count
examples/ipsec-secgw/ipsec-secgw.c | 113 ++++++++++++++++++++++++++++------- examples/ipsec-secgw/ipsec-secgw.h | 68 +++++++++++++++++++++ examples/ipsec-secgw/ipsec.c | 20 +++---- examples/ipsec-secgw/ipsec_process.c | 11 +--- 4 files changed, 171 insertions(+), 41 deletions(-)
examples/ipsec-secgw/ipsec-secgw.c +92 -21
@@ -288,6 +288,59 @@ adjust_ipv6_pktlen(struct rte_mbuf *m, const struct rte_ipv6_hdr *iph, } } +#if (STATS_INTERVAL >0 ) + +/* Print out statistics on packet distribution */ +static void +print_stats (void ) +{ +uint64_t total_packets_dropped, total_packets_tx, total_packets_rx; +unsigned int coreid; +float burst_percent; + + total_packets_dropped =0 ; + total_packets_tx =0 ; + total_packets_rx =0 ; + +const char clr[] = {27 ,'[' ,'2' ,'J' ,'\0' }; + +/* Clear screen and move to top left */ +printf ("%s" , clr); + +printf ("\nCore statistics ====================================" ); + +for (coreid =0 ; coreid <RTE_MAX_LCORE ; coreid++) { +/* skip disabled cores */ +if (rte_lcore_is_enabled (coreid) ==0 ) +continue ; + burst_percent = (float )(core_statistics[coreid].burst_rx *100 )/ + core_statistics[coreid].rx; +printf ("\nStatistics for core %u ------------------------------" +"\nPackets received: %20" PRIu64 +"\nPackets sent: %24" PRIu64 +"\nPackets dropped: %21" PRIu64 +"\nBurst percent: %23.2f" , + coreid, + core_statistics[coreid].rx, + core_statistics[coreid].tx, + core_statistics[coreid].dropped, + burst_percent); + + total_packets_dropped += core_statistics[coreid].dropped; + total_packets_tx += core_statistics[coreid].tx; + total_packets_rx += core_statistics[coreid].rx; + } +printf ("\nAggregate statistics ===============================" +"\nTotal packets received: %14" PRIu64 +"\nTotal packets sent: %18" PRIu64 +"\nTotal packets dropped: %15" PRIu64, + total_packets_rx, + total_packets_tx, + total_packets_dropped); +printf ("\n====================================================\n" ); +} +#endif /* STATS_INTERVAL */ +static inline void prepare_one_packet (struct rte_mbuf *pkt,struct ipsec_traffic *t) {
@@ -333,7 +386,7 @@ prepare_one_packet(struct rte_mbuf *pkt, struct ipsec_traffic *t)/* drop packet when IPv6 header exceeds first segment length */ if (unlikely (l3len > pkt->data_len)) { -rte_pktmbuf_free (pkt); +free_pkt (pkt);return ; }
@@ -350,7 +403,7 @@ prepare_one_packet(struct rte_mbuf *pkt, struct ipsec_traffic *t)/* Unknown/Unsupported type, drop the packet */ RTE_LOG (ERR ,IPSEC ,"Unsupported packet type 0x%x\n" ,rte_be_to_cpu_16 (eth->ether_type)); -rte_pktmbuf_free (pkt); +free_pkt (pkt);return ; }
@@ -477,9 +530,12 @@ send_burst(struct lcore_conf *qconf, uint16_t n, uint16_t port)prepare_tx_burst (m_table, n, port, qconf); ret =rte_eth_tx_burst (port, queueid, m_table, n); + +core_stats_update_tx (ret); +if (unlikely (ret < n)) {do { -rte_pktmbuf_free (m_table[ret]); +free_pkt (m_table[ret]); }while (++ret < n); }
@@ -525,7 +581,7 @@ send_fragment_packet(struct lcore_conf *qconf, struct rte_mbuf *m,"error code: %d\n" , __func__, m->pkt_len, rte_errno); -rte_pktmbuf_free (m); +free_pkt (m);return len; }
@@ -550,7 +606,7 @@ send_single_packet(struct rte_mbuf *m, uint16_t port, uint8_t proto) }else if (frag_tbl_sz >0 ) len =send_fragment_packet (qconf, m, port, proto);else -rte_pktmbuf_free (m); +free_pkt (m);/* enough pkts to be sent */ if (unlikely (len ==MAX_PKT_BURST )) {
@@ -584,19 +640,19 @@ inbound_sp_sa(struct sp_ctx *sp, struct sa_ctx *sa, struct traffic_type *ip,continue ; }if (res ==DISCARD ) { -rte_pktmbuf_free (m); +free_pkt (m);continue ; }/* Only check SPI match for processed IPSec packets */ if (i < lim && ((m->ol_flags &PKT_RX_SEC_OFFLOAD ) ==0 )) { -rte_pktmbuf_free (m); +free_pkt (m);continue ; } sa_idx = res -1 ;if (!inbound_sa_check (sa, m, sa_idx)) { -rte_pktmbuf_free (m); +free_pkt (m);continue ; } ip->pkts[j++] = m;
@@ -631,7 +687,7 @@ split46_traffic(struct ipsec_traffic *trf, struct rte_mbuf *mb[], uint32_t num) offsetof(struct ip6_hdr, ip6_nxt)); n6++; }else -rte_pktmbuf_free (m); +free_pkt (m); } trf->ip4.num = n4;
@@ -683,7 +739,7 @@ outbound_sp(struct sp_ctx *sp, struct traffic_type *ip, m = ip->pkts[i]; sa_idx = ip->res[i] -1 ;if (ip->res[i] ==DISCARD ) -rte_pktmbuf_free (m); +free_pkt (m);else if (ip->res[i] ==BYPASS ) ip->pkts[j++] = m;else {
@@ -702,8 +758,7 @@ process_pkts_outbound(struct ipsec_ctx *ipsec_ctx,uint16_t idx, nb_pkts_out, i;/* Drop any IPsec traffic from protected ports */ -for (i =0 ; i < traffic->ipsec.num; i++) -rte_pktmbuf_free (traffic->ipsec.pkts[i]); +free_pkts (traffic->ipsec.pkts, traffic->ipsec.num); traffic->ipsec.num =0 ;
@@ -743,14 +798,12 @@ process_pkts_inbound_nosp(struct ipsec_ctx *ipsec_ctx,uint32_t nb_pkts_in, i, idx;/* Drop any IPv4 traffic from unprotected ports */ -for (i =0 ; i < traffic->ip4.num; i++) -rte_pktmbuf_free (traffic->ip4.pkts[i]); +free_pkts (traffic->ip4.pkts, traffic->ip4.num); traffic->ip4.num =0 ;/* Drop any IPv6 traffic from unprotected ports */ -for (i =0 ; i < traffic->ip6.num; i++) -rte_pktmbuf_free (traffic->ip6.pkts[i]); +free_pkts (traffic->ip6.pkts, traffic->ip6.num); traffic->ip6.num =0 ;
@@ -786,8 +839,7 @@ process_pkts_outbound_nosp(struct ipsec_ctx *ipsec_ctx,struct ip *ip;/* Drop any IPsec traffic from protected ports */ -for (i =0 ; i < traffic->ipsec.num; i++) -rte_pktmbuf_free (traffic->ipsec.pkts[i]); +free_pkts (traffic->ipsec.pkts, traffic->ipsec.num); n =0 ;
@@ -901,7 +953,7 @@ route4_pkts(struct rt_ctx *rt_ctx, struct rte_mbuf *pkts[], uint8_t nb_pkts) }if ((pkt_hop &RTE_LPM_LOOKUP_SUCCESS ) ==0 ) { -rte_pktmbuf_free (pkts[i]); +free_pkt (pkts[i]);continue ; }send_single_packet (pkts[i], pkt_hop &0xff ,IPPROTO_IP );
@@ -953,7 +1005,7 @@ route6_pkts(struct rt_ctx *rt_ctx, struct rte_mbuf *pkts[], uint8_t nb_pkts) }if (pkt_hop ==-1 ) { -rte_pktmbuf_free (pkts[i]); +free_pkt (pkts[i]);continue ; }send_single_packet (pkts[i], pkt_hop &0xff ,IPPROTO_IPV6 );
@@ -1099,6 +1151,10 @@ ipsec_poll_mode_worker(void)const uint64_t drain_tsc = (rte_get_tsc_hz () +US_PER_S -1 ) /US_PER_S *BURST_TX_DRAIN_US ;struct lcore_rx_queue *rxql; +#if (STATS_INTERVAL >0 ) +const uint64_t timer_period =STATS_INTERVAL *rte_get_timer_hz (); +uint64_t timer_tsc =0 ; +#endif /* STATS_INTERVAL */ prev_tsc =0 ; lcore_id =rte_lcore_id ();
@@ -1159,6 +1215,19 @@ ipsec_poll_mode_worker(void)drain_tx_buffers (qconf);drain_crypto_buffers (qconf); prev_tsc = cur_tsc; +#if (STATS_INTERVAL >0 ) +if (lcore_id ==rte_get_master_lcore ()) { +/* advance the timer */ + timer_tsc += diff_tsc; + +/* if timer has reached its timeout */ +if (unlikely (timer_tsc >= timer_period)) { +print_stats (); +/* reset the timer */ + timer_tsc =0 ; + } + } +#endif /* STATS_INTERVAL */ }for (i =0 ; i < qconf->nb_rx_queue; ++i) {
@@ -1169,8 +1238,10 @@ ipsec_poll_mode_worker(void) nb_rx =rte_eth_rx_burst (portid, queueid, pkts,MAX_PKT_BURST ); -if (nb_rx >0 ) +if (nb_rx >0 ) { +core_stats_update_rx (nb_rx);process_pkts (qconf, pkts, nb_rx, portid); + }/* dequeue and process completed crypto-ops */ if (is_unprotected_port (portid))
examples/ipsec-secgw/ipsec-secgw.h +68 -0
84 lines not on this page · raw message
examples/ipsec-secgw/ipsec.c +10 -10
@@ -500,7 +500,7 @@ enqueue_cop_burst(struct cdev_qp *cqp) cqp->id, cqp->qp, ret, len);/* drop packets that we fail to enqueue */ for (i = ret; i < len; i++) -rte_pktmbuf_free (cqp->buf[i]->sym->m_src); +free_pkt (cqp->buf[i]->sym->m_src); } cqp->in_flight += ret; cqp->len =0 ;
@@ -528,7 +528,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,for (i =0 ; i < nb_pkts; i++) {if (unlikely (sas[i] ==NULL )) { -rte_pktmbuf_free (pkts[i]); +free_pkt (pkts[i]);continue ; }
@@ -549,7 +549,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,if ((unlikely (ips->security.ses ==NULL )) &&create_lookaside_session (ipsec_ctx, sa, ips)) { -rte_pktmbuf_free (pkts[i]); +free_pkt (pkts[i]);continue ; }
@@ -563,7 +563,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,case RTE_SECURITY_ACTION_TYPE_CPU_CRYPTO :RTE_LOG (ERR ,IPSEC ,"CPU crypto is not supported by the" " legacy mode." ); -rte_pktmbuf_free (pkts[i]); +free_pkt (pkts[i]);continue ;case RTE_SECURITY_ACTION_TYPE_NONE :
@@ -575,7 +575,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,if ((unlikely (ips->crypto.ses ==NULL )) &&create_lookaside_session (ipsec_ctx, sa, ips)) { -rte_pktmbuf_free (pkts[i]); +free_pkt (pkts[i]);continue ; }
@@ -584,7 +584,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx, ret =xform_func (pkts[i], sa, &priv->cop);if (unlikely (ret)) { -rte_pktmbuf_free (pkts[i]); +free_pkt (pkts[i]);continue ; }break ;
@@ -608,7 +608,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx, ret =xform_func (pkts[i], sa, &priv->cop);if (unlikely (ret)) { -rte_pktmbuf_free (pkts[i]); +free_pkt (pkts[i]);continue ; }
@@ -643,7 +643,7 @@ ipsec_inline_dequeue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx, sa = priv->sa; ret =xform_func (pkt, sa, &priv->cop);if (unlikely (ret)) { -rte_pktmbuf_free (pkt); +free_pkt (pkt);continue ; } pkts[nb_pkts++] = pkt;
@@ -690,13 +690,13 @@ ipsec_dequeue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx, RTE_SECURITY_ACTION_TYPE_NONE) { ret =xform_func (pkt, sa, cops[j]);if (unlikely (ret)) { -rte_pktmbuf_free (pkt); +free_pkt (pkt);continue ; } } else if (ipsec_get_action_type(sa) ==RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL ) {if (cops[j]->status) { -rte_pktmbuf_free (pkt); +free_pkt (pkt);continue ; } }
examples/ipsec-secgw/ipsec_process.c +1 -10
@@ -12,22 +12,13 @@#include <rte_mbuf.h> #include "ipsec.h" +#include "ipsec-secgw.h" #define SATP_OUT_IPV4 (t) \ ((((t) & RTE_IPSEC_SATP_MODE_MASK) == RTE_IPSEC_SATP_MODE_TRANS && \ (((t) & RTE_IPSEC_SATP_IPV_MASK) == RTE_IPSEC_SATP_IPV4)) || \ ((t) & RTE_IPSEC_SATP_MODE_MASK) == RTE_IPSEC_SATP_MODE_TUNLV4) -/* helper routine to free bulk of packets */ -static inline void -free_pkts (struct rte_mbuf *mb[],uint32_t n) -{ -uint32_t i; - -for (i =0 ; i != n; i++) -rte_pktmbuf_free (mb[i]); -} -/* helper routine to free bulk of crypto-ops and related packets */ static inline void free_cops (struct rte_crypto_op *cop[],uint32_t n)
Signature · 1 line
2.7.4
Quoted 18 lines
examples/ipsec-secgw/ipsec-secgw.c +7
-0@@ -1099,6 +1151,10 @@ ipsec_poll_mode_worker(void)const uint64_t drain_tsc = (rte_get_tsc_hz () +US_PER_S -1 ) /US_PER_S *BURST_TX_DRAIN_US ;struct lcore_rx_queue *rxql; +#if (STATS_INTERVAL >0 ) +const uint64_t timer_period =STATS_INTERVAL *rte_get_timer_hz (); +uint64_t timer_tsc =0 ; +#endif /* STATS_INTERVAL */ prev_tsc =0 ; lcore_id =rte_lcore_id ();@@ -1159,6 +1215,19 @@ ipsec_poll_mode_worker(void)drain_tx_buffers (qconf);drain_crypto_buffers (qconf); prev_tsc = cur_tsc; +#if (STATS_INTERVAL >0 ) +if (lcore_id ==rte_get_master_lcore ()) { +/* advance the timer */
examples/ipsec-secgw/ipsec-secgw.c ipsec_poll_mode_worker(void) · line 1230
Quoted 10 lines
examples/ipsec-secgw/ipsec-secgw.c +10
-0+ timer_tsc += diff_tsc; + +/* if timer has reached its timeout */ +if (unlikely (timer_tsc >= timer_period)) { +print_stats (); +/* reset the timer */ + timer_tsc =0 ; + } + } +#endif /* STATS_INTERVAL */
I still don't understand why to do it in data-path thread. As I said in previous comments, in DPDK there is a control thread that can be used for such house-keeping tasks. Why not to use it (via rte_alarm or so) and keep data-path threads less affected.
Quoted 20 lines
examples/ipsec-secgw/ipsec-secgw.c +3
-1}for (i =0 ; i < qconf->nb_rx_queue; ++i) {@@ -1169,8 +1238,10 @@ ipsec_poll_mode_worker(void) nb_rx =rte_eth_rx_burst (portid, queueid, pkts,MAX_PKT_BURST ); -if (nb_rx >0 ) +if (nb_rx >0 ) { +core_stats_update_rx (nb_rx);process_pkts (qconf, pkts, nb_rx, portid); + }/* dequeue and process completed crypto-ops */ if (is_unprotected_port (portid))examples/ipsec-secgw/ipsec-secgw.h +0
-0
examples/ipsec-secgw/ipsec-secgw.h line 9
Quoted 4 lines
examples/ipsec-secgw/ipsec-secgw.h +1
-0@@ -6,6 +6,8 @@#include <stdbool.h> +#define STATS_INTERVAL 0
Shouldn't it be:
#ifndef STATS_INTERVAL
#define STATS_INTERVAL 0
#endif
?
To allow user specify statis interval via EXTRA_CFLAGS='-DSTATS_INTERVAL=10' or so.
Quoted 69 lines
examples/ipsec-secgw/ipsec-secgw.h +55
-0+#define NB_SOCKETS 4#define MAX_PKT_BURST 32@@ -69,6 +71,17 @@ struct ethaddr_info {uint64_t src, dst; }; +#if (STATS_INTERVAL >0 ) +struct ipsec_core_statistics { +uint64_t tx; +uint64_t rx; +uint64_t dropped; +uint64_t burst_rx; +} __rte_cache_aligned; + +struct ipsec_core_statistics core_statistics[RTE_MAX_LCORE ]; +#endif /* STATS_INTERVAL */ +extern struct ethaddr_info ethaddr_tbl[RTE_MAX_ETHPORTS ];/* Port mask to identify the unprotected ports */ @@ -85,4 +98,59 @@ is_unprotected_port(uint16_t port_id)return unprotected_port_mask & (1 << port_id); } +static inline void +core_stats_update_rx (int n) +{ +#if (STATS_INTERVAL >0 ) +int lcore_id =rte_lcore_id (); + core_statistics[lcore_id].rx += n; +if (n ==MAX_PKT_BURST ) + core_statistics[lcore_id].burst_rx += n; +#else +RTE_SET_USED (n); +#endif /* STATS_INTERVAL */ +} + +static inline void +core_stats_update_tx (int n) +{ +#if (STATS_INTERVAL >0 ) +int lcore_id =rte_lcore_id (); + core_statistics[lcore_id].tx += n; +#else +RTE_SET_USED (n); +#endif /* STATS_INTERVAL */ +} + +static inline void +core_stats_update_drop (int n) +{ +#if (STATS_INTERVAL >0 ) +int lcore_id =rte_lcore_id (); + core_statistics[lcore_id].dropped += n; +#else +RTE_SET_USED (n); +#endif /* STATS_INTERVAL */ +} + +/* helper routine to free bulk of packets */ +static inline void +free_pkts (struct rte_mbuf *mb[],uint32_t n) +{ +uint32_t i; + +for (i =0 ; i != n; i++) +rte_pktmbuf_free (mb[i]);
examples/ipsec-secgw/ipsec-secgw.h is_unprotected_port(uint16_t port_id) · line 153
Quoted 10 lines
examples/ipsec-secgw/ipsec-secgw.h +10
-0+ +core_stats_update_drop (n); +} + +/* helper routine to free single packet */ +static inline void +free_pkt (struct rte_mbuf *mb) +{ +rte_pktmbuf_free (mb); +core_stats_update_drop (1 );
Probably just:free_pkts(&mb, 1);
?
Quoted 3 lines
examples/ipsec-secgw/ipsec-secgw.h +2
-0+} +#endif /* _IPSEC_SECGW_H_ */
Hi Konstantin,
Please see inline.
Thanks,
Anoob
Quoted 36 lines
-----Original Message-----
From: Ananyev, Konstantin <[email protected]>
Sent: Thursday, May 7, 2020 9:42 PM
To: Anoob Joseph <[email protected]>; Akhil Goyal <[email protected]>; Nicolau, Radu <[email protected]>
Cc: Narayana Prasad Raju Athreya <[email protected]>; [email protected]
Subject: [EXT] RE: [PATCH v3] examples/ipsec-secgw: add per core packet statsExternal Email
----------------------------------------------------------------------
Quoted 25 lines
examples/ipsec-secgw/ipsec-secgw.c +14
-0@@ -1099,6 +1151,10 @@ ipsec_poll_mode_worker(void)const uint64_t drain_tsc = (rte_get_tsc_hz () +US_PER_S -1 ) /US_PER_S *BURST_TX_DRAIN_US ;struct lcore_rx_queue *rxql; +#if (STATS_INTERVAL >0 ) +const uint64_t timer_period =STATS_INTERVAL *rte_get_timer_hz (); +uint64_t timer_tsc =0 ; +#endif /* STATS_INTERVAL */ prev_tsc =0 ; lcore_id =rte_lcore_id ();@@ -1159,6 +1215,19 @@ ipsec_poll_mode_worker(void)drain_tx_buffers (qconf);drain_crypto_buffers (qconf); prev_tsc = cur_tsc; +#if (STATS_INTERVAL >0 ) +if (lcore_id ==rte_get_master_lcore ()) { +/* advance the timer */ + timer_tsc += diff_tsc; + +/* if timer has reached its timeout */ +if (unlikely (timer_tsc >= timer_period)) { +print_stats (); +/* reset the timer */ + timer_tsc =0 ;
Quoted 8 lines
examples/ipsec-secgw/ipsec-secgw.c adjust_ipv6_pktlen(struct rte_mbuf *m, const struct rte_ipv6_hdr *iph, · line 342
Quoted 3 lines
examples/ipsec-secgw/ipsec-secgw.c +3
-0+ } + } +#endif /* STATS_INTERVAL */ I still don't understand why to do it in data-path thread.
As I said in previous comments, in DPDK there is a control thread that can be used for such house-keeping tasks.
Why not to use it (viarte_alarmor so) and keep data-path threads less affected.
[Anoob] From Marvell's estimates, this stats collection and reporting will be expensive and so cannot be enabled by default. This is required for analyzing the traffic distribution in cases where the performance isn't scaling as expected. And this patch achieves the desired feature. If Intel would like to improve the approach, that can be taken up as a separate patch.
Quoted 26 lines
Review ipsec_poll_mode_worker(void) · line 1233
Quoted 4 lines
(quoted diff) +0
-0} for (i = 0; i < qconf->nb_rx_queue; ++i) { @@ -1169,8 +1238,10@@
Quoted 21 lines
ipsec_poll_mode_worker(void)examples/ipsec-secgw/ipsec-secgw.c +3
-1nb_rx =rte_eth_rx_burst (portid, queueid, pkts,MAX_PKT_BURST ); -if (nb_rx >0 ) +if (nb_rx >0 ) { +core_stats_update_rx (nb_rx);process_pkts (qconf, pkts, nb_rx, portid); + }/* dequeue and process completed crypto-ops */ if (is_unprotected_port (portid))examples/ipsec-secgw/ipsec-secgw.h +0
-0@@ -6,6 +6,8 @@#include <stdbool.h>
Quoted 7 lines
examples/ipsec-secgw/ipsec-secgw.h line 9
Quoted 1 line
examples/ipsec-secgw/ipsec-secgw.h +1
-0+#define STATS_INTERVAL 0Shouldn't it be:
#ifndefSTATS_INTERVAL
#defineSTATS_INTERVAL0
#endif
?
[Anoob] Will update in v4.
Quoted 72 lines
To allow user specify statis interval via EXTRA_CFLAGS='-DSTATS_INTERVAL=10' or so.
Quoted 68 lines
examples/ipsec-secgw/ipsec-secgw.h +12
-0+#define NB_SOCKETS 4#define MAX_PKT_BURST 32@@ -69,6 +71,17 @@ struct ethaddr_info {uint64_t src, dst; }; +#if (STATS_INTERVAL >0 ) +struct ipsec_core_statistics { +uint64_t tx; +uint64_t rx; +uint64_t dropped; +uint64_t burst_rx; +} __rte_cache_aligned; + +struct ipsec_core_statistics core_statistics[RTE_MAX_LCORE ];#endif +/* STATS_INTERVAL */ +extern struct ethaddr_info ethaddr_tbl[RTE_MAX_ETHPORTS ]; /* Port mask to identify the unprotected ports */ @@ -85,4 +98,59 @@
is_unprotected_port(uint16_t port_id)examples/ipsec-secgw/ipsec-secgw.h +43
-0return unprotected_port_mask & (1 << port_id); } +static inline void +core_stats_update_rx (int n) +{ +#if (STATS_INTERVAL >0 ) +int lcore_id =rte_lcore_id (); + core_statistics[lcore_id].rx += n; +if (n ==MAX_PKT_BURST ) + core_statistics[lcore_id].burst_rx += n;#else +RTE_SET_USED (n); +#endif /* STATS_INTERVAL */ +} + +static inline void +core_stats_update_tx (int n) +{ +#if (STATS_INTERVAL >0 ) +int lcore_id =rte_lcore_id (); + core_statistics[lcore_id].tx += n; +#else +RTE_SET_USED (n); +#endif /* STATS_INTERVAL */ +} + +static inline void +core_stats_update_drop (int n) +{ +#if (STATS_INTERVAL >0 ) +int lcore_id =rte_lcore_id (); + core_statistics[lcore_id].dropped += n;#else +RTE_SET_USED (n); +#endif /* STATS_INTERVAL */ +} + +/* helper routine to free bulk of packets */ static inline void +free_pkts (struct rte_mbuf *mb[],uint32_t n) { +uint32_t i; + +for (i =0 ; i != n; i++) +rte_pktmbuf_free (mb[i]); + +core_stats_update_drop (n); +} +
Quoted 8 lines
examples/ipsec-secgw/ipsec-secgw.h is_unprotected_port(uint16_t port_id) · line 153
Quoted 4 lines
examples/ipsec-secgw/ipsec-secgw.h +4
-0+/* helper routine to free single packet */ static inline void +free_pkt (struct rte_mbuf *mb) { +rte_pktmbuf_free (mb); +core_stats_update_drop (1 );Probably just:
free_pkts(&mb, 1);
?
[Anoob] Will update in v4.
Quoted 4 lines
Quoted 4 lines
examples/ipsec-secgw/ipsec-secgw.h +2
-0+} +#endif /* _IPSEC_SECGW_H_ */
Quoted 35 lines
Quoted 26 lines
Quoted 26 lines
examples/ipsec-secgw/ipsec-secgw.c +14
-0@@ -1099,6 +1151,10 @@ ipsec_poll_mode_worker(void)const uint64_t drain_tsc = (rte_get_tsc_hz () +US_PER_S -1 ) /US_PER_S *BURST_TX_DRAIN_US ;struct lcore_rx_queue *rxql; +#if (STATS_INTERVAL >0 ) +const uint64_t timer_period =STATS_INTERVAL *rte_get_timer_hz (); +uint64_t timer_tsc =0 ; +#endif /* STATS_INTERVAL */ prev_tsc =0 ; lcore_id =rte_lcore_id ();@@ -1159,6 +1215,19 @@ ipsec_poll_mode_worker(void)drain_tx_buffers (qconf);drain_crypto_buffers (qconf); prev_tsc = cur_tsc; +#if (STATS_INTERVAL >0 ) +if (lcore_id ==rte_get_master_lcore ()) { +/* advance the timer */ + timer_tsc += diff_tsc; + +/* if timer has reached its timeout */ +if (unlikely (timer_tsc >= timer_period)) { +print_stats (); +/* reset the timer */ + timer_tsc =0 ;Quoted 8 lines
examples/ipsec-secgw/ipsec-secgw.c adjust_ipv6_pktlen(struct rte_mbuf *m, const struct rte_ipv6_hdr *iph, · line 342
Quoted 3 lines
examples/ipsec-secgw/ipsec-secgw.c +3
-0+ } + } +#endif /* STATS_INTERVAL */ I still don't understand why to do it in data-path thread.
As I said in previous comments, in DPDK there is a control thread that can be used for such house-keeping tasks.
Why not to use it (viarte_alarmor so) and keep data-path threads less affected.
Quoted 2 lines
[Anoob] From Marvell's estimates, this stats collection and reporting will be expensive and so cannot be enabled by default. This is required for analyzing the traffic distribution in cases where the performance isn't scaling as expected.
Understood.
Quoted 1 line
And this patch achieves the desired feature.
Ok, but why not to do it in control (house-keeping) thread? That would achieve desired goal and keep data-path unaffected.
Quoted 1 line
If Intel would like to improve the approach, that can be taken up as a separate patch.
This is not a vendor specific part.
You making changes in common data-path code that is used by all ipsec-secgw users. I think it is everyone benefit (and responsibility) to keep common data-path code clean, tidy and fast.
If we can avoid polluting it with extra code, I don't see a reason not to do it.
Quoted 124 lines
Quoted 27 lines
Review ipsec_poll_mode_worker(void) · line 1233
Quoted 5 lines
(quoted diff) +0
-0} for (i = 0; i < qconf->nb_rx_queue; ++i) { @@ -1169,8 +1238,10@@
Quoted 21 lines
ipsec_poll_mode_worker(void)examples/ipsec-secgw/ipsec-secgw.c +3
-1nb_rx =rte_eth_rx_burst (portid, queueid, pkts,MAX_PKT_BURST ); -if (nb_rx >0 ) +if (nb_rx >0 ) { +core_stats_update_rx (nb_rx);process_pkts (qconf, pkts, nb_rx, portid); + }/* dequeue and process completed crypto-ops */ if (is_unprotected_port (portid))examples/ipsec-secgw/ipsec-secgw.h +0
-0@@ -6,6 +6,8 @@#include <stdbool.h> Quoted 7 lines
examples/ipsec-secgw/ipsec-secgw.h line 9
Quoted 1 line
examples/ipsec-secgw/ipsec-secgw.h +1
-0+#define STATS_INTERVAL 0Shouldn't it be:
#ifndefSTATS_INTERVAL
#defineSTATS_INTERVAL0
#endif
?[Anoob] Will update in v4.
Quoted 72 lines
To allow user specify statis interval via EXTRA_CFLAGS='-DSTATS_INTERVAL=10' or so.
Quoted 68 lines
examples/ipsec-secgw/ipsec-secgw.h +12
-0+#define NB_SOCKETS 4#define MAX_PKT_BURST 32@@ -69,6 +71,17 @@ struct ethaddr_info {uint64_t src, dst; }; +#if (STATS_INTERVAL >0 ) +struct ipsec_core_statistics { +uint64_t tx; +uint64_t rx; +uint64_t dropped; +uint64_t burst_rx; +} __rte_cache_aligned; + +struct ipsec_core_statistics core_statistics[RTE_MAX_LCORE ];#endif +/* STATS_INTERVAL */ +extern struct ethaddr_info ethaddr_tbl[RTE_MAX_ETHPORTS ]; /* Port mask to identify the unprotected ports */ @@ -85,4 +98,59 @@
is_unprotected_port(uint16_t port_id)examples/ipsec-secgw/ipsec-secgw.h +43
-0return unprotected_port_mask & (1 << port_id); } +static inline void +core_stats_update_rx (int n) +{ +#if (STATS_INTERVAL >0 ) +int lcore_id =rte_lcore_id (); + core_statistics[lcore_id].rx += n; +if (n ==MAX_PKT_BURST ) + core_statistics[lcore_id].burst_rx += n;#else +RTE_SET_USED (n); +#endif /* STATS_INTERVAL */ +} + +static inline void +core_stats_update_tx (int n) +{ +#if (STATS_INTERVAL >0 ) +int lcore_id =rte_lcore_id (); + core_statistics[lcore_id].tx += n; +#else +RTE_SET_USED (n); +#endif /* STATS_INTERVAL */ +} + +static inline void +core_stats_update_drop (int n) +{ +#if (STATS_INTERVAL >0 ) +int lcore_id =rte_lcore_id (); + core_statistics[lcore_id].dropped += n;#else +RTE_SET_USED (n); +#endif /* STATS_INTERVAL */ +} + +/* helper routine to free bulk of packets */ static inline void +free_pkts (struct rte_mbuf *mb[],uint32_t n) { +uint32_t i; + +for (i =0 ; i != n; i++) +rte_pktmbuf_free (mb[i]); + +core_stats_update_drop (n); +} +Quoted 8 lines
examples/ipsec-secgw/ipsec-secgw.h is_unprotected_port(uint16_t port_id) · line 153
Quoted 4 lines
examples/ipsec-secgw/ipsec-secgw.h +4
-0+/* helper routine to free single packet */ static inline void +free_pkt (struct rte_mbuf *mb) { +rte_pktmbuf_free (mb); +core_stats_update_drop (1 );Probably just:
free_pkts(&mb, 1);
?[Anoob] Will update in v4.
Quoted 4 lines
Quoted 4 lines
examples/ipsec-secgw/ipsec-secgw.h +2
-0+} +#endif /* _IPSEC_SECGW_H_ */
Hi Konstantin,
Please see inline.
Thanks,
Anoob
Quoted 60 lines
-----Original Message-----
From: Ananyev, Konstantin <[email protected]>
Sent: Wednesday, May 13, 2020 6:13 PM
To: Anoob Joseph <[email protected]>; Akhil Goyal <[email protected]>; Nicolau, Radu <[email protected]>
Cc: Narayana Prasad Raju Athreya <[email protected]>; [email protected]
Subject: [EXT] RE: [PATCH v3] examples/ipsec-secgw: add per core packet statsExternal Email
----------------------------------------------------------------------
Quoted 25 lines
Quoted 25 lines
Quoted 25 lines
examples/ipsec-secgw/ipsec-secgw.c +14
-0@@ -1099,6 +1151,10 @@ ipsec_poll_mode_worker(void)const uint64_t drain_tsc = (rte_get_tsc_hz () +US_PER_S -1 ) /US_PER_S *BURST_TX_DRAIN_US ;struct lcore_rx_queue *rxql; +#if (STATS_INTERVAL >0 ) +const uint64_t timer_period =STATS_INTERVAL *rte_get_timer_hz (); +uint64_t timer_tsc =0 ; +#endif /* STATS_INTERVAL */ prev_tsc =0 ; lcore_id =rte_lcore_id ();@@ -1159,6 +1215,19 @@ ipsec_poll_mode_worker(void)drain_tx_buffers (qconf);drain_crypto_buffers (qconf); prev_tsc = cur_tsc; +#if (STATS_INTERVAL >0 ) +if (lcore_id ==rte_get_master_lcore ()) { +/* advance the timer */ + timer_tsc += diff_tsc; + +/* if timer has reached its timeout */ +if (unlikely (timer_tsc >= timer_period)) { +print_stats (); +/* reset the timer */ + timer_tsc =0 ;Quoted 8 lines
Quoted 8 lines
examples/ipsec-secgw/ipsec-secgw.c adjust_ipv6_pktlen(struct rte_mbuf *m, const struct rte_ipv6_hdr *iph, · line 342
Quoted 3 lines
examples/ipsec-secgw/ipsec-secgw.c +3
-0+ } + } +#endif /* STATS_INTERVAL */ I still don't understand why to do it in data-path thread. As I said in previous comments, in DPDK there is a control thread that can be used for such house-keeping tasks.
Why not to use it (viarte_alarmor so) and keep data-path threads lessaffected.
Quoted 3 lines
[Anoob] From Marvell's estimates, this stats collection and reporting will be expensive and so cannot be enabled by default. This is required for
analyzing the traffic distribution in cases where the performance isn't scaling as expected.
Understood.
Quoted 1 line
And this patch achieves the desired feature.
Ok, but why not to do it in control (house-keeping) thread? That would achieve desired goal and keep data-path unaffected.
Quoted 1 line
If Intel would like to improve the approach, that can be taken up as a separate
Quoted 8 lines
patch.
This is not a vendor specific part.
You making changes in common data-path code that is used by all ipsec-secgw users.
I think it is everyone benefit (and responsibility) to keep common data-path code clean, tidy and fast.
If we can avoid polluting it with extra code, I don't see a reason not to do it.
[Anoob] I cannot say I've fully understood what you have suggested. I've submitted v4 based on what I understood from your comments. Please have a look at it.
Quoted 126 lines
Quoted 36 lines
Quoted 28 lines
Review ipsec_poll_mode_worker(void) · line 1233
Quoted 6 lines
(quoted diff) +0
-0} for (i = 0; i < qconf->nb_rx_queue; ++i) { @@ -1169,8 +1238,10@@
Quoted 21 lines
ipsec_poll_mode_worker(void)examples/ipsec-secgw/ipsec-secgw.c +3
-1nb_rx =rte_eth_rx_burst (portid, queueid, pkts,MAX_PKT_BURST ); -if (nb_rx >0 ) +if (nb_rx >0 ) { +core_stats_update_rx (nb_rx);process_pkts (qconf, pkts, nb_rx, portid); + }/* dequeue and process completed crypto-ops */ if (is_unprotected_port(portid)) diff --git
a/examples/ipsec-secgw/ipsec-secgw.hb/examples/ipsec-secgw/ipsec-secgw.h
index 4b53cb5..5b3561f 100644examples/ipsec-secgw/ipsec-secgw.h +0
-0@@ -6,6 +6,8 @@#include <stdbool.h> Quoted 7 lines
examples/ipsec-secgw/ipsec-secgw.h line 9
Quoted 1 line
examples/ipsec-secgw/ipsec-secgw.h +1
-0+#define STATS_INTERVAL 0Shouldn't it be:
#ifndefSTATS_INTERVAL
#defineSTATS_INTERVAL0
#endif
?Quoted 4 lines
[Anoob] Will update in v4.
Quoted 2 lines
To allow user specify statis interval via EXTRA_CFLAGS='-
DSTATS_INTERVAL=10'
Quoted 85 lines
Quoted 70 lines
or so.
Quoted 68 lines
examples/ipsec-secgw/ipsec-secgw.h +13
-0+#define NB_SOCKETS 4#define MAX_PKT_BURST 32@@ -69,6 +71,17 @@ struct ethaddr_info {uint64_t src, dst; }; +#if (STATS_INTERVAL >0 ) +struct ipsec_core_statistics { +uint64_t tx; +uint64_t rx; +uint64_t dropped; +uint64_t burst_rx; +} __rte_cache_aligned; + +struct ipsec_core_statistics core_statistics[RTE_MAX_LCORE ]; +#endif +/* STATS_INTERVAL */ +extern struct ethaddr_info ethaddr_tbl[RTE_MAX_ETHPORTS ]; /* Port mask to identify the unprotected ports */ @@ -85,4 +98,59@@
is_unprotected_port(uint16_t port_id)examples/ipsec-secgw/ipsec-secgw.h +42
-0return unprotected_port_mask & (1 << port_id); } +static inline void +core_stats_update_rx (int n) +{ +#if (STATS_INTERVAL >0 ) +int lcore_id =rte_lcore_id (); + core_statistics[lcore_id].rx += n; +if (n ==MAX_PKT_BURST ) + core_statistics[lcore_id].burst_rx += n;#else +RTE_SET_USED (n); +#endif /* STATS_INTERVAL */ +} + +static inline void +core_stats_update_tx (int n) +{ +#if (STATS_INTERVAL >0 ) +int lcore_id =rte_lcore_id (); + core_statistics[lcore_id].tx += n;#else +RTE_SET_USED (n); +#endif /* STATS_INTERVAL */ +} + +static inline void +core_stats_update_drop (int n) +{ +#if (STATS_INTERVAL >0 ) +int lcore_id =rte_lcore_id (); + core_statistics[lcore_id].dropped += n;#else +RTE_SET_USED (n); +#endif /* STATS_INTERVAL */ +} + +/* helper routine to free bulk of packets */ static inline void +free_pkts (struct rte_mbuf *mb[],uint32_t n) { +uint32_t i; + +for (i =0 ; i != n; i++) +rte_pktmbuf_free (mb[i]); + +core_stats_update_drop (n); +} +Quoted 8 lines
examples/ipsec-secgw/ipsec-secgw.h is_unprotected_port(uint16_t port_id) · line 153
Quoted 4 lines
examples/ipsec-secgw/ipsec-secgw.h +4
-0+/* helper routine to free single packet */ static inline void +free_pkt (struct rte_mbuf *mb) { +rte_pktmbuf_free (mb); +core_stats_update_drop (1 );Probably just:
free_pkts(&mb, 1);
?[Anoob] Will update in v4.
Quoted 4 lines
Quoted 4 lines
examples/ipsec-secgw/ipsec-secgw.h +2
-0+} +#endif /* _IPSEC_SECGW_H_ */
Quoted 29 lines
examples/ipsec-secgw/ipsec-secgw.c +27
-0+#if (STATS_INTERVAL >0 ) + +/* Print out statistics on packet distribution */ +static void +print_stats (void ) +{ +uint64_t total_packets_dropped, total_packets_tx, total_packets_rx; +unsigned int coreid; +float burst_percent; + + total_packets_dropped =0 ; + total_packets_tx =0 ; + total_packets_rx =0 ; + +const char clr[] = {27 ,'[' ,'2' ,'J' ,'\0' }; + +/* Clear screen and move to top left */ +printf ("%s" , clr); + +printf ("\nCore statistics ====================================" ); + +for (coreid =0 ; coreid <RTE_MAX_LCORE ; coreid++) { +/* skip disabled cores */ +if (rte_lcore_is_enabled (coreid) ==0 ) +continue ; + burst_percent = (float )(core_statistics[coreid].burst_rx *100 )/ + core_statistics[coreid].rx;
examples/ipsec-secgw/ipsec-secgw.c adjust_ipv6_pktlen(struct rte_mbuf *m, const struct rte_ipv6_hdr *iph, · line 327
Quoted 10 lines
examples/ipsec-secgw/ipsec-secgw.c +10
-0+printf ("\nStatistics for core %u ------------------------------" +"\nPackets received: %20" PRIu64 +"\nPackets sent: %24" PRIu64 +"\nPackets dropped: %21" PRIu64 +"\nBurst percent: %23.2f" , + coreid, + core_statistics[coreid].rx, + core_statistics[coreid].tx, + core_statistics[coreid].dropped, + burst_percent);
As one more comment:
Can we also collect number of calls and display average pkt/call (for both rx and tx)?
Quoted 421 lines
examples/ipsec-secgw/ipsec-secgw.c +55
-21+ + total_packets_dropped += core_statistics[coreid].dropped; + total_packets_tx += core_statistics[coreid].tx; + total_packets_rx += core_statistics[coreid].rx; + } +printf ("\nAggregate statistics ===============================" +"\nTotal packets received: %14" PRIu64 +"\nTotal packets sent: %18" PRIu64 +"\nTotal packets dropped: %15" PRIu64, + total_packets_rx, + total_packets_tx, + total_packets_dropped); +printf ("\n====================================================\n" ); +} +#endif /* STATS_INTERVAL */ +static inline void prepare_one_packet (struct rte_mbuf *pkt,struct ipsec_traffic *t) {@@ -333,7 +386,7 @@ prepare_one_packet(struct rte_mbuf *pkt, struct ipsec_traffic *t)/* drop packet when IPv6 header exceeds first segment length */ if (unlikely (l3len > pkt->data_len)) { -rte_pktmbuf_free (pkt); +free_pkt (pkt);return ; }@@ -350,7 +403,7 @@ prepare_one_packet(struct rte_mbuf *pkt, struct ipsec_traffic *t)/* Unknown/Unsupported type, drop the packet */ RTE_LOG (ERR ,IPSEC ,"Unsupported packet type 0x%x\n" ,rte_be_to_cpu_16 (eth->ether_type)); -rte_pktmbuf_free (pkt); +free_pkt (pkt);return ; }@@ -477,9 +530,12 @@ send_burst(struct lcore_conf *qconf, uint16_t n, uint16_t port)prepare_tx_burst (m_table, n, port, qconf); ret =rte_eth_tx_burst (port, queueid, m_table, n); + +core_stats_update_tx (ret); +if (unlikely (ret < n)) {do { -rte_pktmbuf_free (m_table[ret]); +free_pkt (m_table[ret]); }while (++ret < n); }@@ -525,7 +581,7 @@ send_fragment_packet(struct lcore_conf *qconf, struct rte_mbuf *m,"error code: %d\n" , __func__, m->pkt_len, rte_errno); -rte_pktmbuf_free (m); +free_pkt (m);return len; }@@ -550,7 +606,7 @@ send_single_packet(struct rte_mbuf *m, uint16_t port, uint8_t proto) }else if (frag_tbl_sz >0 ) len =send_fragment_packet (qconf, m, port, proto);else -rte_pktmbuf_free (m); +free_pkt (m);/* enough pkts to be sent */ if (unlikely (len ==MAX_PKT_BURST )) {@@ -584,19 +640,19 @@ inbound_sp_sa(struct sp_ctx *sp, struct sa_ctx *sa, struct traffic_type *ip,continue ; }if (res ==DISCARD ) { -rte_pktmbuf_free (m); +free_pkt (m);continue ; }/* Only check SPI match for processed IPSec packets */ if (i < lim && ((m->ol_flags &PKT_RX_SEC_OFFLOAD ) ==0 )) { -rte_pktmbuf_free (m); +free_pkt (m);continue ; } sa_idx = res -1 ;if (!inbound_sa_check (sa, m, sa_idx)) { -rte_pktmbuf_free (m); +free_pkt (m);continue ; } ip->pkts[j++] = m;@@ -631,7 +687,7 @@ split46_traffic(struct ipsec_traffic *trf, struct rte_mbuf *mb[], uint32_t num) offsetof(struct ip6_hdr, ip6_nxt)); n6++; }else -rte_pktmbuf_free (m); +free_pkt (m); } trf->ip4.num = n4;@@ -683,7 +739,7 @@ outbound_sp(struct sp_ctx *sp, struct traffic_type *ip, m = ip->pkts[i]; sa_idx = ip->res[i] -1 ;if (ip->res[i] ==DISCARD ) -rte_pktmbuf_free (m); +free_pkt (m);else if (ip->res[i] ==BYPASS ) ip->pkts[j++] = m;else {@@ -702,8 +758,7 @@ process_pkts_outbound(struct ipsec_ctx *ipsec_ctx,uint16_t idx, nb_pkts_out, i;/* Drop any IPsec traffic from protected ports */ -for (i =0 ; i < traffic->ipsec.num; i++) -rte_pktmbuf_free (traffic->ipsec.pkts[i]); +free_pkts (traffic->ipsec.pkts, traffic->ipsec.num); traffic->ipsec.num =0 ;@@ -743,14 +798,12 @@ process_pkts_inbound_nosp(struct ipsec_ctx *ipsec_ctx,uint32_t nb_pkts_in, i, idx;/* Drop any IPv4 traffic from unprotected ports */ -for (i =0 ; i < traffic->ip4.num; i++) -rte_pktmbuf_free (traffic->ip4.pkts[i]); +free_pkts (traffic->ip4.pkts, traffic->ip4.num); traffic->ip4.num =0 ;/* Drop any IPv6 traffic from unprotected ports */ -for (i =0 ; i < traffic->ip6.num; i++) -rte_pktmbuf_free (traffic->ip6.pkts[i]); +free_pkts (traffic->ip6.pkts, traffic->ip6.num); traffic->ip6.num =0 ;@@ -786,8 +839,7 @@ process_pkts_outbound_nosp(struct ipsec_ctx *ipsec_ctx,struct ip *ip;/* Drop any IPsec traffic from protected ports */ -for (i =0 ; i < traffic->ipsec.num; i++) -rte_pktmbuf_free (traffic->ipsec.pkts[i]); +free_pkts (traffic->ipsec.pkts, traffic->ipsec.num); n =0 ;@@ -901,7 +953,7 @@ route4_pkts(struct rt_ctx *rt_ctx, struct rte_mbuf *pkts[], uint8_t nb_pkts) }if ((pkt_hop &RTE_LPM_LOOKUP_SUCCESS ) ==0 ) { -rte_pktmbuf_free (pkts[i]); +free_pkt (pkts[i]);continue ; }send_single_packet (pkts[i], pkt_hop &0xff ,IPPROTO_IP );@@ -953,7 +1005,7 @@ route6_pkts(struct rt_ctx *rt_ctx, struct rte_mbuf *pkts[], uint8_t nb_pkts) }if (pkt_hop ==-1 ) { -rte_pktmbuf_free (pkts[i]); +free_pkt (pkts[i]);continue ; }send_single_packet (pkts[i], pkt_hop &0xff ,IPPROTO_IPV6 );@@ -1099,6 +1151,10 @@ ipsec_poll_mode_worker(void)const uint64_t drain_tsc = (rte_get_tsc_hz () +US_PER_S -1 ) /US_PER_S *BURST_TX_DRAIN_US ;struct lcore_rx_queue *rxql; +#if (STATS_INTERVAL >0 ) +const uint64_t timer_period =STATS_INTERVAL *rte_get_timer_hz (); +uint64_t timer_tsc =0 ; +#endif /* STATS_INTERVAL */ prev_tsc =0 ; lcore_id =rte_lcore_id ();@@ -1159,6 +1215,19 @@ ipsec_poll_mode_worker(void)drain_tx_buffers (qconf);drain_crypto_buffers (qconf); prev_tsc = cur_tsc; +#if (STATS_INTERVAL >0 ) +if (lcore_id ==rte_get_master_lcore ()) { +/* advance the timer */ + timer_tsc += diff_tsc; + +/* if timer has reached its timeout */ +if (unlikely (timer_tsc >= timer_period)) { +print_stats (); +/* reset the timer */ + timer_tsc =0 ; + } + } +#endif /* STATS_INTERVAL */ }for (i =0 ; i < qconf->nb_rx_queue; ++i) {@@ -1169,8 +1238,10 @@ ipsec_poll_mode_worker(void) nb_rx =rte_eth_rx_burst (portid, queueid, pkts,MAX_PKT_BURST ); -if (nb_rx >0 ) +if (nb_rx >0 ) { +core_stats_update_rx (nb_rx);process_pkts (qconf, pkts, nb_rx, portid); + }/* dequeue and process completed crypto-ops */ if (is_unprotected_port (portid))examples/ipsec-secgw/ipsec-secgw.h +68
-0@@ -6,6 +6,8 @@#include <stdbool.h> +#define STATS_INTERVAL 0 +#define NB_SOCKETS 4#define MAX_PKT_BURST 32@@ -69,6 +71,17 @@ struct ethaddr_info {uint64_t src, dst; }; +#if (STATS_INTERVAL >0 ) +struct ipsec_core_statistics { +uint64_t tx; +uint64_t rx; +uint64_t dropped; +uint64_t burst_rx; +} __rte_cache_aligned; + +struct ipsec_core_statistics core_statistics[RTE_MAX_LCORE ]; +#endif /* STATS_INTERVAL */ +extern struct ethaddr_info ethaddr_tbl[RTE_MAX_ETHPORTS ];/* Port mask to identify the unprotected ports */ @@ -85,4 +98,59 @@ is_unprotected_port(uint16_t port_id)return unprotected_port_mask & (1 << port_id); } +static inline void +core_stats_update_rx (int n) +{ +#if (STATS_INTERVAL >0 ) +int lcore_id =rte_lcore_id (); + core_statistics[lcore_id].rx += n; +if (n ==MAX_PKT_BURST ) + core_statistics[lcore_id].burst_rx += n; +#else +RTE_SET_USED (n); +#endif /* STATS_INTERVAL */ +} + +static inline void +core_stats_update_tx (int n) +{ +#if (STATS_INTERVAL >0 ) +int lcore_id =rte_lcore_id (); + core_statistics[lcore_id].tx += n; +#else +RTE_SET_USED (n); +#endif /* STATS_INTERVAL */ +} + +static inline void +core_stats_update_drop (int n) +{ +#if (STATS_INTERVAL >0 ) +int lcore_id =rte_lcore_id (); + core_statistics[lcore_id].dropped += n; +#else +RTE_SET_USED (n); +#endif /* STATS_INTERVAL */ +} + +/* helper routine to free bulk of packets */ +static inline void +free_pkts (struct rte_mbuf *mb[],uint32_t n) +{ +uint32_t i; + +for (i =0 ; i != n; i++) +rte_pktmbuf_free (mb[i]); + +core_stats_update_drop (n); +} + +/* helper routine to free single packet */ +static inline void +free_pkt (struct rte_mbuf *mb) +{ +rte_pktmbuf_free (mb); +core_stats_update_drop (1 ); +} +#endif /* _IPSEC_SECGW_H_ */ examples/ipsec-secgw/ipsec.c +10
-10@@ -500,7 +500,7 @@ enqueue_cop_burst(struct cdev_qp *cqp) cqp->id, cqp->qp, ret, len);/* drop packets that we fail to enqueue */ for (i = ret; i < len; i++) -rte_pktmbuf_free (cqp->buf[i]->sym->m_src); +free_pkt (cqp->buf[i]->sym->m_src); } cqp->in_flight += ret; cqp->len =0 ;@@ -528,7 +528,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,for (i =0 ; i < nb_pkts; i++) {if (unlikely (sas[i] ==NULL )) { -rte_pktmbuf_free (pkts[i]); +free_pkt (pkts[i]);continue ; }@@ -549,7 +549,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,if ((unlikely (ips->security.ses ==NULL )) &&create_lookaside_session (ipsec_ctx, sa, ips)) { -rte_pktmbuf_free (pkts[i]); +free_pkt (pkts[i]);continue ; }@@ -563,7 +563,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,case RTE_SECURITY_ACTION_TYPE_CPU_CRYPTO :RTE_LOG (ERR ,IPSEC ,"CPU crypto is not supported by the" " legacy mode." ); -rte_pktmbuf_free (pkts[i]); +free_pkt (pkts[i]);continue ;case RTE_SECURITY_ACTION_TYPE_NONE :@@ -575,7 +575,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,if ((unlikely (ips->crypto.ses ==NULL )) &&create_lookaside_session (ipsec_ctx, sa, ips)) { -rte_pktmbuf_free (pkts[i]); +free_pkt (pkts[i]);continue ; }@@ -584,7 +584,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx, ret =xform_func (pkts[i], sa, &priv->cop);if (unlikely (ret)) { -rte_pktmbuf_free (pkts[i]); +free_pkt (pkts[i]);continue ; }break ;@@ -608,7 +608,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx, ret =xform_func (pkts[i], sa, &priv->cop);if (unlikely (ret)) { -rte_pktmbuf_free (pkts[i]); +free_pkt (pkts[i]);continue ; }@@ -643,7 +643,7 @@ ipsec_inline_dequeue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx, sa = priv->sa; ret =xform_func (pkt, sa, &priv->cop);if (unlikely (ret)) { -rte_pktmbuf_free (pkt); +free_pkt (pkt);continue ; } pkts[nb_pkts++] = pkt;@@ -690,13 +690,13 @@ ipsec_dequeue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx, RTE_SECURITY_ACTION_TYPE_NONE) { ret =xform_func (pkt, sa, cops[j]);if (unlikely (ret)) { -rte_pktmbuf_free (pkt); +free_pkt (pkt);continue ; } } else if (ipsec_get_action_type(sa) ==RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL ) {if (cops[j]->status) { -rte_pktmbuf_free (pkt); +free_pkt (pkt);continue ; } }examples/ipsec-secgw/ipsec_process.c +1
-11@@ -12,22 +12,13 @@#include <rte_mbuf.h> #include "ipsec.h" +#include "ipsec-secgw.h" #define SATP_OUT_IPV4 (t) \ ((((t) & RTE_IPSEC_SATP_MODE_MASK) == RTE_IPSEC_SATP_MODE_TRANS && \ (((t) & RTE_IPSEC_SATP_IPV_MASK) == RTE_IPSEC_SATP_IPV4)) || \ ((t) & RTE_IPSEC_SATP_MODE_MASK) == RTE_IPSEC_SATP_MODE_TUNLV4) -/* helper routine to free bulk of packets */ -static inline void -free_pkts (struct rte_mbuf *mb[],uint32_t n) -{ -uint32_t i; - -for (i =0 ; i != n; i++) -rte_pktmbuf_free (mb[i]); -} -/* helper routine to free bulk of crypto-ops and related packets */ static inline void free_cops (struct rte_crypto_op *cop[],uint32_t n) --2.7.4
Hi Konstantin,
Please see inline.
Thanks,
Anoob
Quoted 46 lines
-----Original Message-----
From: Ananyev, Konstantin <[email protected]>
Sent: Friday, May 8, 2020 1:39 PM
To: Anoob Joseph <[email protected]>; Akhil Goyal <[email protected]>; Nicolau, Radu <[email protected]>
Cc: Narayana Prasad Raju Athreya <[email protected]>; [email protected]
Subject: [EXT] RE: [PATCH v3] examples/ipsec-secgw: add per core packet statsExternal Email
----------------------------------------------------------------------
Quoted 34 lines
examples/ipsec-secgw/ipsec-secgw.c +33
-0+#if (STATS_INTERVAL >0 ) + +/* Print out statistics on packet distribution */ static void +print_stats (void ) +{ +uint64_t total_packets_dropped, total_packets_tx, total_packets_rx; +unsigned int coreid; +float burst_percent; + + total_packets_dropped =0 ; + total_packets_tx =0 ; + total_packets_rx =0 ; + +const char clr[] = {27 ,'[' ,'2' ,'J' ,'\0' }; + +/* Clear screen and move to top left */ +printf ("%s" , clr); + +printf ("\nCore statistics ====================================" ); + +for (coreid =0 ; coreid <RTE_MAX_LCORE ; coreid++) { +/* skip disabled cores */ +if (rte_lcore_is_enabled (coreid) ==0 ) +continue ; + burst_percent = (float )(core_statistics[coreid].burst_rx *100 )/ + core_statistics[coreid].rx; + printf("\nStatistics for core %u ------------------------------" + "\nPackets received: %20"PRIu64 + "\nPackets sent: %24"PRIu64 + "\nPackets dropped: %21"PRIu64 + "\nBurst percent: %23.2f", + coreid, + core_statistics[coreid].rx,
Quoted 9 lines
examples/ipsec-secgw/ipsec-secgw.c adjust_ipv6_pktlen(struct rte_mbuf *m, const struct rte_ipv6_hdr *iph, · line 327
Quoted 3 lines
examples/ipsec-secgw/ipsec-secgw.c +3
-0+ core_statistics[coreid].tx, + core_statistics[coreid].dropped, + burst_percent);As one more comment:
Can we also collect number of calls and display average pkt/call (for both rx and tx)?
[Anoob] Can you describe which "call" you meant? We can capture more logs if required.
Quoted 440 lines
examples/ipsec-secgw/ipsec-secgw.c adjust_ipv6_pktlen(struct rte_mbuf *m, const struct rte_ipv6_hdr *iph, · line 343
Quoted 13 lines
examples/ipsec-secgw/ipsec-secgw.c +13
-0+ + total_packets_dropped += core_statistics[coreid].dropped; + total_packets_tx += core_statistics[coreid].tx; + total_packets_rx += core_statistics[coreid].rx; + } +printf ("\nAggregate statistics ===============================" +"\nTotal packets received: %14" PRIu64 +"\nTotal packets sent: %18" PRIu64 +"\nTotal packets dropped: %15" PRIu64, + total_packets_rx, + total_packets_tx, + total_packets_dropped); +printf("\n====================================================\n");
examples/ipsec-secgw/ipsec-secgw.c prepare_one_packet(struct rte_mbuf *pkt, struct · line 388
Quoted 9 lines
examples/ipsec-secgw/ipsec-secgw.c +3
-0+} +#endif /* STATS_INTERVAL */ +static inline void prepare_one_packet (struct rte_mbuf *pkt,struct ipsec_traffic *t) {@@ -333,7 +386,7 @@ prepare_one_packet(struct rte_mbuf *pkt, struct ipsec_traffic *t)/* drop packet when IPv6 header exceeds first segment length */
examples/ipsec-secgw/ipsec-secgw.c prepare_one_packet(struct rte_mbuf *pkt, struct
Quoted 7 lines
examples/ipsec-secgw/ipsec-secgw.c +1
-1if (unlikely (l3len > pkt->data_len)) { -rte_pktmbuf_free (pkt); +free_pkt (pkt);return ; }@@ -350,7 +403,7 @@ prepare_one_packet(struct rte_mbuf *pkt, struct
ipsec_traffic*t)examples/ipsec-secgw/ipsec-secgw.c send_burst(struct lcore_conf *qconf, uint16_t n,
Quoted 9 lines
examples/ipsec-secgw/ipsec-secgw.c +1
-1/* Unknown/Unsupported type, drop the packet */ RTE_LOG (ERR ,IPSEC ,"Unsupported packet type 0x%x\n" ,rte_be_to_cpu_16 (eth->ether_type)); -rte_pktmbuf_free (pkt); +free_pkt (pkt);return ; }@@ -477,9 +530,12 @@ send_burst(struct lcore_conf *qconf, uint16_t n,
uint16_tport)examples/ipsec-secgw/ipsec-secgw.c send_fragment_packet(struct lcore_conf *qconf,
Quoted 14 lines
examples/ipsec-secgw/ipsec-secgw.c +4
-1prepare_tx_burst (m_table, n, port, qconf); ret =rte_eth_tx_burst (port, queueid, m_table, n); + +core_stats_update_tx (ret); +if (unlikely (ret < n)) {do { -rte_pktmbuf_free (m_table[ret]); +free_pkt (m_table[ret]); }while (++ret < n); }@@ -525,7 +581,7 @@ send_fragment_packet(struct lcore_conf *qconf,struct
rte_mbuf*m,examples/ipsec-secgw/ipsec-secgw.c send_single_packet(struct rte_mbuf *m, uint16_t
Quoted 9 lines
examples/ipsec-secgw/ipsec-secgw.c +1
-1"error code: %d\n" , __func__, m->pkt_len, rte_errno); -rte_pktmbuf_free (m); +free_pkt (m);return len; }@@ -550,7 +606,7 @@ send_single_packet(struct rte_mbuf *m, uint16_tport,
uint8_tproto)examples/ipsec-secgw/ipsec-secgw.c split46_traffic(struct ipsec_traffic *trf, struct rte_mbuf
Quoted 32 lines
examples/ipsec-secgw/ipsec-secgw.c +1
-1}else if (frag_tbl_sz >0 ) len =send_fragment_packet (qconf, m, port, proto);else -rte_pktmbuf_free (m); +free_pkt (m);/* enough pkts to be sent */ if (unlikely(len == MAX_PKT_BURST)) { @@ -584,19 +640,19 @@inbound_sp_sa(struct
sp_ctx*sp, structsa_ctx*sa, structtraffic_type*ip,examples/ipsec-secgw/ipsec-secgw.c +3
-3continue ; }if (res ==DISCARD ) { -rte_pktmbuf_free (m); +free_pkt (m);continue ; }/* Only check SPI match for processed IPSec packets */ if (i < lim && ((m->ol_flags &PKT_RX_SEC_OFFLOAD ) ==0 )) { -rte_pktmbuf_free (m); +free_pkt (m);continue ; } sa_idx = res -1 ;if (!inbound_sa_check (sa, m, sa_idx)) { -rte_pktmbuf_free (m); +free_pkt (m);continue ; } ip->pkts[j++] = m;@@ -631,7 +687,7 @@ split46_traffic(struct ipsec_traffic *trf, struct rte_mbuf*mb[],
uint32_tnum)examples/ipsec-secgw/ipsec-secgw.c process_pkts_inbound_nosp(struct ipsec_ctx
Quoted 28 lines
examples/ipsec-secgw/ipsec-secgw.c +3
-4offsetof(struct ip6_hdr, ip6_nxt)); n6++; }else -rte_pktmbuf_free (m); +free_pkt (m); } trf->ip4.num = n4;@@ -683,7 +739,7 @@ outbound_sp(struct sp_ctx *sp, struct traffic_type *ip, m = ip->pkts[i]; sa_idx = ip->res[i] -1 ;if (ip->res[i] ==DISCARD ) -rte_pktmbuf_free (m); +free_pkt (m);else if (ip->res[i] ==BYPASS ) ip->pkts[j++] = m;else {@@ -702,8 +758,7 @@ process_pkts_outbound(struct ipsec_ctx *ipsec_ctx,uint16_t idx, nb_pkts_out, i;/* Drop any IPsec traffic from protected ports */ -for (i =0 ; i < traffic->ipsec.num; i++) -rte_pktmbuf_free (traffic->ipsec.pkts[i]); +free_pkts (traffic->ipsec.pkts, traffic->ipsec.num); traffic->ipsec.num =0 ;@@ -743,14 +798,12 @@ process_pkts_inbound_nosp(struct ipsec_ctx
*ipsec_ctx,examples/ipsec-secgw/ipsec-secgw.c process_pkts_outbound_nosp(struct ipsec_ctx
Quoted 17 lines
examples/ipsec-secgw/ipsec-secgw.c +2
-4uint32_t nb_pkts_in, i, idx;/* Drop any IPv4 traffic from unprotected ports */ -for (i =0 ; i < traffic->ip4.num; i++) -rte_pktmbuf_free (traffic->ip4.pkts[i]); +free_pkts (traffic->ip4.pkts, traffic->ip4.num); traffic->ip4.num =0 ;/* Drop any IPv6 traffic from unprotected ports */ -for (i =0 ; i < traffic->ip6.num; i++) -rte_pktmbuf_free (traffic->ip6.pkts[i]); +free_pkts (traffic->ip6.pkts, traffic->ip6.num); traffic->ip6.num =0 ;@@ -786,8 +839,7 @@ process_pkts_outbound_nosp(struct ipsec_ctx
*ipsec_ctx,examples/ipsec-secgw/ipsec-secgw.c route4_pkts(struct rt_ctx *rt_ctx, struct rte_mbuf
Quoted 10 lines
examples/ipsec-secgw/ipsec-secgw.c +1
-2struct ip *ip;/* Drop any IPsec traffic from protected ports */ -for (i =0 ; i < traffic->ipsec.num; i++) -rte_pktmbuf_free (traffic->ipsec.pkts[i]); +free_pkts (traffic->ipsec.pkts, traffic->ipsec.num); n =0 ;@@ -901,7 +953,7 @@ route4_pkts(struct rt_ctx *rt_ctx, struct rte_mbuf*pkts[],
uint8_tnb_pkts)examples/ipsec-secgw/ipsec-secgw.c route4_pkts(struct rt_ctx *rt_ctx, struct rte_mbuf *pkts[], uint8_t nb_pkts)
Quoted 8 lines
examples/ipsec-secgw/ipsec-secgw.c +1
-1}if ((pkt_hop &RTE_LPM_LOOKUP_SUCCESS ) ==0 ) { -rte_pktmbuf_free (pkts[i]); +free_pkt (pkts[i]);continue ; } send_single_packet(pkts[i], pkt_hop & 0xff, IPPROTO_IP); @@ -953,7
Quoted 1 line
+1005,7 @@ route6_pkts(struct
rt_ctx*rt_ctx, structrte_mbuf*pkts[],
uint8_tnb_pkts)examples/ipsec-secgw/ipsec-secgw.c route6_pkts(struct rt_ctx *rt_ctx, struct rte_mbuf *pkts[], uint8_t nb_pkts) · line 1011
Quoted 8 lines
examples/ipsec-secgw/ipsec-secgw.c +1
-1}if (pkt_hop ==-1 ) { -rte_pktmbuf_free (pkts[i]); +free_pkt (pkts[i]);continue ; }send_single_packet (pkts[i], pkt_hop &0xff ,IPPROTO_IPV6 );@@
Quoted 21 lines
-1099,6 +1151,10 @@
ipsec_poll_mode_worker(void)examples/ipsec-secgw/ipsec-secgw.c +10
-0const uint64_t drain_tsc = (rte_get_tsc_hz () +US_PER_S -1 ) /US_PER_S *BURST_TX_DRAIN_US ;struct lcore_rx_queue *rxql; +#if (STATS_INTERVAL >0 ) +const uint64_t timer_period =STATS_INTERVAL *rte_get_timer_hz (); +uint64_t timer_tsc =0 ; +#endif /* STATS_INTERVAL */ prev_tsc =0 ; lcore_id =rte_lcore_id ();@@ -1159,6 +1215,19 @@ ipsec_poll_mode_worker(void)drain_tx_buffers (qconf);drain_crypto_buffers (qconf); prev_tsc = cur_tsc; +#if (STATS_INTERVAL >0 ) +if (lcore_id ==rte_get_master_lcore ()) { +/* advance the timer */ + timer_tsc += diff_tsc; + +/* if timer has reached its timeout */ examples/ipsec-secgw/ipsec-secgw.c ipsec_poll_mode_worker(void) · line 1233
Quoted 10 lines
examples/ipsec-secgw/ipsec-secgw.c +7
-0+if (unlikely (timer_tsc >= timer_period)) { +print_stats (); +/* reset the timer */ + timer_tsc =0 ; + } + } +#endif /* STATS_INTERVAL */ } for (i = 0; i < qconf->nb_rx_queue; ++i) { @@ -1169,8 +1238,10@@
examples/ipsec-secgw/ipsec.c ipsec_enqueue(ipsec_xform_fn xform_func, struct
Quoted 131 lines
ipsec_poll_mode_worker(void)examples/ipsec-secgw/ipsec-secgw.c +3
-1nb_rx =rte_eth_rx_burst (portid, queueid, pkts,MAX_PKT_BURST ); -if (nb_rx >0 ) +if (nb_rx >0 ) { +core_stats_update_rx (nb_rx);process_pkts (qconf, pkts, nb_rx, portid); + }/* dequeue and process completed crypto-ops */ if (is_unprotected_port (portid))examples/ipsec-secgw/ipsec-secgw.h +13
-0@@ -6,6 +6,8 @@#include <stdbool.h> +#define STATS_INTERVAL 0 +#define NB_SOCKETS 4#define MAX_PKT_BURST 32@@ -69,6 +71,17 @@ struct ethaddr_info {uint64_t src, dst; }; +#if (STATS_INTERVAL >0 ) +struct ipsec_core_statistics { +uint64_t tx; +uint64_t rx; +uint64_t dropped; +uint64_t burst_rx; +} __rte_cache_aligned; + +struct ipsec_core_statistics core_statistics[RTE_MAX_LCORE ];#endif +/* STATS_INTERVAL */ +extern struct ethaddr_info ethaddr_tbl[RTE_MAX_ETHPORTS ]; /* Port mask to identify the unprotected ports */ @@ -85,4 +98,59 @@
is_unprotected_port(uint16_t port_id)examples/ipsec-secgw/ipsec-secgw.h +49
-0return unprotected_port_mask & (1 << port_id); } +static inline void +core_stats_update_rx (int n) +{ +#if (STATS_INTERVAL >0 ) +int lcore_id =rte_lcore_id (); + core_statistics[lcore_id].rx += n; +if (n ==MAX_PKT_BURST ) + core_statistics[lcore_id].burst_rx += n;#else +RTE_SET_USED (n); +#endif /* STATS_INTERVAL */ +} + +static inline void +core_stats_update_tx (int n) +{ +#if (STATS_INTERVAL >0 ) +int lcore_id =rte_lcore_id (); + core_statistics[lcore_id].tx += n; +#else +RTE_SET_USED (n); +#endif /* STATS_INTERVAL */ +} + +static inline void +core_stats_update_drop (int n) +{ +#if (STATS_INTERVAL >0 ) +int lcore_id =rte_lcore_id (); + core_statistics[lcore_id].dropped += n;#else +RTE_SET_USED (n); +#endif /* STATS_INTERVAL */ +} + +/* helper routine to free bulk of packets */ static inline void +free_pkts (struct rte_mbuf *mb[],uint32_t n) { +uint32_t i; + +for (i =0 ; i != n; i++) +rte_pktmbuf_free (mb[i]); + +core_stats_update_drop (n); +} + +/* helper routine to free single packet */ static inline void +free_pkt (struct rte_mbuf *mb) { +rte_pktmbuf_free (mb); +core_stats_update_drop (1 ); +} +#endif /* _IPSEC_SECGW_H_ */ (quoted diff) +0
-0
b/examples/ipsec-secgw/ipsec.cindex bf88d80..351f1f1 100644examples/ipsec-secgw/ipsec.c +3
-3@@ -500,7 +500,7 @@ enqueue_cop_burst(struct cdev_qp *cqp) cqp->id, cqp->qp, ret, len);/* drop packets that we fail to enqueue */ for (i = ret; i < len; i++) -rte_pktmbuf_free (cqp->buf[i]->sym->m_src); +free_pkt (cqp->buf[i]->sym->m_src); } cqp->in_flight += ret; cqp->len =0 ;@@ -528,7 +528,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx, for (i = 0; i < nb_pkts; i++) {if (unlikely (sas[i] ==NULL )) { -rte_pktmbuf_free (pkts[i]); +free_pkt (pkts[i]);continue ; }@@ -549,7 +549,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx, if ((unlikely(ips->security.ses == NULL)) && create_lookaside_session(ipsec_ctx, sa, ips)) { -rte_pktmbuf_free (pkts[i]); +free_pkt (pkts[i]);continue ; }@@ -563,7 +563,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct
ipsec_ctx*ipsec_ctx,examples/ipsec-secgw/ipsec.c ipsec_enqueue(ipsec_xform_fn xform_func, struct · line 564
Quoted 2 lines
examples/ipsec-secgw/ipsec.c +0
-0case RTE_SECURITY_ACTION_TYPE_CPU_CRYPTO : RTE_LOG(ERR, IPSEC, "CPU crypto is not supported bythe"
examples/ipsec-secgw/ipsec.c ipsec_inline_dequeue(ipsec_xform_fn xform_func,
Quoted 37 lines
examples/ipsec-secgw/ipsec.c +3
-3" legacy mode."); -rte_pktmbuf_free (pkts[i]); +free_pkt (pkts[i]);continue ;case RTE_SECURITY_ACTION_TYPE_NONE :@@ -575,7 +575,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx, if ((unlikely(ips->crypto.ses == NULL)) && create_lookaside_session(ipsec_ctx, sa, ips)) { -rte_pktmbuf_free (pkts[i]); +free_pkt (pkts[i]);continue ; }@@ -584,7 +584,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, structipsec_ctx *ipsec_ctx, ret =xform_func (pkts[i], sa, &priv->cop);if (unlikely (ret)) { -rte_pktmbuf_free (pkts[i]); +free_pkt (pkts[i]);continue ; }examples/ipsec-secgw/ipsec.c +1
-1break ;@@ -608,7 +608,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, structipsec_ctx *ipsec_ctx, ret =xform_func (pkts[i], sa, &priv->cop);if (unlikely (ret)) { -rte_pktmbuf_free (pkts[i]); +free_pkt (pkts[i]);continue ; }@@ -643,7 +643,7 @@ ipsec_inline_dequeue(ipsec_xform_fn xform_func,struct
ipsec_ctx*ipsec_ctx,examples/ipsec-secgw/ipsec.c ipsec_dequeue(ipsec_xform_fn xform_func, struct
Quoted 9 lines
examples/ipsec-secgw/ipsec.c +1
-1sa = priv->sa; ret =xform_func (pkt, sa, &priv->cop);if (unlikely (ret)) { -rte_pktmbuf_free (pkt); +free_pkt (pkt);continue ; } pkts[nb_pkts++] = pkt;@@ -690,13 +690,13 @@ ipsec_dequeue(ipsec_xform_fn xform_func, struct
ipsec_ctx*ipsec_ctx,examples/ipsec-secgw/ipsec.c ipsec_dequeue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
Quoted 9 lines
examples/ipsec-secgw/ipsec.c +1
-1RTE_SECURITY_ACTION_TYPE_NONE) { ret =xform_func (pkt, sa, cops[j]);if (unlikely (ret)) { -rte_pktmbuf_free (pkt); +free_pkt (pkt);continue ; } } else if (ipsec_get_action_type(sa) ==RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) {Quoted 12 lines
examples/ipsec-secgw/ipsec.c +1
-1if (cops[j]->status) { -rte_pktmbuf_free (pkt); +free_pkt (pkt);continue ; } }examples/ipsec-secgw/ipsec_process.c +0
-0examples/ipsec-secgw/ipsec_process.c line 18
Quoted 7 lines
examples/ipsec-secgw/ipsec_process.c +1
-0@@ -12,22 +12,13 @@#include <rte_mbuf.h> #include "ipsec.h" +#include "ipsec-secgw.h" #define SATP_OUT_IPV4 (t) \ ((((t) & RTE_IPSEC_SATP_MODE_MASK) ==
RTE_IPSEC_SATP_MODE_TRANS&& \examples/ipsec-secgw/ipsec_process.c line 20
Quoted 2 lines
examples/ipsec-secgw/ipsec_process.c +0
-0(((t) & RTE_IPSEC_SATP_IPV_MASK) == RTE_IPSEC_SATP_IPV4)) || \ ((t) & RTE_IPSEC_SATP_MODE_MASK) ==
RTE_IPSEC_SATP_MODE_TUNLV4)Quoted 13 lines
examples/ipsec-secgw/ipsec_process.c +0
-9-/* helper routine to free bulk of packets */ -static inline void -free_pkts(struct rte_mbuf *mb[], uint32_t n) -{ -uint32_t i; - -for (i =0 ; i != n; i++) -rte_pktmbuf_free (mb[i]); -} -/* helper routine to free bulk of crypto-ops and related packets */ static inline void free_cops (struct rte_crypto_op *cop[],uint32_t n) --2.7.4
Quoted 44 lines
Quoted 34 lines
Quoted 34 lines
examples/ipsec-secgw/ipsec-secgw.c +33
-0+#if (STATS_INTERVAL >0 ) + +/* Print out statistics on packet distribution */ static void +print_stats (void ) +{ +uint64_t total_packets_dropped, total_packets_tx, total_packets_rx; +unsigned int coreid; +float burst_percent; + + total_packets_dropped =0 ; + total_packets_tx =0 ; + total_packets_rx =0 ; + +const char clr[] = {27 ,'[' ,'2' ,'J' ,'\0' }; + +/* Clear screen and move to top left */ +printf ("%s" , clr); + +printf ("\nCore statistics ====================================" ); + +for (coreid =0 ; coreid <RTE_MAX_LCORE ; coreid++) { +/* skip disabled cores */ +if (rte_lcore_is_enabled (coreid) ==0 ) +continue ; + burst_percent = (float )(core_statistics[coreid].burst_rx *100 )/ + core_statistics[coreid].rx; + printf("\nStatistics for core %u ------------------------------" + "\nPackets received: %20"PRIu64 + "\nPackets sent: %24"PRIu64 + "\nPackets dropped: %21"PRIu64 + "\nBurst percent: %23.2f", + coreid, + core_statistics[coreid].rx,Quoted 9 lines
examples/ipsec-secgw/ipsec-secgw.c adjust_ipv6_pktlen(struct rte_mbuf *m, const struct rte_ipv6_hdr *iph, · line 327
Quoted 3 lines
examples/ipsec-secgw/ipsec-secgw.c +3
-0+ core_statistics[coreid].tx, + core_statistics[coreid].dropped, + burst_percent);As one more comment:
Can we also collect number of calls and display average pkt/call (for both rx and tx)?
Quoted 1 line
[Anoob] Can you describe which "call" you meant? We can capture more logs if required.
I meant something like that:
+static inline void
+core_stats_update_rx(int n)
+{
+#if (STATS_INTERVAL > 0)
+ int lcore_id = rte_lcore_id();
+ core_statistics[lcore_id].rx += n;
+ core_statistics[lcore_id].rx_call++;
+ if (n == MAX_PKT_BURST)
+ core_statistics[lcore_id].burst_rx += n;
+#else
+ RTE_SET_USED(n);
+#endif /* STATS_INTERVAL */
+}
And then in print_stats():
"\nPackets received: %20"PRIu64
",RX calls: %" PRIu64
"RX packets/call: %.2f"
core_statistics[coreid].rx, core_statistics[coreid].rx_call, (double)core_statistics[coreid].rx / core_statistics[coreid].rx_call
Quoted 441 lines
Quoted 441 lines
Quoted 4 lines
examples/ipsec-secgw/ipsec-secgw.c +3
-0+ + total_packets_dropped += core_statistics[coreid].dropped; + total_packets_tx += core_statistics[coreid].tx;examples/ipsec-secgw/ipsec-secgw.c adjust_ipv6_pktlen(struct rte_mbuf *m, const struct rte_ipv6_hdr *iph, · line 343
Quoted 10 lines
examples/ipsec-secgw/ipsec-secgw.c +10
-0+ total_packets_rx += core_statistics[coreid].rx; + } +printf ("\nAggregate statistics ===============================" +"\nTotal packets received: %14" PRIu64 +"\nTotal packets sent: %18" PRIu64 +"\nTotal packets dropped: %15" PRIu64, + total_packets_rx, + total_packets_tx, + total_packets_dropped); +printf("\n====================================================\n");
examples/ipsec-secgw/ipsec-secgw.c prepare_one_packet(struct rte_mbuf *pkt, struct · line 388
Quoted 9 lines
examples/ipsec-secgw/ipsec-secgw.c +3
-0+} +#endif /* STATS_INTERVAL */ +static inline void prepare_one_packet (struct rte_mbuf *pkt,struct ipsec_traffic *t) {@@ -333,7 +386,7 @@ prepare_one_packet(struct rte_mbuf *pkt, struct ipsec_traffic *t)/* drop packet when IPv6 header exceeds first segment length */
examples/ipsec-secgw/ipsec-secgw.c prepare_one_packet(struct rte_mbuf *pkt, struct
Quoted 7 lines
examples/ipsec-secgw/ipsec-secgw.c +1
-1if (unlikely (l3len > pkt->data_len)) { -rte_pktmbuf_free (pkt); +free_pkt (pkt);return ; }@@ -350,7 +403,7 @@ prepare_one_packet(struct rte_mbuf *pkt, struct
ipsec_traffic*t)examples/ipsec-secgw/ipsec-secgw.c send_burst(struct lcore_conf *qconf, uint16_t n,
Quoted 9 lines
examples/ipsec-secgw/ipsec-secgw.c +1
-1/* Unknown/Unsupported type, drop the packet */ RTE_LOG (ERR ,IPSEC ,"Unsupported packet type 0x%x\n" ,rte_be_to_cpu_16 (eth->ether_type)); -rte_pktmbuf_free (pkt); +free_pkt (pkt);return ; }@@ -477,9 +530,12 @@ send_burst(struct lcore_conf *qconf, uint16_t n,
uint16_tport)examples/ipsec-secgw/ipsec-secgw.c send_fragment_packet(struct lcore_conf *qconf,
Quoted 14 lines
examples/ipsec-secgw/ipsec-secgw.c +4
-1prepare_tx_burst (m_table, n, port, qconf); ret =rte_eth_tx_burst (port, queueid, m_table, n); + +core_stats_update_tx (ret); +if (unlikely (ret < n)) {do { -rte_pktmbuf_free (m_table[ret]); +free_pkt (m_table[ret]); }while (++ret < n); }@@ -525,7 +581,7 @@ send_fragment_packet(struct lcore_conf *qconf,struct
rte_mbuf*m,examples/ipsec-secgw/ipsec-secgw.c send_single_packet(struct rte_mbuf *m, uint16_t
Quoted 9 lines
examples/ipsec-secgw/ipsec-secgw.c +1
-1"error code: %d\n" , __func__, m->pkt_len, rte_errno); -rte_pktmbuf_free (m); +free_pkt (m);return len; }@@ -550,7 +606,7 @@ send_single_packet(struct rte_mbuf *m, uint16_tport,
uint8_tproto)examples/ipsec-secgw/ipsec-secgw.c split46_traffic(struct ipsec_traffic *trf, struct rte_mbuf
Quoted 32 lines
examples/ipsec-secgw/ipsec-secgw.c +1
-1}else if (frag_tbl_sz >0 ) len =send_fragment_packet (qconf, m, port, proto);else -rte_pktmbuf_free (m); +free_pkt (m);/* enough pkts to be sent */ if (unlikely(len == MAX_PKT_BURST)) { @@ -584,19 +640,19 @@inbound_sp_sa(struct
sp_ctx*sp, structsa_ctx*sa, structtraffic_type*ip,examples/ipsec-secgw/ipsec-secgw.c +3
-3continue ; }if (res ==DISCARD ) { -rte_pktmbuf_free (m); +free_pkt (m);continue ; }/* Only check SPI match for processed IPSec packets */ if (i < lim && ((m->ol_flags &PKT_RX_SEC_OFFLOAD ) ==0 )) { -rte_pktmbuf_free (m); +free_pkt (m);continue ; } sa_idx = res -1 ;if (!inbound_sa_check (sa, m, sa_idx)) { -rte_pktmbuf_free (m); +free_pkt (m);continue ; } ip->pkts[j++] = m;@@ -631,7 +687,7 @@ split46_traffic(struct ipsec_traffic *trf, struct rte_mbuf*mb[],
uint32_tnum)examples/ipsec-secgw/ipsec-secgw.c process_pkts_inbound_nosp(struct ipsec_ctx
Quoted 28 lines
examples/ipsec-secgw/ipsec-secgw.c +3
-4offsetof(struct ip6_hdr, ip6_nxt)); n6++; }else -rte_pktmbuf_free (m); +free_pkt (m); } trf->ip4.num = n4;@@ -683,7 +739,7 @@ outbound_sp(struct sp_ctx *sp, struct traffic_type *ip, m = ip->pkts[i]; sa_idx = ip->res[i] -1 ;if (ip->res[i] ==DISCARD ) -rte_pktmbuf_free (m); +free_pkt (m);else if (ip->res[i] ==BYPASS ) ip->pkts[j++] = m;else {@@ -702,8 +758,7 @@ process_pkts_outbound(struct ipsec_ctx *ipsec_ctx,uint16_t idx, nb_pkts_out, i;/* Drop any IPsec traffic from protected ports */ -for (i =0 ; i < traffic->ipsec.num; i++) -rte_pktmbuf_free (traffic->ipsec.pkts[i]); +free_pkts (traffic->ipsec.pkts, traffic->ipsec.num); traffic->ipsec.num =0 ;@@ -743,14 +798,12 @@ process_pkts_inbound_nosp(struct ipsec_ctx
*ipsec_ctx,examples/ipsec-secgw/ipsec-secgw.c process_pkts_outbound_nosp(struct ipsec_ctx
Quoted 17 lines
examples/ipsec-secgw/ipsec-secgw.c +2
-4uint32_t nb_pkts_in, i, idx;/* Drop any IPv4 traffic from unprotected ports */ -for (i =0 ; i < traffic->ip4.num; i++) -rte_pktmbuf_free (traffic->ip4.pkts[i]); +free_pkts (traffic->ip4.pkts, traffic->ip4.num); traffic->ip4.num =0 ;/* Drop any IPv6 traffic from unprotected ports */ -for (i =0 ; i < traffic->ip6.num; i++) -rte_pktmbuf_free (traffic->ip6.pkts[i]); +free_pkts (traffic->ip6.pkts, traffic->ip6.num); traffic->ip6.num =0 ;@@ -786,8 +839,7 @@ process_pkts_outbound_nosp(struct ipsec_ctx
*ipsec_ctx,examples/ipsec-secgw/ipsec-secgw.c route4_pkts(struct rt_ctx *rt_ctx, struct rte_mbuf
Quoted 10 lines
examples/ipsec-secgw/ipsec-secgw.c +1
-2struct ip *ip;/* Drop any IPsec traffic from protected ports */ -for (i =0 ; i < traffic->ipsec.num; i++) -rte_pktmbuf_free (traffic->ipsec.pkts[i]); +free_pkts (traffic->ipsec.pkts, traffic->ipsec.num); n =0 ;@@ -901,7 +953,7 @@ route4_pkts(struct rt_ctx *rt_ctx, struct rte_mbuf*pkts[],
uint8_tnb_pkts)examples/ipsec-secgw/ipsec-secgw.c route4_pkts(struct rt_ctx *rt_ctx, struct rte_mbuf *pkts[], uint8_t nb_pkts)
Quoted 8 lines
examples/ipsec-secgw/ipsec-secgw.c +1
-1}if ((pkt_hop &RTE_LPM_LOOKUP_SUCCESS ) ==0 ) { -rte_pktmbuf_free (pkts[i]); +free_pkt (pkts[i]);continue ; } send_single_packet(pkts[i], pkt_hop & 0xff, IPPROTO_IP); @@ -953,7
Quoted 1 line
+1005,7 @@ route6_pkts(struct
rt_ctx*rt_ctx, structrte_mbuf*pkts[],
uint8_tnb_pkts)examples/ipsec-secgw/ipsec-secgw.c route6_pkts(struct rt_ctx *rt_ctx, struct rte_mbuf *pkts[], uint8_t nb_pkts) · line 1011
Quoted 8 lines
examples/ipsec-secgw/ipsec-secgw.c +1
-1}if (pkt_hop ==-1 ) { -rte_pktmbuf_free (pkts[i]); +free_pkt (pkts[i]);continue ; }send_single_packet (pkts[i], pkt_hop &0xff ,IPPROTO_IPV6 );@@
Quoted 21 lines
-1099,6 +1151,10 @@
ipsec_poll_mode_worker(void)examples/ipsec-secgw/ipsec-secgw.c +10
-0const uint64_t drain_tsc = (rte_get_tsc_hz () +US_PER_S -1 ) /US_PER_S *BURST_TX_DRAIN_US ;struct lcore_rx_queue *rxql; +#if (STATS_INTERVAL >0 ) +const uint64_t timer_period =STATS_INTERVAL *rte_get_timer_hz (); +uint64_t timer_tsc =0 ; +#endif /* STATS_INTERVAL */ prev_tsc =0 ; lcore_id =rte_lcore_id ();@@ -1159,6 +1215,19 @@ ipsec_poll_mode_worker(void)drain_tx_buffers (qconf);drain_crypto_buffers (qconf); prev_tsc = cur_tsc; +#if (STATS_INTERVAL >0 ) +if (lcore_id ==rte_get_master_lcore ()) { +/* advance the timer */ + timer_tsc += diff_tsc; + +/* if timer has reached its timeout */ examples/ipsec-secgw/ipsec-secgw.c ipsec_poll_mode_worker(void) · line 1233
Quoted 10 lines
examples/ipsec-secgw/ipsec-secgw.c +7
-0+if (unlikely (timer_tsc >= timer_period)) { +print_stats (); +/* reset the timer */ + timer_tsc =0 ; + } + } +#endif /* STATS_INTERVAL */ } for (i = 0; i < qconf->nb_rx_queue; ++i) { @@ -1169,8 +1238,10@@
examples/ipsec-secgw/ipsec.c ipsec_enqueue(ipsec_xform_fn xform_func, struct
Quoted 131 lines
ipsec_poll_mode_worker(void)examples/ipsec-secgw/ipsec-secgw.c +3
-1nb_rx =rte_eth_rx_burst (portid, queueid, pkts,MAX_PKT_BURST ); -if (nb_rx >0 ) +if (nb_rx >0 ) { +core_stats_update_rx (nb_rx);process_pkts (qconf, pkts, nb_rx, portid); + }/* dequeue and process completed crypto-ops */ if (is_unprotected_port (portid))examples/ipsec-secgw/ipsec-secgw.h +13
-0@@ -6,6 +6,8 @@#include <stdbool.h> +#define STATS_INTERVAL 0 +#define NB_SOCKETS 4#define MAX_PKT_BURST 32@@ -69,6 +71,17 @@ struct ethaddr_info {uint64_t src, dst; }; +#if (STATS_INTERVAL >0 ) +struct ipsec_core_statistics { +uint64_t tx; +uint64_t rx; +uint64_t dropped; +uint64_t burst_rx; +} __rte_cache_aligned; + +struct ipsec_core_statistics core_statistics[RTE_MAX_LCORE ];#endif +/* STATS_INTERVAL */ +extern struct ethaddr_info ethaddr_tbl[RTE_MAX_ETHPORTS ]; /* Port mask to identify the unprotected ports */ @@ -85,4 +98,59 @@
is_unprotected_port(uint16_t port_id)examples/ipsec-secgw/ipsec-secgw.h +49
-0return unprotected_port_mask & (1 << port_id); } +static inline void +core_stats_update_rx (int n) +{ +#if (STATS_INTERVAL >0 ) +int lcore_id =rte_lcore_id (); + core_statistics[lcore_id].rx += n; +if (n ==MAX_PKT_BURST ) + core_statistics[lcore_id].burst_rx += n;#else +RTE_SET_USED (n); +#endif /* STATS_INTERVAL */ +} + +static inline void +core_stats_update_tx (int n) +{ +#if (STATS_INTERVAL >0 ) +int lcore_id =rte_lcore_id (); + core_statistics[lcore_id].tx += n; +#else +RTE_SET_USED (n); +#endif /* STATS_INTERVAL */ +} + +static inline void +core_stats_update_drop (int n) +{ +#if (STATS_INTERVAL >0 ) +int lcore_id =rte_lcore_id (); + core_statistics[lcore_id].dropped += n;#else +RTE_SET_USED (n); +#endif /* STATS_INTERVAL */ +} + +/* helper routine to free bulk of packets */ static inline void +free_pkts (struct rte_mbuf *mb[],uint32_t n) { +uint32_t i; + +for (i =0 ; i != n; i++) +rte_pktmbuf_free (mb[i]); + +core_stats_update_drop (n); +} + +/* helper routine to free single packet */ static inline void +free_pkt (struct rte_mbuf *mb) { +rte_pktmbuf_free (mb); +core_stats_update_drop (1 ); +} +#endif /* _IPSEC_SECGW_H_ */ (quoted diff) +0
-0
b/examples/ipsec-secgw/ipsec.cindex bf88d80..351f1f1 100644examples/ipsec-secgw/ipsec.c +3
-3@@ -500,7 +500,7 @@ enqueue_cop_burst(struct cdev_qp *cqp) cqp->id, cqp->qp, ret, len);/* drop packets that we fail to enqueue */ for (i = ret; i < len; i++) -rte_pktmbuf_free (cqp->buf[i]->sym->m_src); +free_pkt (cqp->buf[i]->sym->m_src); } cqp->in_flight += ret; cqp->len =0 ;@@ -528,7 +528,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx, for (i = 0; i < nb_pkts; i++) {if (unlikely (sas[i] ==NULL )) { -rte_pktmbuf_free (pkts[i]); +free_pkt (pkts[i]);continue ; }@@ -549,7 +549,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx, if ((unlikely(ips->security.ses == NULL)) && create_lookaside_session(ipsec_ctx, sa, ips)) { -rte_pktmbuf_free (pkts[i]); +free_pkt (pkts[i]);continue ; }@@ -563,7 +563,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct
ipsec_ctx*ipsec_ctx,examples/ipsec-secgw/ipsec.c ipsec_enqueue(ipsec_xform_fn xform_func, struct · line 564
Quoted 2 lines
examples/ipsec-secgw/ipsec.c +0
-0case RTE_SECURITY_ACTION_TYPE_CPU_CRYPTO : RTE_LOG(ERR, IPSEC, "CPU crypto is not supported bythe"
examples/ipsec-secgw/ipsec.c ipsec_inline_dequeue(ipsec_xform_fn xform_func,
Quoted 37 lines
examples/ipsec-secgw/ipsec.c +3
-3" legacy mode."); -rte_pktmbuf_free (pkts[i]); +free_pkt (pkts[i]);continue ;case RTE_SECURITY_ACTION_TYPE_NONE :@@ -575,7 +575,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx, if ((unlikely(ips->crypto.ses == NULL)) && create_lookaside_session(ipsec_ctx, sa, ips)) { -rte_pktmbuf_free (pkts[i]); +free_pkt (pkts[i]);continue ; }@@ -584,7 +584,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, structipsec_ctx *ipsec_ctx, ret =xform_func (pkts[i], sa, &priv->cop);if (unlikely (ret)) { -rte_pktmbuf_free (pkts[i]); +free_pkt (pkts[i]);continue ; }examples/ipsec-secgw/ipsec.c +1
-1break ;@@ -608,7 +608,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, structipsec_ctx *ipsec_ctx, ret =xform_func (pkts[i], sa, &priv->cop);if (unlikely (ret)) { -rte_pktmbuf_free (pkts[i]); +free_pkt (pkts[i]);continue ; }@@ -643,7 +643,7 @@ ipsec_inline_dequeue(ipsec_xform_fn xform_func,struct
ipsec_ctx*ipsec_ctx,examples/ipsec-secgw/ipsec.c ipsec_dequeue(ipsec_xform_fn xform_func, struct
Quoted 9 lines
examples/ipsec-secgw/ipsec.c +1
-1sa = priv->sa; ret =xform_func (pkt, sa, &priv->cop);if (unlikely (ret)) { -rte_pktmbuf_free (pkt); +free_pkt (pkt);continue ; } pkts[nb_pkts++] = pkt;@@ -690,13 +690,13 @@ ipsec_dequeue(ipsec_xform_fn xform_func, struct
ipsec_ctx*ipsec_ctx,examples/ipsec-secgw/ipsec.c ipsec_dequeue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
Quoted 9 lines
examples/ipsec-secgw/ipsec.c +1
-1RTE_SECURITY_ACTION_TYPE_NONE) { ret =xform_func (pkt, sa, cops[j]);if (unlikely (ret)) { -rte_pktmbuf_free (pkt); +free_pkt (pkt);continue ; } } else if (ipsec_get_action_type(sa) ==RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) {Quoted 12 lines
examples/ipsec-secgw/ipsec.c +1
-1if (cops[j]->status) { -rte_pktmbuf_free (pkt); +free_pkt (pkt);continue ; } }examples/ipsec-secgw/ipsec_process.c +0
-0examples/ipsec-secgw/ipsec_process.c line 18
Quoted 7 lines
examples/ipsec-secgw/ipsec_process.c +1
-0@@ -12,22 +12,13 @@#include <rte_mbuf.h> #include "ipsec.h" +#include "ipsec-secgw.h" #define SATP_OUT_IPV4 (t) \ ((((t) & RTE_IPSEC_SATP_MODE_MASK) ==
RTE_IPSEC_SATP_MODE_TRANS&& \examples/ipsec-secgw/ipsec_process.c line 20
Quoted 2 lines
examples/ipsec-secgw/ipsec_process.c +0
-0(((t) & RTE_IPSEC_SATP_IPV_MASK) == RTE_IPSEC_SATP_IPV4)) || \ ((t) & RTE_IPSEC_SATP_MODE_MASK) ==
RTE_IPSEC_SATP_MODE_TUNLV4)Quoted 13 lines
examples/ipsec-secgw/ipsec_process.c +0
-9-/* helper routine to free bulk of packets */ -static inline void -free_pkts(struct rte_mbuf *mb[], uint32_t n) -{ -uint32_t i; - -for (i =0 ; i != n; i++) -rte_pktmbuf_free (mb[i]); -} -/* helper routine to free bulk of crypto-ops and related packets */ static inline void free_cops (struct rte_crypto_op *cop[],uint32_t n) --2.7.4
PATCH v4 1/1 examples/ipsec-secgw: add per core packet stats
In reply to Anoob JosephAdding per core packet handling stats to analyze traffic distribution when multiple cores are engaged.
Since aggregating the packet stats across cores would affect performance, keeping the feature disabled using compile time flags.
---
v4:
* Moved print stats routine to control thread
* Added stats for rx/tx pkts per call
* Replaced free_pkt(m) with free_pkts(&m, 1)
v3:
* Added wrapper functions for updating rx, tx & dropped counts
* Updated free_pkts() so that stats is updated internally
* Introduced similar free_pkt() function which updates stats and frees the packet
* Moved all inline functions and macros to ipsec-secgw.h
* Made STATS_INTERVAL macro to control the interval of the stats update. STATS_INTERVAL = 0 would disable the feature.
v2:
* Added lookup failure cases to drop count
examples/ipsec-secgw/ipsec-secgw.c | 114 ++++++++++++++++++++++++++++------- examples/ipsec-secgw/ipsec-secgw.h | 66 ++++++++++++++++++++ examples/ipsec-secgw/ipsec.c | 20 +++--- examples/ipsec-secgw/ipsec_process.c | 11 +--- 4 files changed, 170 insertions(+), 41 deletions(-)
examples/ipsec-secgw/ipsec-secgw.c +93 -21
@@ -47,6 +47,7 @@#include <rte_eventdev.h> #include <rte_ip.h> #include <rte_ip_frag.h> +#include <rte_alarm.h> #include "event_helper.h" #include "ipsec.h"
@@ -287,6 +288,70 @@ adjust_ipv6_pktlen(struct rte_mbuf *m, const struct rte_ipv6_hdr *iph, } } +#if (STATS_INTERVAL >0 ) + +/* Print out statistics on packet distribution */ +static void +print_stats_cb(__rte_unused void *param) +{ +uint64_t total_packets_dropped, total_packets_tx, total_packets_rx; +float burst_percent, rx_per_call, tx_per_call; +unsigned int coreid; + + total_packets_dropped =0 ; + total_packets_tx =0 ; + total_packets_rx =0 ; + +const char clr[] = {27 ,'[' ,'2' ,'J' ,'\0' }; +const char topLeft[] = {27 ,'[' ,'1' ,';' ,'1' ,'H' ,'\0' }; + +/* Clear screen and move to top left */ +printf ("%s%s" , clr, topLeft); + +printf ("\nCore statistics ====================================" ); + +for (coreid =0 ; coreid <RTE_MAX_LCORE ; coreid++) { +/* skip disabled cores */ +if (rte_lcore_is_enabled (coreid) ==0 ) +continue ; + burst_percent = (float )(core_statistics[coreid].burst_rx *100 )/ + core_statistics[coreid].rx; + rx_per_call = (float )(core_statistics[coreid].rx)/ + core_statistics[coreid].rx_call; + tx_per_call = (float )(core_statistics[coreid].tx)/ + core_statistics[coreid].tx_call; +printf ("\nStatistics for core %u ------------------------------" +"\nPackets received: %20" PRIu64 +"\nPackets sent: %24" PRIu64 +"\nPackets dropped: %21" PRIu64 +"\nBurst percent: %23.2f" +"\nPackets per Rx call: %17.2f" +"\nPackets per Tx call: %17.2f" , + coreid, + core_statistics[coreid].rx, + core_statistics[coreid].tx, + core_statistics[coreid].dropped, + burst_percent, + rx_per_call, + tx_per_call); + + total_packets_dropped += core_statistics[coreid].dropped; + total_packets_tx += core_statistics[coreid].tx; + total_packets_rx += core_statistics[coreid].rx; + } +printf ("\nAggregate statistics ===============================" +"\nTotal packets received: %14" PRIu64 +"\nTotal packets sent: %18" PRIu64 +"\nTotal packets dropped: %15" PRIu64, + total_packets_rx, + total_packets_tx, + total_packets_dropped); +printf ("\n====================================================\n" ); + +rte_eal_alarm_set (STATS_INTERVAL *US_PER_S , print_stats_cb,NULL ); +} +#endif /* STATS_INTERVAL */ +static inline void prepare_one_packet (struct rte_mbuf *pkt,struct ipsec_traffic *t) {
@@ -332,7 +397,7 @@ prepare_one_packet(struct rte_mbuf *pkt, struct ipsec_traffic *t)/* drop packet when IPv6 header exceeds first segment length */ if (unlikely (l3len > pkt->data_len)) { -rte_pktmbuf_free (pkt); +free_pkts (&pkt,1 );return ; }
@@ -349,7 +414,7 @@ prepare_one_packet(struct rte_mbuf *pkt, struct ipsec_traffic *t)/* Unknown/Unsupported type, drop the packet */ RTE_LOG (ERR ,IPSEC ,"Unsupported packet type 0x%x\n" ,rte_be_to_cpu_16 (eth->ether_type)); -rte_pktmbuf_free (pkt); +free_pkts (&pkt,1 );return ; }
@@ -476,9 +541,12 @@ send_burst(struct lcore_conf *qconf, uint16_t n, uint16_t port)prepare_tx_burst (m_table, n, port, qconf); ret =rte_eth_tx_burst (port, queueid, m_table, n); + +core_stats_update_tx (ret); +if (unlikely (ret < n)) {do { -rte_pktmbuf_free (m_table[ret]); +free_pkts (&m_table[ret],1 ); }while (++ret < n); }
@@ -524,7 +592,7 @@ send_fragment_packet(struct lcore_conf *qconf, struct rte_mbuf *m,"error code: %d\n" , __func__, m->pkt_len, rte_errno); -rte_pktmbuf_free (m); +free_pkts (&m,1 );return len; }
@@ -549,7 +617,7 @@ send_single_packet(struct rte_mbuf *m, uint16_t port, uint8_t proto) }else if (frag_tbl_sz >0 ) len =send_fragment_packet (qconf, m, port, proto);else -rte_pktmbuf_free (m); +free_pkts (&m,1 );/* enough pkts to be sent */ if (unlikely (len ==MAX_PKT_BURST )) {
@@ -583,19 +651,19 @@ inbound_sp_sa(struct sp_ctx *sp, struct sa_ctx *sa, struct traffic_type *ip,continue ; }if (res ==DISCARD ) { -rte_pktmbuf_free (m); +free_pkts (&m,1 );continue ; }/* Only check SPI match for processed IPSec packets */ if (i < lim && ((m->ol_flags &PKT_RX_SEC_OFFLOAD ) ==0 )) { -rte_pktmbuf_free (m); +free_pkts (&m,1 );continue ; } sa_idx = res -1 ;if (!inbound_sa_check (sa, m, sa_idx)) { -rte_pktmbuf_free (m); +free_pkts (&m,1 );continue ; } ip->pkts[j++] = m;
@@ -630,7 +698,7 @@ split46_traffic(struct ipsec_traffic *trf, struct rte_mbuf *mb[], uint32_t num) offsetof(struct ip6_hdr, ip6_nxt)); n6++; }else -rte_pktmbuf_free (m); +free_pkts (&m,1 ); } trf->ip4.num = n4;
@@ -682,7 +750,7 @@ outbound_sp(struct sp_ctx *sp, struct traffic_type *ip, m = ip->pkts[i]; sa_idx = ip->res[i] -1 ;if (ip->res[i] ==DISCARD ) -rte_pktmbuf_free (m); +free_pkts (&m,1 );else if (ip->res[i] ==BYPASS ) ip->pkts[j++] = m;else {
@@ -701,8 +769,7 @@ process_pkts_outbound(struct ipsec_ctx *ipsec_ctx,uint16_t idx, nb_pkts_out, i;/* Drop any IPsec traffic from protected ports */ -for (i =0 ; i < traffic->ipsec.num; i++) -rte_pktmbuf_free (traffic->ipsec.pkts[i]); +free_pkts (traffic->ipsec.pkts, traffic->ipsec.num); traffic->ipsec.num =0 ;
@@ -742,14 +809,12 @@ process_pkts_inbound_nosp(struct ipsec_ctx *ipsec_ctx,uint32_t nb_pkts_in, i, idx;/* Drop any IPv4 traffic from unprotected ports */ -for (i =0 ; i < traffic->ip4.num; i++) -rte_pktmbuf_free (traffic->ip4.pkts[i]); +free_pkts (traffic->ip4.pkts, traffic->ip4.num); traffic->ip4.num =0 ;/* Drop any IPv6 traffic from unprotected ports */ -for (i =0 ; i < traffic->ip6.num; i++) -rte_pktmbuf_free (traffic->ip6.pkts[i]); +free_pkts (traffic->ip6.pkts, traffic->ip6.num); traffic->ip6.num =0 ;
@@ -785,8 +850,7 @@ process_pkts_outbound_nosp(struct ipsec_ctx *ipsec_ctx,struct ip *ip;/* Drop any IPsec traffic from protected ports */ -for (i =0 ; i < traffic->ipsec.num; i++) -rte_pktmbuf_free (traffic->ipsec.pkts[i]); +free_pkts (traffic->ipsec.pkts, traffic->ipsec.num); n =0 ;
@@ -900,7 +964,7 @@ route4_pkts(struct rt_ctx *rt_ctx, struct rte_mbuf *pkts[], uint8_t nb_pkts) }if ((pkt_hop &RTE_LPM_LOOKUP_SUCCESS ) ==0 ) { -rte_pktmbuf_free (pkts[i]); +free_pkts (&pkts[i],1 );continue ; }send_single_packet (pkts[i], pkt_hop &0xff ,IPPROTO_IP );
@@ -952,7 +1016,7 @@ route6_pkts(struct rt_ctx *rt_ctx, struct rte_mbuf *pkts[], uint8_t nb_pkts) }if (pkt_hop ==-1 ) { -rte_pktmbuf_free (pkts[i]); +free_pkts (&pkts[i],1 );continue ; }send_single_packet (pkts[i], pkt_hop &0xff ,IPPROTO_IPV6 );
@@ -1168,8 +1232,10 @@ ipsec_poll_mode_worker(void) nb_rx =rte_eth_rx_burst (portid, queueid, pkts,MAX_PKT_BURST ); -if (nb_rx >0 ) +if (nb_rx >0 ) { +core_stats_update_rx (nb_rx);process_pkts (qconf, pkts, nb_rx, portid); + }/* dequeue and process completed crypto-ops */ if (is_unprotected_port (portid))
@@ -2916,6 +2982,12 @@ main(int32_t argc, char **argv)check_all_ports_link_status (enabled_port_mask); +#if (STATS_INTERVAL >0 ) +rte_eal_alarm_set (STATS_INTERVAL *US_PER_S , print_stats_cb,NULL ); +#else +RTE_LOG (INFO ,IPSEC ,"Stats display disabled\n" ); +#endif /* STATS_INTERVAL */ +/* launch per-lcore init on every lcore */ rte_eal_mp_remote_launch (ipsec_launch_one_lcore, eh_conf,CALL_MASTER );RTE_LCORE_FOREACH_SLAVE (lcore_id) {
examples/ipsec-secgw/ipsec-secgw.h +66 -0
82 lines not on this page · raw message
examples/ipsec-secgw/ipsec.c +10 -10
@@ -500,7 +500,7 @@ enqueue_cop_burst(struct cdev_qp *cqp) cqp->id, cqp->qp, ret, len);/* drop packets that we fail to enqueue */ for (i = ret; i < len; i++) -rte_pktmbuf_free (cqp->buf[i]->sym->m_src); +free_pkts (&cqp->buf[i]->sym->m_src,1 ); } cqp->in_flight += ret; cqp->len =0 ;
@@ -528,7 +528,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,for (i =0 ; i < nb_pkts; i++) {if (unlikely (sas[i] ==NULL )) { -rte_pktmbuf_free (pkts[i]); +free_pkts (&pkts[i],1 );continue ; }
@@ -549,7 +549,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,if ((unlikely (ips->security.ses ==NULL )) &&create_lookaside_session (ipsec_ctx, sa, ips)) { -rte_pktmbuf_free (pkts[i]); +free_pkts (&pkts[i],1 );continue ; }
@@ -563,7 +563,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,case RTE_SECURITY_ACTION_TYPE_CPU_CRYPTO :RTE_LOG (ERR ,IPSEC ,"CPU crypto is not supported by the" " legacy mode." ); -rte_pktmbuf_free (pkts[i]); +free_pkts (&pkts[i],1 );continue ;case RTE_SECURITY_ACTION_TYPE_NONE :
@@ -575,7 +575,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,if ((unlikely (ips->crypto.ses ==NULL )) &&create_lookaside_session (ipsec_ctx, sa, ips)) { -rte_pktmbuf_free (pkts[i]); +free_pkts (&pkts[i],1 );continue ; }
@@ -584,7 +584,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx, ret =xform_func (pkts[i], sa, &priv->cop);if (unlikely (ret)) { -rte_pktmbuf_free (pkts[i]); +free_pkts (&pkts[i],1 );continue ; }break ;
@@ -608,7 +608,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx, ret =xform_func (pkts[i], sa, &priv->cop);if (unlikely (ret)) { -rte_pktmbuf_free (pkts[i]); +free_pkts (&pkts[i],1 );continue ; }
@@ -643,7 +643,7 @@ ipsec_inline_dequeue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx, sa = priv->sa; ret =xform_func (pkt, sa, &priv->cop);if (unlikely (ret)) { -rte_pktmbuf_free (pkt); +free_pkts (&pkt,1 );continue ; } pkts[nb_pkts++] = pkt;
@@ -690,13 +690,13 @@ ipsec_dequeue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx, RTE_SECURITY_ACTION_TYPE_NONE) { ret =xform_func (pkt, sa, cops[j]);if (unlikely (ret)) { -rte_pktmbuf_free (pkt); +free_pkts (&pkt,1 );continue ; } } else if (ipsec_get_action_type(sa) ==RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL ) {if (cops[j]->status) { -rte_pktmbuf_free (pkt); +free_pkts (&pkt,1 );continue ; } }
examples/ipsec-secgw/ipsec_process.c +1 -10
@@ -12,22 +12,13 @@#include <rte_mbuf.h> #include "ipsec.h" +#include "ipsec-secgw.h" #define SATP_OUT_IPV4 (t) \ ((((t) & RTE_IPSEC_SATP_MODE_MASK) == RTE_IPSEC_SATP_MODE_TRANS && \ (((t) & RTE_IPSEC_SATP_IPV_MASK) == RTE_IPSEC_SATP_IPV4)) || \ ((t) & RTE_IPSEC_SATP_MODE_MASK) == RTE_IPSEC_SATP_MODE_TUNLV4) -/* helper routine to free bulk of packets */ -static inline void -free_pkts (struct rte_mbuf *mb[],uint32_t n) -{ -uint32_t i; - -for (i =0 ; i != n; i++) -rte_pktmbuf_free (mb[i]); -} -/* helper routine to free bulk of crypto-ops and related packets */ static inline void free_cops (struct rte_crypto_op *cop[],uint32_t n)
Signature · 1 line
2.7.4
Quoted 67 lines
Adding per core packet handling stats to analyze traffic distribution when multiple cores are engaged.
Since aggregating the packet stats across cores would affect performance, keeping the feature disabled using compile time flags.
Signed-off-by Anoob Joseph <[email protected]>---
v4:
* Moved print stats routine to control thread
* Added stats for rx/tx pkts per call
* Replacedfree_pkt(m)withfree_pkts(&m, 1)v3:
* Added wrapper functions for updating rx, tx & dropped counts
* Updatedfree_pkts()so that stats is updated internally
* Introduced similarfree_pkt()function which updates stats and frees the packet
* Moved all inline functions and macros toipsec-secgw.h
* MadeSTATS_INTERVALmacro to control the interval of the stats update.STATS_INTERVAL= 0 would disable the feature.v2:
* Added lookup failure cases to drop countexamples/ipsec-secgw/ipsec-secgw.c | 114 ++++++++++++++++++++++++++++------- examples/ipsec-secgw/ipsec-secgw.h | 66 ++++++++++++++++++++ examples/ipsec-secgw/ipsec.c | 20 +++--- examples/ipsec-secgw/ipsec_process.c | 11 +--- 4 files changed, 170 insertions(+), 41 deletions(-)examples/ipsec-secgw/ipsec-secgw.c +21
-0@@ -47,6 +47,7 @@#include <rte_eventdev.h> #include <rte_ip.h> #include <rte_ip_frag.h> +#include <rte_alarm.h> #include "event_helper.h" #include "ipsec.h" @@ -287,6 +288,70 @@ adjust_ipv6_pktlen(struct rte_mbuf *m, const struct rte_ipv6_hdr *iph, } } +#if (STATS_INTERVAL >0 ) + +/* Print out statistics on packet distribution */ +static void +print_stats_cb(__rte_unused void *param) +{ +uint64_t total_packets_dropped, total_packets_tx, total_packets_rx; +float burst_percent, rx_per_call, tx_per_call; +unsigned int coreid; + + total_packets_dropped =0 ; + total_packets_tx =0 ; + total_packets_rx =0 ; + +const char clr[] = {27 ,'[' ,'2' ,'J' ,'\0' }; +const char topLeft[] = {27 ,'[' ,'1' ,';' ,'1' ,'H' ,'\0' }; + +/* Clear screen and move to top left */ +printf ("%s%s" , clr, topLeft); +
examples/ipsec-secgw/ipsec-secgw.c adjust_ipv6_pktlen(struct rte_mbuf *m, const struct rte_ipv6_hdr *iph, · line 320
Quoted 10 lines
examples/ipsec-secgw/ipsec-secgw.c +10
-0+printf ("\nCore statistics ====================================" ); + +for (coreid =0 ; coreid <RTE_MAX_LCORE ; coreid++) { +/* skip disabled cores */ +if (rte_lcore_is_enabled (coreid) ==0 ) +continue ; + burst_percent = (float )(core_statistics[coreid].burst_rx *100 )/ + core_statistics[coreid].rx; + rx_per_call = (float )(core_statistics[coreid].rx)/ + core_statistics[coreid].rx_call;
As a nit - probably better to use double, and check that divisors are not zero. Apart from that - LGTM.
Quoted 34 lines
examples/ipsec-secgw/ipsec-secgw.c +34
-0+ tx_per_call = (float )(core_statistics[coreid].tx)/ + core_statistics[coreid].tx_call; +printf ("\nStatistics for core %u ------------------------------" +"\nPackets received: %20" PRIu64 +"\nPackets sent: %24" PRIu64 +"\nPackets dropped: %21" PRIu64 +"\nBurst percent: %23.2f" +"\nPackets per Rx call: %17.2f" +"\nPackets per Tx call: %17.2f" , + coreid, + core_statistics[coreid].rx, + core_statistics[coreid].tx, + core_statistics[coreid].dropped, + burst_percent, + rx_per_call, + tx_per_call); + + total_packets_dropped += core_statistics[coreid].dropped; + total_packets_tx += core_statistics[coreid].tx; + total_packets_rx += core_statistics[coreid].rx; + } +printf ("\nAggregate statistics ===============================" +"\nTotal packets received: %14" PRIu64 +"\nTotal packets sent: %18" PRIu64 +"\nTotal packets dropped: %15" PRIu64, + total_packets_rx, + total_packets_tx, + total_packets_dropped); +printf ("\n====================================================\n" ); + +rte_eal_alarm_set (STATS_INTERVAL *US_PER_S , print_stats_cb,NULL ); +} +#endif /* STATS_INTERVAL */ +
Hi Anoob/Konstantin,
Quoted 9 lines
Quoted 9 lines
Adding per core packet handling stats to analyze traffic distribution when multiple cores are engaged.
Since aggregating the packet stats across cores would affect performance, keeping the feature disabled using compile time flags.
Signed-off-by Anoob Joseph <[email protected]>---
Since this patch got acked pretty late in the release cycle, we need to defer This patch for next release.
We cannot have these kind of patches in RC3.
Regards,
Akhil
Hi Akhil,
Quoted 11 lines
----------------------------------------------------------------------
Hi Anoob/Konstantin,Quoted 7 lines
Quoted 7 lines
Adding per core packet handling stats to analyze traffic distribution when multiple cores are engaged.
Since aggregating the packet stats across cores would affect performance, keeping the feature disabled using compile time flags.
Quoted 7 lines
Quoted 2 lines
Quoted 2 lines
Signed-off-by Anoob Joseph <[email protected]>---
Since this patch got acked pretty late in the release cycle, we need to defer This patch for next release.
We cannot have these kind of patches in RC3.
Not a problem. Do you want me to resubmit this patch for next release? Or will you be able to merge it early in the next cycle?
Thanks,
Anoob
Quoted 16 lines
Hi Akhil,
Quoted 14 lines
----------------------------------------------------------------------
Hi Anoob/Konstantin,Quoted 9 lines
Quoted 9 lines
Adding per core packet handling stats to analyze traffic distribution when multiple cores are engaged.
Since aggregating the packet stats across cores would affect performance, keeping the feature disabled using compile time flags.
Signed-off-by Anoob Joseph <[email protected]>---
Quoted 9 lines
Quoted 1 line
Since this patch got acked pretty late in the release cycle, we need to defer
This
Quoted 3 lines
patch for next release.
We cannot have these kind of patches in RC3.Not a problem. Do you want me to resubmit this patch for next release? Or will you be able to merge it early in the next cycle?
No need to send it again as of now. I will merge it early in the next cycle.
Quoted 24 lines
Quoted 24 lines
Hi Akhil,
Quoted 8 lines
----------------------------------------------------------------------
Hi Anoob/Konstantin,Quoted 4 lines
Quoted 4 lines
Adding per core packet handling stats to analyze traffic distribution when multiple cores are engaged.
Quoted 7 lines
Quoted 5 lines
Quoted 5 lines
Since aggregating the packet stats across cores would affect performance, keeping the feature disabled using compile time flags.
Signed-off-by Anoob Joseph <[email protected]>---
Since this patch got acked pretty late in the release cycle, we need to defer
This
Quoted 3 lines
patch for next release.
We cannot have these kind of patches in RC3.
Quoted 5 lines
Quoted 1 line
Not a problem. Do you want me to resubmit this patch for next release? Or
will
Quoted 2 lines
you be able to merge it early in the next cycle?
No need to send it again as of now. I will merge it early in the next cycle.
Applied to dpdk-next-crypto
Thanks.