net/pcap: reduce time for stopping device
PATCH 1/1 net/pcap: reduce time for stopping device
The pcap file will be synchronized to the disk when stopping the device. It takes a long time if the file is large that would cause the 'detach sync request' timeout when the device is closed under multi-process scenario.
This commit fixes the issue by performing synchronization in Tx path
--- drivers/net/pcap/pcap_ethdev.c | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-)
drivers/net/pcap/pcap_ethdev.c +16 -2
@@ -3,7 +3,7 @@* Copyright(c) 2014 6WIND S.A. * All rights reserved. */ - +#include <unistd.h> #include <time.h> #include <pcap.h>
@@ -38,6 +38,8 @@#define RTE_PMD_PCAP_MAX_QUEUES 16 +#define ETH_PCAP_SYNC_THRESHOLD 0x20000000 +static char errbuf[PCAP_ERRBUF_SIZE ];static struct timespec start_time;static uint64_t start_cycles;
@@ -47,6 +49,8 @@ static uint8_t iface_idx;static uint64_t timestamp_rx_dynflag;static int timestamp_dynfield_offset =-1 ; +RTE_DEFINE_PER_LCORE (uint64_t, _pcap_cached_bytes); +struct queue_stat {volatile unsigned long pkts;volatile unsigned long bytes;
@@ -144,6 +148,16 @@ static struct rte_eth_link pmd_link = {RTE_LOG_REGISTER_DEFAULT (eth_pcap_logtype,NOTICE ); +static inline void +pcap_dumper_data_sync (pcap_dumper_t *dumper,uint32_t bytes) +{ +RTE_PER_LCORE (_pcap_cached_bytes) += bytes; +if (unlikely (RTE_PER_LCORE (_pcap_cached_bytes) >ETH_PCAP_SYNC_THRESHOLD )) { +if (!fdatasync (fileno (pcap_dump_file (dumper)))) +RTE_PER_LCORE (_pcap_cached_bytes) =0 ; + } +} +static struct queue_missed_stat *queue_missed_stat_update (struct rte_eth_dev *dev,unsigned int qid) {
@@ -421,7 +435,7 @@ eth_pcap_tx_dumper(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts)* process stops and to make sure the pcap file is actually written, * we flush the pcap dumper within each burst. */ -pcap_dump_flush (dumper); +pcap_dumper_data_sync (dumper, tx_bytes); dumper_q->tx_stat.pkts += num_tx; dumper_q->tx_stat.bytes += tx_bytes; dumper_q->tx_stat.err_pkts += nb_pkts - num_tx;
Signature · 1 line
2.34.1
Quoted 64 lines · Yiding Zhou
The pcap file will be synchronized to the disk when stopping the device. It takes a long time if the file is large that would cause the 'detach sync request' timeout when the device is closed under multi-process scenario.
This commit fixes the issue by performing synchronization in Tx path
Fixes 4c173302c307 ("pcap: add new driver")Signed-off-by Yiding Zhou <[email protected]>--- drivers/net/pcap/pcap_ethdev.c | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-)drivers/net/pcap/pcap_ethdev.c +15
-1@@ -3,7 +3,7 @@* Copyright(c) 2014 6WIND S.A. * All rights reserved. */ - +#include <unistd.h> #include <time.h> #include <pcap.h> @@ -38,6 +38,8 @@#define RTE_PMD_PCAP_MAX_QUEUES 16 +#define ETH_PCAP_SYNC_THRESHOLD 0x20000000 +static char errbuf[PCAP_ERRBUF_SIZE ];static struct timespec start_time;static uint64_t start_cycles;@@ -47,6 +49,8 @@ static uint8_t iface_idx;static uint64_t timestamp_rx_dynflag;static int timestamp_dynfield_offset =-1 ; +RTE_DEFINE_PER_LCORE (uint64_t, _pcap_cached_bytes); +struct queue_stat {volatile unsigned long pkts;volatile unsigned long bytes;@@ -144,6 +148,16 @@ static struct rte_eth_link pmd_link = {RTE_LOG_REGISTER_DEFAULT (eth_pcap_logtype,NOTICE ); +static inline void +pcap_dumper_data_sync (pcap_dumper_t *dumper,uint32_t bytes) +{ +RTE_PER_LCORE (_pcap_cached_bytes) += bytes; +if (unlikely (RTE_PER_LCORE (_pcap_cached_bytes) >ETH_PCAP_SYNC_THRESHOLD )) { +if (!fdatasync (fileno (pcap_dump_file (dumper)))) +RTE_PER_LCORE (_pcap_cached_bytes) =0 ; + } +} +static struct queue_missed_stat *queue_missed_stat_update (struct rte_eth_dev *dev,unsigned int qid) {
drivers/net/pcap/pcap_ethdev.c eth_pcap_tx_dumper(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts) · line 438
Quoted 5 lines
drivers/net/pcap/pcap_ethdev.c +1
-1@@ -421,7 +435,7 @@ eth_pcap_tx_dumper(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts)* process stops and to make sure the pcap file is actually written, * we flush the pcap dumper within each burst. */ -pcap_dump_flush (dumper); +pcap_dumper_data_sync (dumper, tx_bytes);
'pcap_dump_flush()' should be doing the same thing, to write buffer to file, isn't it working?
Can you check the return value of the 'pcap_dump_flush()' API, I wonder if it keeps failing, for some reason?
Quoted 3 lines
drivers/net/pcap/pcap_ethdev.c +0
-0dumper_q->tx_stat.pkts += num_tx; dumper_q->tx_stat.bytes += tx_bytes; dumper_q->tx_stat.err_pkts += nb_pkts - num_tx;
Quoted 84 lines
-----Original Message-----
From: Ferruh Yigit <[email protected]>
Sent: Thursday, August 25, 2022 6:09 PM
To: Zhou, YidingX <[email protected]>; [email protected]
Cc: Zhang, Qi Z <[email protected]>; [email protected]
Subject: Re: [PATCH] net/pcap: reduce time for stopping deviceQuoted 47 lines · Yiding Zhou
The pcap file will be synchronized to the disk when stopping the device. It takes a long time if the file is large that would cause the 'detach sync request' timeout when the device is closed under multi-process scenario.
This commit fixes the issue by performing synchronization in Tx path
Fixes 4c173302c307 ("pcap: add new driver")Signed-off-by Yiding Zhou <[email protected]>--- drivers/net/pcap/pcap_ethdev.c | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-)(quoted diff) +0
-0
b/drivers/net/pcap/pcap_ethdev.cindexec29fd6bc5..52eafa5674100644drivers/net/pcap/pcap_ethdev.c +5
-1@@ -3,7 +3,7 @@* Copyright(c) 2014 6WIND S.A. * All rights reserved. */ - +#include <unistd.h> #include <time.h> #include <pcap.h> @@ -38,6 +38,8 @@#define RTE_PMD_PCAP_MAX_QUEUES 16 +#define ETH_PCAP_SYNC_THRESHOLD 0x20000000 +static char errbuf[PCAP_ERRBUF_SIZE ];static struct timespec start_time;static uint64_t start_cycles;@@ -47,6 +49,8 @@ static uint8_t iface_idx;static uint64_t timestamp_rx_dynflag;static int timestamp_dynfield_offset =-1 ; +RTE_DEFINE_PER_LCORE (uint64_t, _pcap_cached_bytes); +struct queue_stat {volatile unsigned long pkts;volatile unsigned long bytes;drivers/net/pcap/pcap_ethdev.c static struct rte_eth_link pmd_link = { · line 154
Quoted 7 lines
drivers/net/pcap/pcap_ethdev.c +4
-0@@ -144,6 +148,16 @@ static struct rte_eth_link pmd_link = {RTE_LOG_REGISTER_DEFAULT (eth_pcap_logtype,NOTICE ); +static inline void +pcap_dumper_data_sync (pcap_dumper_t *dumper,uint32_t bytes) { +RTE_PER_LCORE (_pcap_cached_bytes) += bytes; + if (unlikely(RTE_PER_LCORE(_pcap_cached_bytes) >
ETH_PCAP_SYNC_THRESHOLD)) {drivers/net/pcap/pcap_ethdev.c eth_pcap_tx_dumper(void *queue, struct rte_mbuf
Quoted 9 lines
drivers/net/pcap/pcap_ethdev.c +5
-0+if (!fdatasync (fileno (pcap_dump_file (dumper)))) +RTE_PER_LCORE (_pcap_cached_bytes) =0 ; + } +} +static struct queue_missed_stat *queue_missed_stat_update (struct rte_eth_dev *dev,unsigned int qid) {@@ -421,7 +435,7 @@ eth_pcap_tx_dumper(void *queue, struct rte_mbuf**bufs,
uint16_tnb_pkts)drivers/net/pcap/pcap_ethdev.c eth_pcap_tx_dumper(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts) · line 438
Quoted 5 lines
drivers/net/pcap/pcap_ethdev.c +1
-1* process stops and to make sure the pcap file is actually written, * we flush the pcap dumper within each burst. */ -pcap_dump_flush (dumper); +pcap_dumper_data_sync (dumper, tx_bytes);'
pcap_dump_flush()' should be doing the same thing, to write buffer to file, isn't it working?
Quoted 3 lines
Can you check the return value of the '
pcap_dump_flush()' API, I wonder if it keeps failing, for some reason?
'pcap_dump_flush()' returns 0 each time without error, it calls 'fflush()' to flush userspace buffers to kernel buffers, not disk. 'fdatasync()' to ensure data is written to disk.
Quoted 3 lines
Quoted 3 lines
drivers/net/pcap/pcap_ethdev.c +0
-0dumper_q->tx_stat.pkts += num_tx; dumper_q->tx_stat.bytes += tx_bytes; dumper_q->tx_stat.err_pkts += nb_pkts - num_tx;
Quoted 37 lines · YidingX
Quoted 37 lines
-----Original Message-----
From: Ferruh Yigit <[email protected]>
Sent: Thursday, August 25, 2022 6:09 PM
To: Zhou, YidingX <[email protected]>; [email protected]
Cc: Zhang, Qi Z <[email protected]>; [email protected]
Subject: Re: [PATCH] net/pcap: reduce time for stopping deviceQuoted 27 lines · Yiding Zhou
The pcap file will be synchronized to the disk when stopping the device. It takes a long time if the file is large that would cause the 'detach sync request' timeout when the device is closed under multi-process scenario.
This commit fixes the issue by performing synchronization in Tx path
Fixes 4c173302c307 ("pcap: add new driver")Signed-off-by Yiding Zhou <[email protected]>--- drivers/net/pcap/pcap_ethdev.c | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-)(quoted diff) +0
-0
b/drivers/net/pcap/pcap_ethdev.cindexec29fd6bc5..52eafa5674100644drivers/net/pcap/pcap_ethdev.c +1
-1@@ -3,7 +3,7 @@* Copyright(c) 2014 6WIND S.A. * All rights reserved. */ - +#include <unistd.h> #include <time.h>
Quoted 7 lines
Quoted 7 lines
drivers/net/pcap/pcap_ethdev.c line 42
Quoted 7 lines
drivers/net/pcap/pcap_ethdev.c +2
-0#include <pcap.h> @@ -38,6 +38,8 @@#define RTE_PMD_PCAP_MAX_QUEUES 16 +#define ETH_PCAP_SYNC_THRESHOLD 0x20000000 +
I guess this is 512MB, can you please comment this.
Is there any specific reason to select this value, or is it arbitrary?
Quoted 14 lines
Quoted 14 lines
Quoted 14 lines
drivers/net/pcap/pcap_ethdev.c +2
-0static char errbuf[PCAP_ERRBUF_SIZE ];static struct timespec start_time;static uint64_t start_cycles;@@ -47,6 +49,8 @@ static uint8_t iface_idx;static uint64_t timestamp_rx_dynflag;static int timestamp_dynfield_offset =-1 ; +RTE_DEFINE_PER_LCORE (uint64_t, _pcap_cached_bytes); +struct queue_stat {volatile unsigned long pkts;volatile unsigned long bytes;@@ -144,6 +148,16 @@ static struct rte_eth_link pmd_link = {
Quoted 4 lines
Quoted 4 lines
drivers/net/pcap/pcap_ethdev.c static struct rte_eth_link pmd_link = {
Quoted 4 lines
drivers/net/pcap/pcap_ethdev.c +2
-0RTE_LOG_REGISTER_DEFAULT (eth_pcap_logtype,NOTICE ); +static inline void +pcap_dumper_data_sync (pcap_dumper_t *dumper,uint32_t bytes) {
'pcap_' is the namespace for the libpcap, can you select another prefix, like 'eth_pcap_' as many driver functions does.
Quoted 8 lines
Quoted 8 lines
drivers/net/pcap/pcap_ethdev.c static struct rte_eth_link pmd_link = { · line 154
Quoted 2 lines
drivers/net/pcap/pcap_ethdev.c +2
-0+RTE_PER_LCORE (_pcap_cached_bytes) += bytes; + if (unlikely(RTE_PER_LCORE(_pcap_cached_bytes) >
ETH_PCAP_SYNC_THRESHOLD)) {drivers/net/pcap/pcap_ethdev.c static struct rte_eth_link pmd_link = { · line 160
Quoted 5 lines
drivers/net/pcap/pcap_ethdev.c +5
-0+if (!fdatasync (fileno (pcap_dump_file (dumper)))) +RTE_PER_LCORE (_pcap_cached_bytes) =0 ; + } +} +
pcap supports multiple queue, and each queue creates a new pcap dumper and single core/thread can be used for this multiple dumpers. In that case I think above per lcore variable logic doesn't work.
And instead of having a global value, what do you think to add a variable to 'struct pcap_tx_queue' for this purpose?
Quoted 14 lines
Quoted 14 lines
drivers/net/pcap/pcap_ethdev.c eth_pcap_tx_dumper(void *queue, struct rte_mbuf
Quoted 4 lines
drivers/net/pcap/pcap_ethdev.c +0
-0static struct queue_missed_stat *queue_missed_stat_update (struct rte_eth_dev *dev,unsigned int qid) {@@ -421,7 +435,7 @@ eth_pcap_tx_dumper(void *queue, struct rte_mbuf**bufs,
uint16_tnb_pkts)drivers/net/pcap/pcap_ethdev.c eth_pcap_tx_dumper(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts) · line 438
Quoted 5 lines
drivers/net/pcap/pcap_ethdev.c +1
-1* process stops and to make sure the pcap file is actually written, * we flush the pcap dumper within each burst. */ -pcap_dump_flush (dumper); +pcap_dumper_data_sync (dumper, tx_bytes);'
pcap_dump_flush()' should be doing the same thing, to write buffer to file, isn't it working?
Quoted 6 lines
Quoted 3 lines
Can you check the return value of the '
pcap_dump_flush()' API, I wonder if it keeps failing, for some reason?'
pcap_dump_flush()' returns 0 each time without error, it calls 'fflush()' to flush userspace buffers to kernel buffers, not disk. 'fdatasync()' to ensure data is written to disk.
'pcap_dump_flush()' API documentation says "flushes the output buffer to the ``savefile,''", but as you said it uses 'fflush()' internally, so there is a chance that data is not written to the disk.
In this case, won't need to keep both, first flush and later fsync/fdatasync?
Do you observe any performance difference after change, since now writing to actual disk on datapath?
Quoted 4 lines
Quoted 3 lines
Quoted 3 lines
drivers/net/pcap/pcap_ethdev.c +0
-0dumper_q->tx_stat.pkts += num_tx; dumper_q->tx_stat.bytes += tx_bytes; dumper_q->tx_stat.err_pkts += nb_pkts - num_tx;
Quoted 48 lines
-----Original Message-----
From: Ferruh Yigit <[email protected]>
Sent: Thursday, August 25, 2022 8:22 PM
To: Zhou, YidingX <[email protected]>; [email protected]
Cc: Zhang, Qi Z <[email protected]>; [email protected]
Subject: Re: [PATCH] net/pcap: reduce time for stopping deviceQuoted 40 lines · YidingX
Quoted 40 lines
-----Original Message-----
From: Ferruh Yigit <[email protected]>
Sent: Thursday, August 25, 2022 6:09 PM
To: Zhou, YidingX <[email protected]>; [email protected]
Cc: Zhang, Qi Z <[email protected]>; [email protected]
Subject: Re: [PATCH] net/pcap: reduce time for stopping deviceQuoted 30 lines · Yiding Zhou
The pcap file will be synchronized to the disk when stopping the device. It takes a long time if the file is large that would cause the 'detach sync request' timeout when the device is closed under multi-process scenario.
This commit fixes the issue by performing synchronization in Tx path
Fixes 4c173302c307 ("pcap: add new driver")Signed-off-by Yiding Zhou <[email protected]>--- drivers/net/pcap/pcap_ethdev.c | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-)(quoted diff) +0
-0
b/drivers/net/pcap/pcap_ethdev.cindexec29fd6bc5..52eafa5674100644drivers/net/pcap/pcap_ethdev.c +1
-1@@ -3,7 +3,7 @@* Copyright(c) 2014 6WIND S.A. * All rights reserved. */ - +#include <unistd.h> #include <time.h> #include <pcap.h> @@ -38,6 +38,8 @@
Quoted 9 lines
Quoted 4 lines
Quoted 4 lines
drivers/net/pcap/pcap_ethdev.c line 42
Quoted 4 lines
drivers/net/pcap/pcap_ethdev.c +2
-0#define RTE_PMD_PCAP_MAX_QUEUES 16 +#define ETH_PCAP_SYNC_THRESHOLD 0x20000000 +I guess this is 512MB, can you please comment this.
Is there any specific reason to select this value, or is it arbitrary?
512M is arbitrary, because there is no API to get the disk cache size associated with a specific file. I will test the performance impact of different values.
Quoted 35 lines
Quoted 14 lines
Quoted 14 lines
Quoted 14 lines
drivers/net/pcap/pcap_ethdev.c +2
-0static char errbuf[PCAP_ERRBUF_SIZE ];static struct timespec start_time;static uint64_t start_cycles;@@ -47,6 +49,8 @@ static uint8_t iface_idx;static uint64_t timestamp_rx_dynflag;static int timestamp_dynfield_offset =-1 ; +RTE_DEFINE_PER_LCORE (uint64_t, _pcap_cached_bytes); +struct queue_stat {volatile unsigned long pkts;volatile unsigned long bytes;@@ -144,6 +148,16 @@ static struct rte_eth_link pmd_link = {Quoted 4 lines
Quoted 4 lines
drivers/net/pcap/pcap_ethdev.c static struct rte_eth_link pmd_link = {
Quoted 4 lines
drivers/net/pcap/pcap_ethdev.c +2
-0RTE_LOG_REGISTER_DEFAULT (eth_pcap_logtype,NOTICE ); +static inline void +pcap_dumper_data_sync (pcap_dumper_t *dumper,uint32_t bytes) {'
pcap_' is the namespace for the libpcap, can you select another prefix, like 'eth_pcap_' as many driver functions does.Quoted 8 lines
Quoted 8 lines
drivers/net/pcap/pcap_ethdev.c static struct rte_eth_link pmd_link = { · line 154
Quoted 2 lines
drivers/net/pcap/pcap_ethdev.c +2
-0+RTE_PER_LCORE (_pcap_cached_bytes) += bytes; + if (unlikely(RTE_PER_LCORE(_pcap_cached_bytes) >
ETH_PCAP_SYNC_THRESHOLD)) {drivers/net/pcap/pcap_ethdev.c static struct rte_eth_link pmd_link = { · line 160
Quoted 5 lines
drivers/net/pcap/pcap_ethdev.c +5
-0+if (!fdatasync (fileno (pcap_dump_file (dumper)))) +RTE_PER_LCORE (_pcap_cached_bytes) =0 ; + } +} +pcap supports multiple queue, and each queue creates a new pcap dumper and single core/thread can be used for this multiple dumpers. In that case I think above per lcore variable logic doesn't work.
Quoted 3 lines
And instead of having a global value, what do you think to add a variable to 'struct
pcap_tx_queue' for this purpose?
Thanks for the comments, I will follow this.
Quoted 23 lines
Quoted 14 lines
Quoted 14 lines
drivers/net/pcap/pcap_ethdev.c eth_pcap_tx_dumper(void *queue, struct rte_mbuf
Quoted 4 lines
drivers/net/pcap/pcap_ethdev.c +0
-0static struct queue_missed_stat *queue_missed_stat_update (struct rte_eth_dev *dev,unsigned int qid) {@@ -421,7 +435,7 @@ eth_pcap_tx_dumper(void *queue, struct rte_mbuf**bufs,
uint16_tnb_pkts)drivers/net/pcap/pcap_ethdev.c eth_pcap_tx_dumper(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts) · line 438
Quoted 5 lines
drivers/net/pcap/pcap_ethdev.c +1
-1* process stops and to make sure the pcap file is actually written, * we flush the pcap dumper within each burst. */ -pcap_dump_flush (dumper); +pcap_dumper_data_sync (dumper, tx_bytes);'
pcap_dump_flush()' should be doing the same thing, to write buffer to file, isn't it working?Quoted 5 lines
Quoted 3 lines
Can you check the return value of the '
pcap_dump_flush()' API, I wonder if it keeps failing, for some reason?'
pcap_dump_flush()' returns 0 each time without error, it calls 'fflush()' toflush userspace buffers to kernel buffers, not disk. '
fdatasync()' to ensure data is written to disk.Quoted 1 line
Quoted 6 lines
'
pcap_dump_flush()' API documentation says "flushes the output buffer to the ``savefile,''", but as you said it uses 'fflush()' internally, so there is a chance that data is not written to the disk.In this case, won't need to keep both, first flush and later fsync/fdatasync?
I draw a diagram to describe it more clearly
fwrite fclose/fflush fclose/fdatasync
--------->| libc buffer |----------------> | disk cache in RAM |---------------------> |disk|
| 4096 Bytes | | size is determined by OS | | |
When the libc buffer is full, the system will automatically sync it to the disk cache. It is easily full as it's only 4096 B size. so there is no need to call 'fflush()' every time. The real time consuming action is syncing the disk cache to disk.
Because the disk cache is very large, it will take a long time to sync all at one time during 'fclose()', so need to call 'fdatasync()' periodically to amortize the time.
Quoted 3 lines
Do you observe any performance difference after change, since now writing to actual disk on datapath?
I will verify any performance difference under these 2 scenarios.
Quoted 4 lines
Quoted 4 lines
Quoted 3 lines
Quoted 3 lines
drivers/net/pcap/pcap_ethdev.c +0
-0dumper_q->tx_stat.pkts += num_tx; dumper_q->tx_stat.bytes += tx_bytes; dumper_q->tx_stat.err_pkts += nb_pkts - num_tx;
Quoted 11 lines · YidingX
I draw a diagram to describe it more clearly
fwrite fclose/fflush fclose/fdatasync
--------->| libc buffer |----------------> | disk cache in RAM |---------------------> |disk|
| 4096 Bytes | | size is determined by OS | | |When the libc buffer is full, the system will automatically sync it to the disk cache. It is easily full as it's only 4096 B size. so there is no need to call '
fflush()' every time. The real time consuming action is syncing the disk cache to disk.
Because the disk cache is very large, it will take a long time to sync all at one time during 'fclose()', so need to call 'fdatasync()' periodically to amortize the time.
If you want to speed up this, then get rid of stdio and use a faster API like io_uring. What you tried can help but using a better API would make bigger gains.
Hi, Ferruh
Quoted 136 lines
Quoted 32 lines
Quoted 32 lines
Quoted 32 lines
Quoted 32 lines
The pcap file will be synchronized to the disk when stopping the device. It takes a long time if the file is large that would cause the 'detach sync request' timeout when the device is closed under multi-process scenario.
This commit fixes the issue by performing synchronization in Tx path
Fixes 4c173302c307 ("pcap: add new driver")Signed-off-by Yiding Zhou <[email protected]>--- drivers/net/pcap/pcap_ethdev.c | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-)(quoted diff) +0
-0
b/drivers/net/pcap/pcap_ethdev.cindexec29fd6bc5..52eafa5674100644drivers/net/pcap/pcap_ethdev.c +1
-1@@ -3,7 +3,7 @@* Copyright(c) 2014 6WIND S.A. * All rights reserved. */ - +#include <unistd.h> #include <time.h> #include <pcap.h> @@ -38,6 +38,8 @@Quoted 9 lines
Quoted 4 lines
Quoted 4 lines
drivers/net/pcap/pcap_ethdev.c line 42
Quoted 4 lines
drivers/net/pcap/pcap_ethdev.c +2
-0#define RTE_PMD_PCAP_MAX_QUEUES 16 +#define ETH_PCAP_SYNC_THRESHOLD 0x20000000 +I guess this is 512MB, can you please comment this.
Is there any specific reason to select this value, or is it arbitrary?512M is arbitrary, because there is no API to get the disk cache size associated with a specific file.
I will test the performance impact of different values.Quoted 35 lines
Quoted 14 lines
Quoted 14 lines
Quoted 14 lines
drivers/net/pcap/pcap_ethdev.c +2
-0static char errbuf[PCAP_ERRBUF_SIZE ];static struct timespec start_time;static uint64_t start_cycles;@@ -47,6 +49,8 @@ static uint8_t iface_idx;static uint64_t timestamp_rx_dynflag;static int timestamp_dynfield_offset =-1 ; +RTE_DEFINE_PER_LCORE (uint64_t, _pcap_cached_bytes); +struct queue_stat {volatile unsigned long pkts;volatile unsigned long bytes;@@ -144,6 +148,16 @@ static struct rte_eth_link pmd_link = {Quoted 4 lines
Quoted 4 lines
drivers/net/pcap/pcap_ethdev.c static struct rte_eth_link pmd_link = {
Quoted 4 lines
drivers/net/pcap/pcap_ethdev.c +2
-0RTE_LOG_REGISTER_DEFAULT (eth_pcap_logtype,NOTICE ); +static inline void +pcap_dumper_data_sync (pcap_dumper_t *dumper,uint32_t bytes) {'
pcap_' is the namespace for the libpcap, can you select another prefix, like 'eth_pcap_' as many driver functions does.Quoted 8 lines
Quoted 8 lines
drivers/net/pcap/pcap_ethdev.c static struct rte_eth_link pmd_link = { · line 154
Quoted 2 lines
drivers/net/pcap/pcap_ethdev.c +2
-0+RTE_PER_LCORE (_pcap_cached_bytes) += bytes; + if (unlikely(RTE_PER_LCORE(_pcap_cached_bytes) >
ETH_PCAP_SYNC_THRESHOLD)) {drivers/net/pcap/pcap_ethdev.c static struct rte_eth_link pmd_link = { · line 160
Quoted 5 lines
drivers/net/pcap/pcap_ethdev.c +5
-0+if (!fdatasync (fileno (pcap_dump_file (dumper)))) +RTE_PER_LCORE (_pcap_cached_bytes) =0 ; + } +} +pcap supports multiple queue, and each queue creates a new pcap dumper and single core/thread can be used for this multiple dumpers. In that case I think above per lcore variable logic doesn't work.
Quoted 3 lines
And instead of having a global value, what do you think to add a variable to 'struct
pcap_tx_queue' for this purpose?Thanks for the comments, I will follow this.
Quoted 25 lines
Quoted 15 lines
Quoted 15 lines
Quoted 5 lines
drivers/net/pcap/pcap_ethdev.c +0
-0static struct queue_missed_stat *queue_missed_stat_update (struct rte_eth_dev *dev,unsigned int qid) {@@ -421,7 +435,7 @@ eth_pcap_tx_dumper(void *queue, struct
rte_mbuf**bufs,
uint16_tnb_pkts)drivers/net/pcap/pcap_ethdev.c eth_pcap_tx_dumper(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts) · line 438
Quoted 5 lines
* process stops and to make sure the pcap file is actually written,
* we flush the pcap dumper within each burst.
*/drivers/net/pcap/pcap_ethdev.c +1
-1-pcap_dump_flush (dumper); +pcap_dumper_data_sync (dumper, tx_bytes);'
pcap_dump_flush()' should be doing the same thing, to write buffer to file, isn't it working?Quoted 6 lines
Quoted 3 lines
Can you check the return value of the '
pcap_dump_flush()' API, I wonder if it keeps failing, for some reason?'
pcap_dump_flush()' returns 0 each time without error, it calls 'fflush()' toflush userspace buffers to kernel buffers, not disk. '
fdatasync()' to ensure data is written to disk.Quoted 1 line
Quoted 6 lines
'
pcap_dump_flush()' API documentation says "flushes the output buffer to the ``savefile,''", but as you said it uses 'fflush()' internally, so there is a chance that data is not written to the disk.In this case, won't need to keep both, first flush and later fsync/fdatasync?
I draw a diagram to describe it more clearly
fwrite fclose/fflush fclose/fdatasync
--------->| libc buffer |----------------> | disk cache in RAM |--------------------->|disk| | 4096 Bytes | | size is determined by OS | | |When the libc buffer is full, the system will automatically sync it to the disk cache.
It is easily full as it's only 4096 B size. so there is no need to call 'fflush()' every time.
The real time consuming action is syncing the disk cache to disk.
Because the disk cache is very large, it will take a long time to sync all at one time during 'fclose()', so need to call 'fdatasync()' periodically to amortize the time.
Quoted 6 lines
Quoted 3 lines
Do you observe any performance difference after change, since now writing to actual disk on datapath?
I will verify any performance difference under these 2 scenarios.
I have done some tests, using 'fdatasync()' will cause the packet loss rate to increase by 15%-20%, It's not a good solution, need to rework. I plan to use a timer to call pcap_dump_close to fix this, do you have any suggestion?
thanks
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 3 lines
Quoted 3 lines
dumper_q->tx_stat.pkts += num_tx; dumper_q->tx_stat.bytes += tx_bytes; dumper_q->tx_stat.err_pkts += nb_pkts - num_tx;
PATCH v2 1/1 net/pcap: fix timeout of stopping device
In reply to Yiding ZhouThe pcap file will be synchronized to the disk when stopping the device. It takes a long time if the file is large that would cause the 'detach sync request' timeout when the device is closed under multi-process scenario.
This commit fixes the issue by using alarm handler to release dumper.
---
v2: use alarm handler to release dumper
--- drivers/net/pcap/pcap_ethdev.c | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-)
drivers/net/pcap/pcap_ethdev.c +21 -1
@@ -17,6 +17,7 @@#include <rte_mbuf_dyn.h> #include <rte_bus_vdev.h> #include <rte_os_shim.h> +#include <rte_alarm.h> #include "pcap_osdep.h"
@@ -664,6 +665,25 @@ eth_dev_start(struct rte_eth_dev *dev)return 0 ; } +static void eth_pcap_dumper_release (void *arg) +{ +pcap_dump_close ((pcap_dumper_t *)arg); +} + +static void +eth_pcap_dumper_close (pcap_dumper_t *dumper) +{ +if (rte_eal_process_type () ==RTE_PROC_PRIMARY ) { +/* +* Delay 30 seconds before releasing dumper to wait for file sync +* to complete to avoid blocking alarm thread in PRIMARY process +*/ +rte_eal_alarm_set (30000000 , eth_pcap_dumper_release, dumper); + }else { +rte_eal_alarm_set (1 , eth_pcap_dumper_release, dumper); + } +} +/* * This function gets called when the current port gets stopped. * Is the only place for us to close all the tx streams dumpers.
@@ -689,7 +709,7 @@ eth_dev_stop(struct rte_eth_dev *dev)for (i =0 ; i < dev->data->nb_tx_queues; i++) {if (pp->tx_dumper[i] !=NULL ) { -pcap_dump_close (pp->tx_dumper[i]); +eth_pcap_dumper_close (pp->tx_dumper[i]); pp->tx_dumper[i] =NULL ; }
Signature · 1 line
2.34.1
Re: [PATCH v2] net/pcap: fix timeout of stopping device
Quoted 11 lines · On Tue, 6 Sep 2022 16:05:11 +0800 Yiding Zhou <[email protected]> wrote:
The pcap file will be synchronized to the disk when stopping the device. It takes a long time if the file is large that would cause the 'detach sync request' timeout when the device is closed under multi-process scenario.
This commit fixes the issue by using alarm handler to release dumper.
Fixes 0ecfb6c04d54 ("net/pcap: move handler to process private")Signed-off-by Yiding Zhou <[email protected]>
I think you need to redesign the handshake if this the case. Forcing 30 second delay at the end of all uses of pcap is not acceptable.
RE: [PATCH v2] net/pcap: fix timeout of stopping device
Quoted 11 lines · On Tue, 6 Sep 2022 16:05:11 +0800 Yiding Zhou <[email protected]> wrote:
The pcap file will be synchronized to the disk when stopping the device. It takes a long time if the file is large that would cause the 'detach sync request' timeout when the device is closed under multi-process scenario.
This commit fixes the issue by using alarm handler to release dumper.
Fixes 0ecfb6c04d54 ("net/pcap: move handler to process private")Signed-off-by Yiding Zhou <[email protected]>
I think you need to redesign the handshake if this the case. Forcing 30 second delay at the end of all uses of pcap is not acceptable.
Thanks for your comments.
According to my test, the time required to sync a 100G pcap file is about 20s,
so I set a delay of 30s. I also tried to use AIO, but some issue in the multi-process scenario. And I also consider io_uring, it is only supported in linux-5.x, we need to consider compatibility,. Maybe better way is to do more work to redesign the handshake.
RE: [PATCH v2] net/pcap: fix timeout of stopping device
In reply to Stephen HemmingerQuoted 21 lines
-----Original Message-----
From: Stephen Hemminger <[email protected]>
Sent: Tuesday, September 6, 2022 10:58 PM
To: Zhou, YidingX <[email protected]>
Cc: [email protected]; Zhang, Qi Z <[email protected]>; Burakov, Anatoly <[email protected]>; He, Xingguang <[email protected]>; [email protected]
Subject: Re: [PATCH v2] net/pcap: fix timeout of stopping deviceQuoted 7 lines · On Tue, 6 Sep 2022 16:05:11 +0800 Yiding Zhou <[email protected]> wrote:
The pcap file will be synchronized to the disk when stopping the device. It takes a long time if the file is large that would cause the 'detach sync request' timeout when the device is closed under multi-process scenario.
This commit fixes the issue by using alarm handler to release dumper.
Quoted 8 lines
Quoted 4 lines
Fixes 0ecfb6c04d54 ("net/pcap: move handler to process private")Signed-off-by Yiding Zhou <[email protected]>I think you need to redesign the handshake if this the case. Forcing 30 second delay at the end of all uses of pcap is not acceptable.
@Zhang, Qi Z Do we need to redesign the handshake to fix this?
Re: [PATCH v2] net/pcap: fix timeout of stopping device
Quoted 24 lines · YidingX
Quoted 24 lines
-----Original Message-----
From: Stephen Hemminger <[email protected]>
Sent: Tuesday, September 6, 2022 10:58 PM
To: Zhou, YidingX <[email protected]>
Cc: [email protected]; Zhang, Qi Z <[email protected]>; Burakov, Anatoly <[email protected]>; He, Xingguang <[email protected]>; [email protected]
Subject: Re: [PATCH v2] net/pcap: fix timeout of stopping deviceQuoted 10 lines · On Tue, 6 Sep 2022 16:05:11 +0800 Yiding Zhou <[email protected]> wrote:
The pcap file will be synchronized to the disk when stopping the device. It takes a long time if the file is large that would cause the 'detach sync request' timeout when the device is closed under multi-process scenario.
This commit fixes the issue by using alarm handler to release dumper.
Fixes 0ecfb6c04d54 ("net/pcap: move handler to process private")
Quoted 7 lines
Quoted 5 lines
Quoted 1 line
Signed-off-by Yiding Zhou <[email protected]>I think you need to redesign the handshake if this the case. Forcing 30 second delay at the end of all uses of pcap is not acceptable.
@Zhang, Qi Z Do we need to redesign the handshake to fix this?
Hi Yiding,
Your approach works, but Stephen's comment is to not add this delay by default, because this delay is not always required.
Can you please provide more details on multi-process communication and call trace, to help us think about a solution to address this issue in a more generic way (not just for pcap but for any case device close takes more than multi-process timeout)?
RE: [PATCH v2] net/pcap: fix timeout of stopping device
In reply to Zhou, YidingXQuoted 35 lines
-----Original Message-----
From: Zhou, YidingX <[email protected]>
Sent: Wednesday, September 21, 2022 3:15 PM
To: Stephen Hemminger <[email protected]>; Zhang, Qi Z <[email protected]>
Cc: [email protected]; Burakov, Anatoly <[email protected]>; He, Xingguang <[email protected]>; [email protected]
Subject: RE: [PATCH v2] net/pcap: fix timeout of stopping deviceQuoted 22 lines
-----Original Message-----
From: Stephen Hemminger <mailto:[email protected]>
Sent: Tuesday, September 6, 2022 10:58 PM
To: Zhou, YidingX <mailto:[email protected]>
Cc: mailto:[email protected]; Zhang, Qi Z <mailto:[email protected]>; Burakov, Anatoly <mailto:[email protected]>; He, Xingguang <mailto:[email protected]>; mailto:[email protected]
Subject: Re: [PATCH v2] net/pcap: fix timeout of stopping deviceQuoted 10 lines · On Tue, 6 Sep 2022 16:05:11 +0800 Yiding Zhou <mailto:[email protected]>…
The pcap file will be synchronized to the disk when stopping the device. It takes a long time if the file is large that would cause the 'detach sync request' timeout when the device is closed under multi-process scenario.
This commit fixes the issue by using alarm handler to release dumper.
Fixes 0ecfb6c04d54 ("net/pcap: move handler to process private")Cc mailto:[email protected]
Quoted 7 lines
Quoted 5 lines
Quoted 1 line
Signed-off-by Yiding Zhou <mailto:[email protected]>I think you need to redesign the handshake if this the case. Forcing 30 second delay at the end of all uses of pcap is not acceptable.
@Zhang, Qi Z Do we need to redesign the handshake to fix this?
Hi, Ferruh
Sorry for the late reply.
I did not receive your email on Oct 6, I got your comments from patchwork.
"Can you please provide more details on multi-process communication and call trace, to help us think about a solution to address this issue in a more generic way (not just for pcap but for any case device close takes more than multi-process timeout)?"
I try to explain this issue with a sequence diagram, hope it can be displayed correctly in the mail.
thread intr thread intr thread thread
of secondary of secondary of primary of primary
| | | |
| | | |rte_eal_hotplug_removerte_dev_removeeal_dev_hotplug_request_to_primaryrte_mp_request_sync ------------------------------------------------------->|
|
handle_secondary_request
|<-----------------|
|
__handle_secondary_request
eal_dev_hotplug_request_to_secondary
|<------------------------------------- rte_mp_request_sync
|
handle_primary_request--------->|
|
__handle_primary_request
local_dev_remove(this will take long time)
rte_mp_reply -------------------------------->|
|
local_dev_remove
|<------------------------------------------------- rte_mp_replyThe marked 'local_dev_remove()' in the secondary process will perform a pcap file synchronization operation. When the pcap file is too large, it will take a lot of time (according to my test 100G takes 20+ seconds). This caused the processing of hot_plug message to time out.
Re: [PATCH v2] net/pcap: fix timeout of stopping device
Quoted 77 lines · YidingX
Quoted 33 lines
-----Original Message-----
From: Zhou, YidingX <[email protected]>
Sent: Wednesday, September 21, 2022 3:15 PM
To: Stephen Hemminger <[email protected]>; Zhang, Qi Z <[email protected]>
Cc: [email protected]; Burakov, Anatoly <[email protected]>; He, Xingguang <[email protected]>; [email protected]
Subject: RE: [PATCH v2] net/pcap: fix timeout of stopping deviceQuoted 22 lines
-----Original Message-----
From: Stephen Hemminger <mailto:[email protected]>
Sent: Tuesday, September 6, 2022 10:58 PM
To: Zhou, YidingX <mailto:[email protected]>
Cc: mailto:[email protected]; Zhang, Qi Z <mailto:[email protected]>; Burakov, Anatoly <mailto:[email protected]>; He, Xingguang <mailto:[email protected]>; mailto:[email protected]
Subject: Re: [PATCH v2] net/pcap: fix timeout of stopping deviceQuoted 10 lines · On Tue, 6 Sep 2022 16:05:11 +0800 Yiding Zhou <mailto:[email protected]>…
The pcap file will be synchronized to the disk when stopping the device. It takes a long time if the file is large that would cause the 'detach sync request' timeout when the device is closed under multi-process scenario.
This commit fixes the issue by using alarm handler to release dumper.
Fixes 0ecfb6c04d54 ("net/pcap: move handler to process private")Cc mailto:[email protected]Quoted 7 lines
Quoted 5 lines
Quoted 1 line
Signed-off-by Yiding Zhou <mailto:[email protected]>I think you need to redesign the handshake if this the case. Forcing 30 second delay at the end of all uses of pcap is not acceptable.
@Zhang, Qi Z Do we need to redesign the handshake to fix this?
Hi, Ferruh
Sorry for the late reply.
I did not receive your email on Oct 6, I got your comments from patchwork."Can you please provide more details on multi-process communication and call trace, to help us think about a solution to address this issue in a more generic way (not just for pcap but for any case device close takes more than multi-process timeout)?"
I try to explain this issue with a sequence diagram, hope it can be displayed correctly in the mail.
thread intr thread intr thread thread
of secondary of secondary of primary of primary
| | | |
| | | |rte_eal_hotplug_removerte_dev_removeeal_dev_hotplug_request_to_primaryrte_mp_request_sync------------------------------------------------------->|| handle_secondary_request |<-----------------| | __handle_secondary_request eal_dev_hotplug_request_to_secondary |<------------------------------------- rte_mp_request_sync | handle_primary_request--------->| | __handle_primary_request local_dev_remove(this will take long time) rte_mp_reply -------------------------------->| | local_dev_remove |<------------------------------------------------- rte_mp_reply
Quoted 3 lines
The marked '
local_dev_remove()' in the secondary process will perform a pcap file synchronization operation. When the pcap file is too large, it will take a lot of time (according to my test 100G takes 20+ seconds). This caused the processing ofhot_plugmessage to time out.
Part of the problem maybe a hidden file sync in some library. Normally, closing a file should be fast even with lots of outstanding data. The actual write done by OS will continue from file cache.
I wonder if doing some kind of fadvise call might help see POSIX_FADV_SEQUENTIAL or POSIX_FADV_DONTNEED
RE: [PATCH v2] net/pcap: fix timeout of stopping device
Quoted 68 lines
Quoted 26 lines
Quoted 14 lines
Quoted 14 lines
Quoted 11 lines · On Tue, 6 Sep 2022 16:05:11 +0800 Yiding Zhou <mailto:[email protected]>…
The pcap file will be synchronized to the disk when stopping the device. It takes a long time if the file is large that would cause the 'detach sync request' timeout when the device is closed under multi-process scenario.
This commit fixes the issue by using alarm handler to release dumper.
Fixes 0ecfb6c04d54 ("net/pcap: move handler to processprivate")
Cc: mailto:[email protected]Quoted 7 lines
Quoted 5 lines
Quoted 1 line
Signed-off-by Yiding Zhou <mailto:[email protected]>I think you need to redesign the handshake if this the case. Forcing 30 second delay at the end of all uses of pcap is not acceptable.
@Zhang, Qi Z Do we need to redesign the handshake to fix this?
Hi, Ferruh
Sorry for the late reply.
I did not receive your email on Oct 6, I got your comments from patchwork.Quoted 6 lines
"Can you please provide more details on multi-process communication and call trace, to help us think about a solution to address this issue in a more generic way (not just for pcap but for any case device close takes more than multi-process timeout)?"
I try to explain this issue with a sequence diagram, hope it can be displayed
correctly in the mail.
Quoted 11 lines
thread intr thread intr thread thread
of secondary of secondary of primary of primary
| | | |
| | | |rte_eal_hotplug_removerte_dev_removeeal_dev_hotplug_request_to_primaryrte_mp_request_sync------------------------------------------------------->||
handle_secondary_requestQuoted 14 lines
|<-----------------| | __handle_secondary_request eal_dev_hotplug_request_to_secondary |<------------------------------------- rte_mp_request_sync | handle_primary_request--------->| | __handle_primary_request local_dev_remove(this will take long time) rte_mp_reply -------------------------------->| | local_dev_remove |<-------------------------------------------------Quoted 3 lines
rte_mp_replyThe marked '
local_dev_remove()' in the secondary process will perform apcap file synchronization operation.
Quoted 1 line
When the pcap file is too large, it will take a lot of time (according to my test
100G takes 20+ seconds).
Quoted 1 line
This caused the processing of
hot_plugmessage to time out.
Quoted 6 lines
Part of the problem maybe a hidden file sync in some library. Normally, closing a file should be fast even with lots of outstanding data. The actual write done by OS will continue from file cache.
I wonder if doing some kind of fadvise call might help see
POSIX_FADV_SEQUENTIALorPOSIX_FADV_DONTNEED
Thanks for your advice, I will try it and give you feedback.
Re: [PATCH v2] net/pcap: fix timeout of stopping device
In reply to Zhou, YidingXQuoted 79 lines · YidingX
Quoted 35 lines
-----Original Message-----
From: Zhou, YidingX <[email protected]>
Sent: Wednesday, September 21, 2022 3:15 PM
To: Stephen Hemminger <[email protected]>; Zhang, Qi Z <[email protected]>
Cc: [email protected]; Burakov, Anatoly <[email protected]>; He, Xingguang <[email protected]>; [email protected]
Subject: RE: [PATCH v2] net/pcap: fix timeout of stopping deviceQuoted 22 lines
-----Original Message-----
From: Stephen Hemminger <mailto:[email protected]>
Sent: Tuesday, September 6, 2022 10:58 PM
To: Zhou, YidingX <mailto:[email protected]>
Cc: mailto:[email protected]; Zhang, Qi Z <mailto:[email protected]>; Burakov, Anatoly <mailto:[email protected]>; He, Xingguang <mailto:[email protected]>; mailto:[email protected]
Subject: Re: [PATCH v2] net/pcap: fix timeout of stopping deviceQuoted 10 lines · On Tue, 6 Sep 2022 16:05:11 +0800 Yiding Zhou <mailto:[email protected]>…
The pcap file will be synchronized to the disk when stopping the device. It takes a long time if the file is large that would cause the 'detach sync request' timeout when the device is closed under multi-process scenario.
This commit fixes the issue by using alarm handler to release dumper.
Fixes 0ecfb6c04d54 ("net/pcap: move handler to process private")Cc mailto:[email protected]Quoted 7 lines
Quoted 5 lines
Quoted 1 line
Signed-off-by Yiding Zhou <mailto:[email protected]>I think you need to redesign the handshake if this the case. Forcing 30 second delay at the end of all uses of pcap is not acceptable.
@Zhang, Qi Z Do we need to redesign the handshake to fix this?
Hi, Ferruh
Sorry for the late reply.
I did not receive your email on Oct 6, I got your comments from patchwork."Can you please provide more details on multi-process communication and call trace, to help us think about a solution to address this issue in a more generic way (not just for pcap but for any case device close takes more than multi-process timeout)?"
I try to explain this issue with a sequence diagram, hope it can be displayed correctly in the mail.
thread intr thread intr thread thread
of secondary of secondary of primary of primary
| | | |
| | | |rte_eal_hotplug_removerte_dev_removeeal_dev_hotplug_request_to_primaryrte_mp_request_sync------------------------------------------------------->|| handle_secondary_request |<-----------------| | __handle_secondary_request eal_dev_hotplug_request_to_secondary |<------------------------------------- rte_mp_request_sync | handle_primary_request--------->| | __handle_primary_request local_dev_remove(this will take long time) rte_mp_reply -------------------------------->| | local_dev_remove |<------------------------------------------------- rte_mp_reply
Quoted 3 lines
The marked '
local_dev_remove()' in the secondary process will perform a pcap file synchronization operation. When the pcap file is too large, it will take a lot of time (according to my test 100G takes 20+ seconds). This caused the processing ofhot_plugmessage to time out.
Hi Yiding,
Thanks for the information,
Right now all MP operations timeout is hardcoded in the code and it is 5 seconds.
Do you think does it work to have an API to set custom timeout, something like rte_mp_timeout_set(), and call this from pdump?
This gives a generic solution for similar cases, not just for pcap. But my concern is if this is too much multi-process related internal detail to update, @Anatoly may comment on this.
RE: [PATCH v2] net/pcap: fix timeout of stopping device
Quoted 72 lines
Quoted 27 lines
Quoted 15 lines
Quoted 15 lines
Quoted 10 lines · On Tue, 6 Sep 2022 16:05:11 +0800 Yiding Zhou <mailto:[email protected]>…
The pcap file will be synchronized to the disk when stopping the device. It takes a long time if the file is large that would cause the 'detach sync request' timeout when the device is closed under multi-process scenario.
This commit fixes the issue by using alarm handler to release dumper.
Fixes 0ecfb6c04d54 ("net/pcap: move handler to process private")Cc mailto:[email protected]Quoted 7 lines
Quoted 5 lines
Quoted 1 line
Signed-off-by Yiding Zhou <mailto:[email protected]>I think you need to redesign the handshake if this the case. Forcing 30 second delay at the end of all uses of pcap is not acceptable.
@Zhang, Qi Z Do we need to redesign the handshake to fix this?
Hi, Ferruh
Sorry for the late reply.
I did not receive your email on Oct 6, I got your comments from patchwork.Quoted 6 lines
"Can you please provide more details on multi-process communication and call trace, to help us think about a solution to address this issue in a more generic way (not just for pcap but for any case device close takes more than multi-process timeout)?"
I try to explain this issue with a sequence diagram, hope it can be displayed
correctly in the mail.
Quoted 11 lines
thread intr thread intr thread thread
of secondary of secondary of primary of primary
| | | |
| | | |rte_eal_hotplug_removerte_dev_removeeal_dev_hotplug_request_to_primaryrte_mp_request_sync------------------------------------------------------->||
handle_secondary_requestQuoted 14 lines
|<-----------------| | __handle_secondary_request eal_dev_hotplug_request_to_secondary |<------------------------------------- rte_mp_request_sync | handle_primary_request--------->| | __handle_primary_request local_dev_remove(this will take long time) rte_mp_reply -------------------------------->| | local_dev_remove |<-------------------------------------------------Quoted 3 lines
rte_mp_replyThe marked '
local_dev_remove()' in the secondary process will perform apcap file synchronization operation.
Quoted 1 line
When the pcap file is too large, it will take a lot of time (according to my test
100G takes 20+ seconds).
Quoted 1 line
This caused the processing of
hot_plugmessage to time out.Hi Yiding,
Thanks for the information,
Quoted 8 lines
Right now all MP operations timeout is hardcoded in the code and it is 5 seconds.
Do you think does it work to have an API to set custom timeout, something likerte_mp_timeout_set(), and call this from pdump?This gives a generic solution for similar cases, not just for pcap. But my concern is if this is too much multi-process related internal detail to update, @Anatoly may comment on this.
Hi, Ferruh
For pdump case only, I think the timeout is affected by pcap's size and other system components, such as the type of FS, system memory size. It may be difficult to predict the specific time value for setting.
Re: [PATCH v2] net/pcap: fix timeout of stopping device
Quoted 77 lines · YidingX
Quoted 77 lines
Quoted 27 lines
Quoted 15 lines
Quoted 15 lines
Quoted 10 lines · On Tue, 6 Sep 2022 16:05:11 +0800 Yiding Zhou <mailto:[email protected]>…
The pcap file will be synchronized to the disk when stopping the device. It takes a long time if the file is large that would cause the 'detach sync request' timeout when the device is closed under multi-process scenario.
This commit fixes the issue by using alarm handler to release dumper.
Fixes 0ecfb6c04d54 ("net/pcap: move handler to process private")Cc mailto:[email protected]Quoted 7 lines
Quoted 5 lines
Quoted 1 line
Signed-off-by Yiding Zhou <mailto:[email protected]>I think you need to redesign the handshake if this the case. Forcing 30 second delay at the end of all uses of pcap is not acceptable.
@Zhang, Qi Z Do we need to redesign the handshake to fix this?
Hi, Ferruh
Sorry for the late reply.
I did not receive your email on Oct 6, I got your comments from patchwork.Quoted 6 lines
"Can you please provide more details on multi-process communication and call trace, to help us think about a solution to address this issue in a more generic way (not just for pcap but for any case device close takes more than multi-process timeout)?"
I try to explain this issue with a sequence diagram, hope it can be displayed
correctly in the mail.
Quoted 11 lines
thread intr thread intr thread thread
of secondary of secondary of primary of primary
| | | |
| | | |rte_eal_hotplug_removerte_dev_removeeal_dev_hotplug_request_to_primaryrte_mp_request_sync------------------------------------------------------->||
handle_secondary_requestQuoted 14 lines
|<-----------------| | __handle_secondary_request eal_dev_hotplug_request_to_secondary |<------------------------------------- rte_mp_request_sync | handle_primary_request--------->| | __handle_primary_request local_dev_remove(this will take long time) rte_mp_reply -------------------------------->| | local_dev_remove |<-------------------------------------------------Quoted 3 lines
rte_mp_replyThe marked '
local_dev_remove()' in the secondary process will perform apcap file synchronization operation.
Quoted 1 line
When the pcap file is too large, it will take a lot of time (according to my test
100G takes 20+ seconds).
Quoted 1 line
This caused the processing of
hot_plugmessage to time out.Hi Yiding,
Thanks for the information,
Right now all MP operations timeout is hardcoded in the code and it is 5 seconds.
Do you think does it work to have an API to set custom timeout, something likerte_mp_timeout_set(), and call this from pdump?
Quoted 7 lines
Quoted 3 lines
This gives a generic solution for similar cases, not just for pcap. But my concern is if this is too much multi-process related internal detail to update, @Anatoly may comment on this.
Hi, Ferruh
For pdump case only, I think the timeout is affected by pcap's size and other system components, such as the type of FS, system memory size. It may be difficult to predict the specific time value for setting.
It doesn't have to be specific.
Point here is to have a multi process API to set timeout, instead of put a hardcoded timeout in pcap PMD.
RE: [PATCH v2] net/pcap: fix timeout of stopping device
Quoted 87 lines
Quoted 30 lines
Quoted 30 lines
Quoted 30 lines
Quoted 13 lines
Quoted 13 lines
Quoted 10 lines · On Tue, 6 Sep 2022 16:05:11 +0800 Yiding Zhou <mailto:[email protected]>…
The pcap file will be synchronized to the disk when stopping the device. It takes a long time if the file is large that would cause the 'detach sync request' timeout when the device is closed under multi-process scenario.
This commit fixes the issue by using alarm handler to release dumper.
Fixes 0ecfb6c04d54 ("net/pcap: move handler to process private")Cc mailto:[email protected]Quoted 7 lines
Quoted 5 lines
Quoted 1 line
Signed-off-by Yiding Zhou <mailto:[email protected]>I think you need to redesign the handshake if this the case. Forcing 30 second delay at the end of all uses of pcap is not acceptable.
@Zhang, Qi Z Do we need to redesign the handshake to fix this?
Hi, Ferruh
Sorry for the late reply.
I did not receive your email on Oct 6, I got your comments from patchwork."Can you please provide more details on multi-process communication and call trace, to help us think about a solution to address this issue in a more generic way (not just for pcap but for any case device close takes more than multi-process timeout)?"
Quoted 6 lines
Quoted 6 lines
Quoted 2 lines
I try to explain this issue with a sequence diagram, hope it can be displayed
correctly in the mail.
Quoted 3 lines
thread intr thread intr thread thread
of secondary of secondary of primary ofprimary
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
| | | | | | | |
rte_eal_hotplug_removerte_dev_removeeal_dev_hotplug_request_to_primaryrte_mp_request_sync------------------------------------------------------->|Quoted 7 lines
Quoted 7 lines
Quoted 2 lines
|
handle_secondary_requestQuoted 4 lines
|<-----------------| |
__handle_secondary_request
eal_dev_hotplug_request_to_secondaryQuoted 29 lines
Quoted 29 lines
Quoted 10 lines
|<------------------------------------- rte_mp_request_sync | handle_primary_request--------->| | __handle_primary_request local_dev_remove(this will take long time) rte_mp_reply -------------------------------->| | local_dev_remove |<-------------------------------------------------Quoted 4 lines
rte_mp_replyThe marked '
local_dev_remove()' in the secondary process will perform apcap file synchronization operation.
Quoted 2 lines
When the pcap file is too large, it will take a lot of time (according to my test
100G takes 20+ seconds).
Quoted 1 line
This caused the processing of
hot_plugmessage to time out.Hi Yiding,
Thanks for the information,
Right now all MP operations timeout is hardcoded in the code and it is 5 seconds.
Do you think does it work to have an API to set custom timeout, something likerte_mp_timeout_set(), and call this from pdump?Quoted 6 lines
Quoted 3 lines
This gives a generic solution for similar cases, not just for pcap. But my concern is if this is too much multi-process related internal detail to update, @Anatoly may comment on this.
Hi, Ferruh
For pdump case only, I think the timeout is affected by pcap's size and other
Quoted 7 lines
system components, such as the type of FS, system memory size.
Quoted 1 line
It may be difficult to predict the specific time value for setting.
It doesn't have to be specific.
Point here is to have a multi process API to set timeout, instead of put a hardcoded timeout in pcap PMD.
OK, I understood.