Skip to contentSkip to conversation

net/pcap: reduce time for stopping device

PATCH In review Yiding Zhou 4 participants 19 messages Aug 25, 2022 – Dec 5, 2022 Network drivers

lore
Share & tools
lore
#0 lore raw

PATCH 1/1 net/pcap: reduce time for stopping device

The pcap file will be synchronized to the disk when stopping the device. It takes a long time if the file is large that would cause the 'detach sync request' timeout when the device is closed under multi-process scenario.

This commit fixes the issue by performing synchronization in Tx path

Fixes 4c173302c307 ("pcap: add new driver")
---
 drivers/net/pcap/pcap_ethdev.c | 18 ++++++++++++++++--
 1 file changed, 16 insertions(+), 2 deletions(-)
drivers/net/pcap/pcap_ethdev.c +16 -2
index ec29fd6bc5..52eafa5674 100644
@@ -3,7 +3,7 @@
  * Copyright(c) 2014 6WIND S.A.
  * All rights reserved.
  */
-
+#include <unistd.h>
 #include <time.h>
 
 #include <pcap.h>
@@ -38,6 +38,8 @@
 
 #define RTE_PMD_PCAP_MAX_QUEUES 16
 
+#define ETH_PCAP_SYNC_THRESHOLD 0x20000000
+
 static char errbuf[PCAP_ERRBUF_SIZE];
 static struct timespec start_time;
 static uint64_t start_cycles;
@@ -47,6 +49,8 @@ static uint8_t iface_idx;
 static uint64_t timestamp_rx_dynflag;
 static int timestamp_dynfield_offset = -1;
 
+RTE_DEFINE_PER_LCORE(uint64_t, _pcap_cached_bytes);
+
 struct queue_stat {
 	volatile unsigned long pkts;
 	volatile unsigned long bytes;
@@ -144,6 +148,16 @@ static struct rte_eth_link pmd_link = {
 
 RTE_LOG_REGISTER_DEFAULT(eth_pcap_logtype, NOTICE);
 
+static inline void
+pcap_dumper_data_sync(pcap_dumper_t *dumper, uint32_t bytes)
+{
+	RTE_PER_LCORE(_pcap_cached_bytes) += bytes;
+	if (unlikely(RTE_PER_LCORE(_pcap_cached_bytes) > ETH_PCAP_SYNC_THRESHOLD)) {
+		if (!fdatasync(fileno(pcap_dump_file(dumper))))
+			RTE_PER_LCORE(_pcap_cached_bytes) = 0;
+	}
+}
+
 static struct queue_missed_stat*
 queue_missed_stat_update(struct rte_eth_dev *dev, unsigned int qid)
 {
@@ -421,7 +435,7 @@ eth_pcap_tx_dumper(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts)
 	 * process stops and to make sure the pcap file is actually written,
 	 * we flush the pcap dumper within each burst.
 	 */
-	pcap_dump_flush(dumper);
+	pcap_dumper_data_sync(dumper, tx_bytes);
 	dumper_q->tx_stat.pkts += num_tx;
 	dumper_q->tx_stat.bytes += tx_bytes;
 	dumper_q->tx_stat.err_pkts += nb_pkts - num_tx;
Signature · 1 line
2.34.1
#1 lore raw
Quoted 64 lines · Yiding Zhou

The pcap file will be synchronized to the disk when stopping the device. It takes a long time if the file is large that would cause the 'detach sync request' timeout when the device is closed under multi-process scenario.

This commit fixes the issue by performing synchronization in Tx path

Fixes 4c173302c307 ("pcap: add new driver")
---
  drivers/net/pcap/pcap_ethdev.c | 18 ++++++++++++++++--
  1 file changed, 16 insertions(+), 2 deletions(-)
drivers/net/pcap/pcap_ethdev.c +15 -1
index ec29fd6bc5..52eafa5674 100644
@@ -3,7 +3,7 @@
   * Copyright(c) 2014 6WIND S.A.
   * All rights reserved.
   */
-
+#include <unistd.h>
  #include <time.h>
 
  #include <pcap.h>
@@ -38,6 +38,8 @@
 
  #define RTE_PMD_PCAP_MAX_QUEUES 16
 
+#define ETH_PCAP_SYNC_THRESHOLD 0x20000000
+
  static char errbuf[PCAP_ERRBUF_SIZE];
  static struct timespec start_time;
  static uint64_t start_cycles;
@@ -47,6 +49,8 @@ static uint8_t iface_idx;
  static uint64_t timestamp_rx_dynflag;
  static int timestamp_dynfield_offset = -1;
 
+RTE_DEFINE_PER_LCORE(uint64_t, _pcap_cached_bytes);
+
  struct queue_stat {
  	volatile unsigned long pkts;
  	volatile unsigned long bytes;
@@ -144,6 +148,16 @@ static struct rte_eth_link pmd_link = {
 
  RTE_LOG_REGISTER_DEFAULT(eth_pcap_logtype, NOTICE);
 
+static inline void
+pcap_dumper_data_sync(pcap_dumper_t *dumper, uint32_t bytes)
+{
+	RTE_PER_LCORE(_pcap_cached_bytes) += bytes;
+	if (unlikely(RTE_PER_LCORE(_pcap_cached_bytes) > ETH_PCAP_SYNC_THRESHOLD)) {
+		if (!fdatasync(fileno(pcap_dump_file(dumper))))
+			RTE_PER_LCORE(_pcap_cached_bytes) = 0;
+	}
+}
+
  static struct queue_missed_stat*
  queue_missed_stat_update(struct rte_eth_dev *dev, unsigned int qid)
  {

drivers/net/pcap/pcap_ethdev.c eth_pcap_tx_dumper(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts) · line 438

Quoted 5 lines
drivers/net/pcap/pcap_ethdev.c +1 -1
@@ -421,7 +435,7 @@ eth_pcap_tx_dumper(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts)
  	 * process stops and to make sure the pcap file is actually written,
  	 * we flush the pcap dumper within each burst.
  	 */
-	pcap_dump_flush(dumper);
+	pcap_dumper_data_sync(dumper, tx_bytes);

'pcap_dump_flush()' should be doing the same thing, to write buffer to file, isn't it working?

Can you check the return value of the 'pcap_dump_flush()' API, I wonder if it keeps failing, for some reason?

Quoted 3 lines
drivers/net/pcap/pcap_ethdev.c +0 -0
  	dumper_q->tx_stat.pkts += num_tx;
  	dumper_q->tx_stat.bytes += tx_bytes;
  	dumper_q->tx_stat.err_pkts += nb_pkts - num_tx;
#2 lore raw
Quoted 84 lines

-----Original Message-----
From: Ferruh Yigit <[email protected]>
Sent: Thursday, August 25, 2022 6:09 PM
To: Zhou, YidingX <[email protected]>; [email protected]
Cc: Zhang, Qi Z <[email protected]>; [email protected]
Subject: Re: [PATCH] net/pcap: reduce time for stopping device

Quoted 47 lines · Yiding Zhou

The pcap file will be synchronized to the disk when stopping the device. It takes a long time if the file is large that would cause the 'detach sync request' timeout when the device is closed under multi-process scenario.

This commit fixes the issue by performing synchronization in Tx path

Fixes 4c173302c307 ("pcap: add new driver")
---
  drivers/net/pcap/pcap_ethdev.c | 18 ++++++++++++++++--
  1 file changed, 16 insertions(+), 2 deletions(-)
(quoted diff) +0 -0

b/drivers/net/pcap/pcap_ethdev.c index ec29fd6bc5..52eafa5674 100644

drivers/net/pcap/pcap_ethdev.c +5 -1
@@ -3,7 +3,7 @@
   * Copyright(c) 2014 6WIND S.A.
   * All rights reserved.
   */
-
+#include <unistd.h>
  #include <time.h>
 
  #include <pcap.h>
@@ -38,6 +38,8 @@
 
  #define RTE_PMD_PCAP_MAX_QUEUES 16
 
+#define ETH_PCAP_SYNC_THRESHOLD 0x20000000
+
  static char errbuf[PCAP_ERRBUF_SIZE];
  static struct timespec start_time;
  static uint64_t start_cycles;
@@ -47,6 +49,8 @@ static uint8_t iface_idx;
  static uint64_t timestamp_rx_dynflag;
  static int timestamp_dynfield_offset = -1;
 
+RTE_DEFINE_PER_LCORE(uint64_t, _pcap_cached_bytes);
+
  struct queue_stat {
  	volatile unsigned long pkts;
  	volatile unsigned long bytes;

drivers/net/pcap/pcap_ethdev.c static struct rte_eth_link pmd_link = { · line 154

Quoted 7 lines
drivers/net/pcap/pcap_ethdev.c +4 -0
@@ -144,6 +148,16 @@ static struct rte_eth_link pmd_link = {
 
  RTE_LOG_REGISTER_DEFAULT(eth_pcap_logtype, NOTICE);
 
+static inline void
+pcap_dumper_data_sync(pcap_dumper_t *dumper, uint32_t bytes) {
+	RTE_PER_LCORE(_pcap_cached_bytes) += bytes;
+	if (unlikely(RTE_PER_LCORE(_pcap_cached_bytes) >

ETH_PCAP_SYNC_THRESHOLD)) {

drivers/net/pcap/pcap_ethdev.c eth_pcap_tx_dumper(void *queue, struct rte_mbuf

Quoted 9 lines
drivers/net/pcap/pcap_ethdev.c +5 -0
+		if (!fdatasync(fileno(pcap_dump_file(dumper))))
+			RTE_PER_LCORE(_pcap_cached_bytes) = 0;
+	}
+}
+
  static struct queue_missed_stat*
  queue_missed_stat_update(struct rte_eth_dev *dev, unsigned int qid)
  {
@@ -421,7 +435,7 @@ eth_pcap_tx_dumper(void *queue, struct rte_mbuf

**bufs, uint16_t nb_pkts)

drivers/net/pcap/pcap_ethdev.c eth_pcap_tx_dumper(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts) · line 438

Quoted 5 lines
drivers/net/pcap/pcap_ethdev.c +1 -1
  	 * process stops and to make sure the pcap file is actually written,
  	 * we flush the pcap dumper within each burst.
  	 */
-	pcap_dump_flush(dumper);
+	pcap_dumper_data_sync(dumper, tx_bytes);

'pcap_dump_flush()' should be doing the same thing, to write buffer to file, isn't it working?

Quoted 3 lines

Can you check the return value of the 'pcap_dump_flush()' API, I wonder if it keeps failing, for some reason?

'pcap_dump_flush()' returns 0 each time without error, it calls 'fflush()' to flush userspace buffers to kernel buffers, not disk. 'fdatasync()' to ensure data is written to disk.

Quoted 3 lines
Quoted 3 lines
drivers/net/pcap/pcap_ethdev.c +0 -0
  	dumper_q->tx_stat.pkts += num_tx;
  	dumper_q->tx_stat.bytes += tx_bytes;
  	dumper_q->tx_stat.err_pkts += nb_pkts - num_tx;
#3 lore raw
Quoted 37 lines · YidingX
Quoted 37 lines

-----Original Message-----
From: Ferruh Yigit <[email protected]>
Sent: Thursday, August 25, 2022 6:09 PM
To: Zhou, YidingX <[email protected]>; [email protected]
Cc: Zhang, Qi Z <[email protected]>; [email protected]
Subject: Re: [PATCH] net/pcap: reduce time for stopping device

Quoted 27 lines · Yiding Zhou

The pcap file will be synchronized to the disk when stopping the device. It takes a long time if the file is large that would cause the 'detach sync request' timeout when the device is closed under multi-process scenario.

This commit fixes the issue by performing synchronization in Tx path

Fixes 4c173302c307 ("pcap: add new driver")
---
   drivers/net/pcap/pcap_ethdev.c | 18 ++++++++++++++++--
   1 file changed, 16 insertions(+), 2 deletions(-)
(quoted diff) +0 -0

b/drivers/net/pcap/pcap_ethdev.c index ec29fd6bc5..52eafa5674 100644

drivers/net/pcap/pcap_ethdev.c +1 -1
@@ -3,7 +3,7 @@
    * Copyright(c) 2014 6WIND S.A.
    * All rights reserved.
    */
-
+#include <unistd.h>
   #include <time.h>
 
Quoted 7 lines
Quoted 7 lines

drivers/net/pcap/pcap_ethdev.c line 42

Quoted 7 lines
drivers/net/pcap/pcap_ethdev.c +2 -0
   #include <pcap.h>
@@ -38,6 +38,8 @@
 
   #define RTE_PMD_PCAP_MAX_QUEUES 16
 
+#define ETH_PCAP_SYNC_THRESHOLD 0x20000000
+

I guess this is 512MB, can you please comment this.
Is there any specific reason to select this value, or is it arbitrary?

Quoted 14 lines
Quoted 14 lines
Quoted 14 lines
drivers/net/pcap/pcap_ethdev.c +2 -0
   static char errbuf[PCAP_ERRBUF_SIZE];
   static struct timespec start_time;
   static uint64_t start_cycles;
@@ -47,6 +49,8 @@ static uint8_t iface_idx;
   static uint64_t timestamp_rx_dynflag;
   static int timestamp_dynfield_offset = -1;
 
+RTE_DEFINE_PER_LCORE(uint64_t, _pcap_cached_bytes);
+
   struct queue_stat {
   	volatile unsigned long pkts;
   	volatile unsigned long bytes;
@@ -144,6 +148,16 @@ static struct rte_eth_link pmd_link = {
 
Quoted 4 lines
Quoted 4 lines

drivers/net/pcap/pcap_ethdev.c static struct rte_eth_link pmd_link = {

Quoted 4 lines
drivers/net/pcap/pcap_ethdev.c +2 -0
   RTE_LOG_REGISTER_DEFAULT(eth_pcap_logtype, NOTICE);
 
+static inline void
+pcap_dumper_data_sync(pcap_dumper_t *dumper, uint32_t bytes) {

'pcap_' is the namespace for the libpcap, can you select another prefix, like 'eth_pcap_' as many driver functions does.

Quoted 8 lines
Quoted 8 lines

drivers/net/pcap/pcap_ethdev.c static struct rte_eth_link pmd_link = { · line 154

Quoted 2 lines
drivers/net/pcap/pcap_ethdev.c +2 -0
+	RTE_PER_LCORE(_pcap_cached_bytes) += bytes;
+	if (unlikely(RTE_PER_LCORE(_pcap_cached_bytes) >

ETH_PCAP_SYNC_THRESHOLD)) {

drivers/net/pcap/pcap_ethdev.c static struct rte_eth_link pmd_link = { · line 160

Quoted 5 lines
drivers/net/pcap/pcap_ethdev.c +5 -0
+		if (!fdatasync(fileno(pcap_dump_file(dumper))))
+			RTE_PER_LCORE(_pcap_cached_bytes) = 0;
+	}
+}
+

pcap supports multiple queue, and each queue creates a new pcap dumper and single core/thread can be used for this multiple dumpers. In that case I think above per lcore variable logic doesn't work.

And instead of having a global value, what do you think to add a variable to 'struct pcap_tx_queue' for this purpose?

Quoted 14 lines
Quoted 14 lines

drivers/net/pcap/pcap_ethdev.c eth_pcap_tx_dumper(void *queue, struct rte_mbuf

Quoted 4 lines
drivers/net/pcap/pcap_ethdev.c +0 -0
   static struct queue_missed_stat*
   queue_missed_stat_update(struct rte_eth_dev *dev, unsigned int qid)
   {
@@ -421,7 +435,7 @@ eth_pcap_tx_dumper(void *queue, struct rte_mbuf

**bufs, uint16_t nb_pkts)

drivers/net/pcap/pcap_ethdev.c eth_pcap_tx_dumper(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts) · line 438

Quoted 5 lines
drivers/net/pcap/pcap_ethdev.c +1 -1
   	 * process stops and to make sure the pcap file is actually written,
   	 * we flush the pcap dumper within each burst.
   	 */
-	pcap_dump_flush(dumper);
+	pcap_dumper_data_sync(dumper, tx_bytes);

'pcap_dump_flush()' should be doing the same thing, to write buffer to file, isn't it working?

Quoted 6 lines
Quoted 3 lines

Can you check the return value of the 'pcap_dump_flush()' API, I wonder if it keeps failing, for some reason?

'pcap_dump_flush()' returns 0 each time without error, it calls 'fflush()' to flush userspace buffers to kernel buffers, not disk. 'fdatasync()' to ensure data is written to disk.

'pcap_dump_flush()' API documentation says "flushes the output buffer to the ``savefile,''", but as you said it uses 'fflush()' internally, so there is a chance that data is not written to the disk.

In this case, won't need to keep both, first flush and later fsync/fdatasync?

Do you observe any performance difference after change, since now writing to actual disk on datapath?

Quoted 4 lines
Quoted 3 lines
Quoted 3 lines
drivers/net/pcap/pcap_ethdev.c +0 -0
   	dumper_q->tx_stat.pkts += num_tx;
   	dumper_q->tx_stat.bytes += tx_bytes;
   	dumper_q->tx_stat.err_pkts += nb_pkts - num_tx;
#4 lore raw
Quoted 48 lines

-----Original Message-----
From: Ferruh Yigit <[email protected]>
Sent: Thursday, August 25, 2022 8:22 PM
To: Zhou, YidingX <[email protected]>; [email protected]
Cc: Zhang, Qi Z <[email protected]>; [email protected]
Subject: Re: [PATCH] net/pcap: reduce time for stopping device

Quoted 40 lines · YidingX
Quoted 40 lines

-----Original Message-----
From: Ferruh Yigit <[email protected]>
Sent: Thursday, August 25, 2022 6:09 PM
To: Zhou, YidingX <[email protected]>; [email protected]
Cc: Zhang, Qi Z <[email protected]>; [email protected]
Subject: Re: [PATCH] net/pcap: reduce time for stopping device

Quoted 30 lines · Yiding Zhou

The pcap file will be synchronized to the disk when stopping the device. It takes a long time if the file is large that would cause the 'detach sync request' timeout when the device is closed under multi-process scenario.

This commit fixes the issue by performing synchronization in Tx path

Fixes 4c173302c307 ("pcap: add new driver")
---
   drivers/net/pcap/pcap_ethdev.c | 18 ++++++++++++++++--
   1 file changed, 16 insertions(+), 2 deletions(-)
(quoted diff) +0 -0

b/drivers/net/pcap/pcap_ethdev.c index ec29fd6bc5..52eafa5674 100644

drivers/net/pcap/pcap_ethdev.c +1 -1
@@ -3,7 +3,7 @@
    * Copyright(c) 2014 6WIND S.A.
    * All rights reserved.
    */
-
+#include <unistd.h>
   #include <time.h>
 
   #include <pcap.h>
@@ -38,6 +38,8 @@
 
Quoted 9 lines
Quoted 4 lines
Quoted 4 lines

drivers/net/pcap/pcap_ethdev.c line 42

Quoted 4 lines
drivers/net/pcap/pcap_ethdev.c +2 -0
   #define RTE_PMD_PCAP_MAX_QUEUES 16
 
+#define ETH_PCAP_SYNC_THRESHOLD 0x20000000
+

I guess this is 512MB, can you please comment this.
Is there any specific reason to select this value, or is it arbitrary?

512M is arbitrary, because there is no API to get the disk cache size associated with a specific file. I will test the performance impact of different values.

Quoted 35 lines
Quoted 14 lines
Quoted 14 lines
Quoted 14 lines
drivers/net/pcap/pcap_ethdev.c +2 -0
   static char errbuf[PCAP_ERRBUF_SIZE];
   static struct timespec start_time;
   static uint64_t start_cycles;
@@ -47,6 +49,8 @@ static uint8_t iface_idx;
   static uint64_t timestamp_rx_dynflag;
   static int timestamp_dynfield_offset = -1;
 
+RTE_DEFINE_PER_LCORE(uint64_t, _pcap_cached_bytes);
+
   struct queue_stat {
   	volatile unsigned long pkts;
   	volatile unsigned long bytes;
@@ -144,6 +148,16 @@ static struct rte_eth_link pmd_link = {
 
Quoted 4 lines
Quoted 4 lines

drivers/net/pcap/pcap_ethdev.c static struct rte_eth_link pmd_link = {

Quoted 4 lines
drivers/net/pcap/pcap_ethdev.c +2 -0
   RTE_LOG_REGISTER_DEFAULT(eth_pcap_logtype, NOTICE);
 
+static inline void
+pcap_dumper_data_sync(pcap_dumper_t *dumper, uint32_t bytes) {

'pcap_' is the namespace for the libpcap, can you select another prefix, like 'eth_pcap_' as many driver functions does.

Quoted 8 lines
Quoted 8 lines

drivers/net/pcap/pcap_ethdev.c static struct rte_eth_link pmd_link = { · line 154

Quoted 2 lines
drivers/net/pcap/pcap_ethdev.c +2 -0
+	RTE_PER_LCORE(_pcap_cached_bytes) += bytes;
+	if (unlikely(RTE_PER_LCORE(_pcap_cached_bytes) >

ETH_PCAP_SYNC_THRESHOLD)) {

drivers/net/pcap/pcap_ethdev.c static struct rte_eth_link pmd_link = { · line 160

Quoted 5 lines
drivers/net/pcap/pcap_ethdev.c +5 -0
+		if (!fdatasync(fileno(pcap_dump_file(dumper))))
+			RTE_PER_LCORE(_pcap_cached_bytes) = 0;
+	}
+}
+

pcap supports multiple queue, and each queue creates a new pcap dumper and single core/thread can be used for this multiple dumpers. In that case I think above per lcore variable logic doesn't work.

Quoted 3 lines

And instead of having a global value, what do you think to add a variable to 'struct pcap_tx_queue' for this purpose?

Thanks for the comments, I will follow this.

Quoted 23 lines
Quoted 14 lines
Quoted 14 lines

drivers/net/pcap/pcap_ethdev.c eth_pcap_tx_dumper(void *queue, struct rte_mbuf

Quoted 4 lines
drivers/net/pcap/pcap_ethdev.c +0 -0
   static struct queue_missed_stat*
   queue_missed_stat_update(struct rte_eth_dev *dev, unsigned int qid)
   {
@@ -421,7 +435,7 @@ eth_pcap_tx_dumper(void *queue, struct rte_mbuf

**bufs, uint16_t nb_pkts)

drivers/net/pcap/pcap_ethdev.c eth_pcap_tx_dumper(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts) · line 438

Quoted 5 lines
drivers/net/pcap/pcap_ethdev.c +1 -1
   	 * process stops and to make sure the pcap file is actually written,
   	 * we flush the pcap dumper within each burst.
   	 */
-	pcap_dump_flush(dumper);
+	pcap_dumper_data_sync(dumper, tx_bytes);

'pcap_dump_flush()' should be doing the same thing, to write buffer to file, isn't it working?

Quoted 5 lines
Quoted 3 lines

Can you check the return value of the 'pcap_dump_flush()' API, I wonder if it keeps failing, for some reason?

'pcap_dump_flush()' returns 0 each time without error, it calls 'fflush()' to

flush userspace buffers to kernel buffers, not disk. 'fdatasync()' to ensure data is written to disk.

Quoted 1 line
Quoted 6 lines

'pcap_dump_flush()' API documentation says "flushes the output buffer to the ``savefile,''", but as you said it uses 'fflush()' internally, so there is a chance that data is not written to the disk.

In this case, won't need to keep both, first flush and later fsync/fdatasync?

I draw a diagram to describe it more clearly

fwrite fclose/fflush fclose/fdatasync
--------->| libc buffer |----------------> | disk cache in RAM |---------------------> |disk|
| 4096 Bytes | | size is determined by OS | | |

When the libc buffer is full, the system will automatically sync it to the disk cache. It is easily full as it's only 4096 B size. so there is no need to call 'fflush()' every time. The real time consuming action is syncing the disk cache to disk.
Because the disk cache is very large, it will take a long time to sync all at one time during 'fclose()', so need to call 'fdatasync()' periodically to amortize the time.

Quoted 3 lines

Do you observe any performance difference after change, since now writing to actual disk on datapath?

I will verify any performance difference under these 2 scenarios.

Quoted 4 lines
Quoted 4 lines
Quoted 3 lines
Quoted 3 lines
drivers/net/pcap/pcap_ethdev.c +0 -0
   	dumper_q->tx_stat.pkts += num_tx;
   	dumper_q->tx_stat.bytes += tx_bytes;
   	dumper_q->tx_stat.err_pkts += nb_pkts - num_tx;
#5 lore raw
Quoted 11 lines · YidingX

I draw a diagram to describe it more clearly

fwrite fclose/fflush fclose/fdatasync
--------->| libc buffer |----------------> | disk cache in RAM |---------------------> |disk|
| 4096 Bytes | | size is determined by OS | | |

When the libc buffer is full, the system will automatically sync it to the disk cache. It is easily full as it's only 4096 B size. so there is no need to call 'fflush()' every time. The real time consuming action is syncing the disk cache to disk.
Because the disk cache is very large, it will take a long time to sync all at one time during 'fclose()', so need to call 'fdatasync()' periodically to amortize the time.

If you want to speed up this, then get rid of stdio and use a faster API like io_uring. What you tried can help but using a better API would make bigger gains.

#6 lore raw
In reply to Zhou, YidingX

Hi, Ferruh

Quoted 136 lines
Quoted 32 lines
Quoted 32 lines
Quoted 32 lines
Quoted 32 lines

The pcap file will be synchronized to the disk when stopping the device. It takes a long time if the file is large that would cause the 'detach sync request' timeout when the device is closed under multi-process scenario.

This commit fixes the issue by performing synchronization in Tx path

Fixes 4c173302c307 ("pcap: add new driver")
---
   drivers/net/pcap/pcap_ethdev.c | 18 ++++++++++++++++--
   1 file changed, 16 insertions(+), 2 deletions(-)
(quoted diff) +0 -0

b/drivers/net/pcap/pcap_ethdev.c index ec29fd6bc5..52eafa5674 100644

drivers/net/pcap/pcap_ethdev.c +1 -1
@@ -3,7 +3,7 @@
    * Copyright(c) 2014 6WIND S.A.
    * All rights reserved.
    */
-
+#include <unistd.h>
   #include <time.h>
 
   #include <pcap.h>
@@ -38,6 +38,8 @@
 
Quoted 9 lines
Quoted 4 lines
Quoted 4 lines

drivers/net/pcap/pcap_ethdev.c line 42

Quoted 4 lines
drivers/net/pcap/pcap_ethdev.c +2 -0
   #define RTE_PMD_PCAP_MAX_QUEUES 16
 
+#define ETH_PCAP_SYNC_THRESHOLD 0x20000000
+

I guess this is 512MB, can you please comment this.
Is there any specific reason to select this value, or is it arbitrary?

512M is arbitrary, because there is no API to get the disk cache size associated with a specific file.
I will test the performance impact of different values.

Quoted 35 lines
Quoted 14 lines
Quoted 14 lines
Quoted 14 lines
drivers/net/pcap/pcap_ethdev.c +2 -0
   static char errbuf[PCAP_ERRBUF_SIZE];
   static struct timespec start_time;
   static uint64_t start_cycles;
@@ -47,6 +49,8 @@ static uint8_t iface_idx;
   static uint64_t timestamp_rx_dynflag;
   static int timestamp_dynfield_offset = -1;
 
+RTE_DEFINE_PER_LCORE(uint64_t, _pcap_cached_bytes);
+
   struct queue_stat {
   	volatile unsigned long pkts;
   	volatile unsigned long bytes;
@@ -144,6 +148,16 @@ static struct rte_eth_link pmd_link = {
 
Quoted 4 lines
Quoted 4 lines

drivers/net/pcap/pcap_ethdev.c static struct rte_eth_link pmd_link = {

Quoted 4 lines
drivers/net/pcap/pcap_ethdev.c +2 -0
   RTE_LOG_REGISTER_DEFAULT(eth_pcap_logtype, NOTICE);
 
+static inline void
+pcap_dumper_data_sync(pcap_dumper_t *dumper, uint32_t bytes) {

'pcap_' is the namespace for the libpcap, can you select another prefix, like 'eth_pcap_' as many driver functions does.

Quoted 8 lines
Quoted 8 lines

drivers/net/pcap/pcap_ethdev.c static struct rte_eth_link pmd_link = { · line 154

Quoted 2 lines
drivers/net/pcap/pcap_ethdev.c +2 -0
+	RTE_PER_LCORE(_pcap_cached_bytes) += bytes;
+	if (unlikely(RTE_PER_LCORE(_pcap_cached_bytes) >

ETH_PCAP_SYNC_THRESHOLD)) {

drivers/net/pcap/pcap_ethdev.c static struct rte_eth_link pmd_link = { · line 160

Quoted 5 lines
drivers/net/pcap/pcap_ethdev.c +5 -0
+		if (!fdatasync(fileno(pcap_dump_file(dumper))))
+			RTE_PER_LCORE(_pcap_cached_bytes) = 0;
+	}
+}
+

pcap supports multiple queue, and each queue creates a new pcap dumper and single core/thread can be used for this multiple dumpers. In that case I think above per lcore variable logic doesn't work.

Quoted 3 lines

And instead of having a global value, what do you think to add a variable to 'struct pcap_tx_queue' for this purpose?

Thanks for the comments, I will follow this.

Quoted 25 lines
Quoted 15 lines
Quoted 15 lines
Quoted 5 lines
drivers/net/pcap/pcap_ethdev.c +0 -0
   static struct queue_missed_stat*
   queue_missed_stat_update(struct rte_eth_dev *dev, unsigned int qid)
   {
@@ -421,7 +435,7 @@ eth_pcap_tx_dumper(void *queue, struct

rte_mbuf

**bufs, uint16_t nb_pkts)

drivers/net/pcap/pcap_ethdev.c eth_pcap_tx_dumper(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts) · line 438

Quoted 5 lines

* process stops and to make sure the pcap file is actually written,
* we flush the pcap dumper within each burst.
*/

drivers/net/pcap/pcap_ethdev.c +1 -1
-	pcap_dump_flush(dumper);
+	pcap_dumper_data_sync(dumper, tx_bytes);

'pcap_dump_flush()' should be doing the same thing, to write buffer to file, isn't it working?

Quoted 6 lines
Quoted 3 lines

Can you check the return value of the 'pcap_dump_flush()' API, I wonder if it keeps failing, for some reason?

'pcap_dump_flush()' returns 0 each time without error, it calls 'fflush()' to

flush userspace buffers to kernel buffers, not disk. 'fdatasync()' to ensure data is written to disk.

Quoted 1 line
Quoted 6 lines

'pcap_dump_flush()' API documentation says "flushes the output buffer to the ``savefile,''", but as you said it uses 'fflush()' internally, so there is a chance that data is not written to the disk.

In this case, won't need to keep both, first flush and later fsync/fdatasync?

I draw a diagram to describe it more clearly

fwrite fclose/fflush fclose/fdatasync
--------->| libc buffer |----------------> | disk cache in RAM |--------------------->

|disk|
              | 4096 Bytes |                         | size is determined by OS |
|       |

When the libc buffer is full, the system will automatically sync it to the disk cache.
It is easily full as it's only 4096 B size. so there is no need to call 'fflush()' every time.
The real time consuming action is syncing the disk cache to disk.
Because the disk cache is very large, it will take a long time to sync all at one time during 'fclose()', so need to call 'fdatasync()' periodically to amortize the time.

Quoted 6 lines
Quoted 3 lines

Do you observe any performance difference after change, since now writing to actual disk on datapath?

I will verify any performance difference under these 2 scenarios.

I have done some tests, using 'fdatasync()' will cause the packet loss rate to increase by 15%-20%, It's not a good solution, need to rework. I plan to use a timer to call pcap_dump_close to fix this, do you have any suggestion?
thanks

Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 3 lines
Quoted 3 lines
   	dumper_q->tx_stat.pkts += num_tx;
   	dumper_q->tx_stat.bytes += tx_bytes;
   	dumper_q->tx_stat.err_pkts += nb_pkts - num_tx;
#7 lore raw

PATCH v2 1/1 net/pcap: fix timeout of stopping device

In reply to Yiding Zhou

The pcap file will be synchronized to the disk when stopping the device. It takes a long time if the file is large that would cause the 'detach sync request' timeout when the device is closed under multi-process scenario.

This commit fixes the issue by using alarm handler to release dumper.

Fixes 0ecfb6c04d54 ("net/pcap: move handler to process private")

---
v2: use alarm handler to release dumper

---
 drivers/net/pcap/pcap_ethdev.c | 22 +++++++++++++++++++++-
 1 file changed, 21 insertions(+), 1 deletion(-)
drivers/net/pcap/pcap_ethdev.c +21 -1
index ec29fd6bc5..5c643a0277 100644
@@ -17,6 +17,7 @@
 #include <rte_mbuf_dyn.h>
 #include <rte_bus_vdev.h>
 #include <rte_os_shim.h>
+#include <rte_alarm.h>
 
 #include "pcap_osdep.h"
 
@@ -664,6 +665,25 @@ eth_dev_start(struct rte_eth_dev *dev)
 	return 0;
 }
 
+static void eth_pcap_dumper_release(void *arg)
+{
+	pcap_dump_close((pcap_dumper_t *)arg);
+}
+
+static void
+eth_pcap_dumper_close(pcap_dumper_t *dumper)
+{
+	if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
+		/*
+		 * Delay 30 seconds before releasing dumper to wait for file sync
+		 * to complete to avoid blocking alarm thread in PRIMARY process
+		 */
+		rte_eal_alarm_set(30000000, eth_pcap_dumper_release, dumper);
+	} else {
+		rte_eal_alarm_set(1, eth_pcap_dumper_release, dumper);
+	}
+}
+
 /*
  * This function gets called when the current port gets stopped.
  * Is the only place for us to close all the tx streams dumpers.
@@ -689,7 +709,7 @@ eth_dev_stop(struct rte_eth_dev *dev)
 
 	for (i = 0; i < dev->data->nb_tx_queues; i++) {
 		if (pp->tx_dumper[i] != NULL) {
-			pcap_dump_close(pp->tx_dumper[i]);
+			eth_pcap_dumper_close(pp->tx_dumper[i]);
 			pp->tx_dumper[i] = NULL;
 		}
 
Signature · 1 line
2.34.1
#8 lore raw

Re: [PATCH v2] net/pcap: fix timeout of stopping device

Quoted 11 lines · On Tue, 6 Sep 2022 16:05:11 +0800 Yiding Zhou <[email protected]> wrote:

The pcap file will be synchronized to the disk when stopping the device. It takes a long time if the file is large that would cause the 'detach sync request' timeout when the device is closed under multi-process scenario.

This commit fixes the issue by using alarm handler to release dumper.

Fixes 0ecfb6c04d54 ("net/pcap: move handler to process private")

I think you need to redesign the handshake if this the case. Forcing 30 second delay at the end of all uses of pcap is not acceptable.

#9 lore raw

RE: [PATCH v2] net/pcap: fix timeout of stopping device

Quoted 11 lines · On Tue, 6 Sep 2022 16:05:11 +0800 Yiding Zhou <[email protected]> wrote:

The pcap file will be synchronized to the disk when stopping the device. It takes a long time if the file is large that would cause the 'detach sync request' timeout when the device is closed under multi-process scenario.

This commit fixes the issue by using alarm handler to release dumper.

Fixes 0ecfb6c04d54 ("net/pcap: move handler to process private")

I think you need to redesign the handshake if this the case. Forcing 30 second delay at the end of all uses of pcap is not acceptable.

Thanks for your comments.
According to my test, the time required to sync a 100G pcap file is about 20s,
so I set a delay of 30s. I also tried to use AIO, but some issue in the multi-process scenario. And I also consider io_uring, it is only supported in linux-5.x, we need to consider compatibility,. Maybe better way is to do more work to redesign the handshake.

#10 lore raw

RE: [PATCH v2] net/pcap: fix timeout of stopping device

In reply to Stephen Hemminger
Quoted 21 lines

-----Original Message-----
From: Stephen Hemminger <[email protected]>
Sent: Tuesday, September 6, 2022 10:58 PM
To: Zhou, YidingX <[email protected]>
Cc: [email protected]; Zhang, Qi Z <[email protected]>; Burakov, Anatoly <[email protected]>; He, Xingguang <[email protected]>; [email protected]
Subject: Re: [PATCH v2] net/pcap: fix timeout of stopping device

Quoted 7 lines · On Tue, 6 Sep 2022 16:05:11 +0800 Yiding Zhou <[email protected]> wrote:

The pcap file will be synchronized to the disk when stopping the device. It takes a long time if the file is large that would cause the 'detach sync request' timeout when the device is closed under multi-process scenario.

This commit fixes the issue by using alarm handler to release dumper.

Quoted 8 lines
Quoted 4 lines
Fixes 0ecfb6c04d54 ("net/pcap: move handler to process private")

I think you need to redesign the handshake if this the case. Forcing 30 second delay at the end of all uses of pcap is not acceptable.

@Zhang, Qi Z Do we need to redesign the handshake to fix this?

#11 lore raw

Re: [PATCH v2] net/pcap: fix timeout of stopping device

Quoted 24 lines · YidingX
Quoted 24 lines

-----Original Message-----
From: Stephen Hemminger <[email protected]>
Sent: Tuesday, September 6, 2022 10:58 PM
To: Zhou, YidingX <[email protected]>
Cc: [email protected]; Zhang, Qi Z <[email protected]>; Burakov, Anatoly <[email protected]>; He, Xingguang <[email protected]>; [email protected]
Subject: Re: [PATCH v2] net/pcap: fix timeout of stopping device

Quoted 10 lines · On Tue, 6 Sep 2022 16:05:11 +0800 Yiding Zhou <[email protected]> wrote:

The pcap file will be synchronized to the disk when stopping the device. It takes a long time if the file is large that would cause the 'detach sync request' timeout when the device is closed under multi-process scenario.

This commit fixes the issue by using alarm handler to release dumper.

Fixes 0ecfb6c04d54 ("net/pcap: move handler to process private")
Quoted 7 lines
Quoted 5 lines
Quoted 1 line

I think you need to redesign the handshake if this the case. Forcing 30 second delay at the end of all uses of pcap is not acceptable.

@Zhang, Qi Z Do we need to redesign the handshake to fix this?

Hi Yiding,

Your approach works, but Stephen's comment is to not add this delay by default, because this delay is not always required.

Can you please provide more details on multi-process communication and call trace, to help us think about a solution to address this issue in a more generic way (not just for pcap but for any case device close takes more than multi-process timeout)?

#12 lore raw

RE: [PATCH v2] net/pcap: fix timeout of stopping device

In reply to Zhou, YidingX
Quoted 35 lines

-----Original Message-----
From: Zhou, YidingX <[email protected]>
Sent: Wednesday, September 21, 2022 3:15 PM
To: Stephen Hemminger <[email protected]>; Zhang, Qi Z <[email protected]>
Cc: [email protected]; Burakov, Anatoly <[email protected]>; He, Xingguang <[email protected]>; [email protected]
Subject: RE: [PATCH v2] net/pcap: fix timeout of stopping device

Quoted 22 lines

-----Original Message-----
From: Stephen Hemminger <mailto:[email protected]>
Sent: Tuesday, September 6, 2022 10:58 PM
To: Zhou, YidingX <mailto:[email protected]>
Cc: mailto:[email protected]; Zhang, Qi Z <mailto:[email protected]>; Burakov, Anatoly <mailto:[email protected]>; He, Xingguang <mailto:[email protected]>; mailto:[email protected]
Subject: Re: [PATCH v2] net/pcap: fix timeout of stopping device

Quoted 10 lines · On Tue, 6 Sep 2022 16:05:11 +0800 Yiding Zhou <mailto:[email protected]>…

The pcap file will be synchronized to the disk when stopping the device. It takes a long time if the file is large that would cause the 'detach sync request' timeout when the device is closed under multi-process scenario.

This commit fixes the issue by using alarm handler to release dumper.

Fixes 0ecfb6c04d54 ("net/pcap: move handler to process private")
Quoted 7 lines
Quoted 5 lines
Quoted 1 line

I think you need to redesign the handshake if this the case. Forcing 30 second delay at the end of all uses of pcap is not acceptable.

@Zhang, Qi Z Do we need to redesign the handshake to fix this?

Hi, Ferruh
Sorry for the late reply.
I did not receive your email on Oct 6, I got your comments from patchwork.

"Can you please provide more details on multi-process communication and call trace, to help us think about a solution to address this issue in a more generic way (not just for pcap but for any case device close takes more than multi-process timeout)?"

I try to explain this issue with a sequence diagram, hope it can be displayed correctly in the mail.

thread intr thread intr thread thread
of secondary of secondary of primary of primary
| | | |
| | | |
rte_eal_hotplug_remove
rte_dev_remove
eal_dev_hotplug_request_to_primary
rte_mp_request_sync ------------------------------------------------------->|

                                                                                                                    |
                                                                                              handle_secondary_request
                                                                                         |<-----------------|
                                                                                         |
                                                                   __handle_secondary_request
                                                          eal_dev_hotplug_request_to_secondary
           |<------------------------------------- rte_mp_request_sync
           |
handle_primary_request--------->|
                                                           |
                            __handle_primary_request
                               local_dev_remove(this will take long time)
                                            rte_mp_reply -------------------------------->|                              
                                                                                         |
                                                                             local_dev_remove
          |<------------------------------------------------- rte_mp_reply

The marked 'local_dev_remove()' in the secondary process will perform a pcap file synchronization operation. When the pcap file is too large, it will take a lot of time (according to my test 100G takes 20+ seconds). This caused the processing of hot_plug message to time out.

#13 lore raw

Re: [PATCH v2] net/pcap: fix timeout of stopping device

Quoted 77 lines · YidingX
Quoted 33 lines

-----Original Message-----
From: Zhou, YidingX <[email protected]>
Sent: Wednesday, September 21, 2022 3:15 PM
To: Stephen Hemminger <[email protected]>; Zhang, Qi Z <[email protected]>
Cc: [email protected]; Burakov, Anatoly <[email protected]>; He, Xingguang <[email protected]>; [email protected]
Subject: RE: [PATCH v2] net/pcap: fix timeout of stopping device

Quoted 22 lines

-----Original Message-----
From: Stephen Hemminger <mailto:[email protected]>
Sent: Tuesday, September 6, 2022 10:58 PM
To: Zhou, YidingX <mailto:[email protected]>
Cc: mailto:[email protected]; Zhang, Qi Z <mailto:[email protected]>; Burakov, Anatoly <mailto:[email protected]>; He, Xingguang <mailto:[email protected]>; mailto:[email protected]
Subject: Re: [PATCH v2] net/pcap: fix timeout of stopping device

Quoted 10 lines · On Tue, 6 Sep 2022 16:05:11 +0800 Yiding Zhou <mailto:[email protected]>…

The pcap file will be synchronized to the disk when stopping the device. It takes a long time if the file is large that would cause the 'detach sync request' timeout when the device is closed under multi-process scenario.

This commit fixes the issue by using alarm handler to release dumper.

Fixes 0ecfb6c04d54 ("net/pcap: move handler to process private")
Quoted 7 lines
Quoted 5 lines
Quoted 1 line

I think you need to redesign the handshake if this the case. Forcing 30 second delay at the end of all uses of pcap is not acceptable.

@Zhang, Qi Z Do we need to redesign the handshake to fix this?

Hi, Ferruh
Sorry for the late reply.
I did not receive your email on Oct 6, I got your comments from patchwork.

"Can you please provide more details on multi-process communication and call trace, to help us think about a solution to address this issue in a more generic way (not just for pcap but for any case device close takes more than multi-process timeout)?"

I try to explain this issue with a sequence diagram, hope it can be displayed correctly in the mail.

thread intr thread intr thread thread
of secondary of secondary of primary of primary
| | | |
| | | |
rte_eal_hotplug_remove
rte_dev_remove
eal_dev_hotplug_request_to_primary
rte_mp_request_sync ------------------------------------------------------->|

                                                                                                                    |
                                                                                              handle_secondary_request
                                                                                         |<-----------------|
                                                                                         |
                                                                   __handle_secondary_request
                                                          eal_dev_hotplug_request_to_secondary
           |<------------------------------------- rte_mp_request_sync
           |
handle_primary_request--------->|
                                                           |
                            __handle_primary_request
                               local_dev_remove(this will take long time)
                                            rte_mp_reply -------------------------------->|                              
                                                                                         |
                                                                             local_dev_remove
          |<------------------------------------------------- rte_mp_reply
Quoted 3 lines

The marked 'local_dev_remove()' in the secondary process will perform a pcap file synchronization operation. When the pcap file is too large, it will take a lot of time (according to my test 100G takes 20+ seconds). This caused the processing of hot_plug message to time out.

Part of the problem maybe a hidden file sync in some library. Normally, closing a file should be fast even with lots of outstanding data. The actual write done by OS will continue from file cache.

I wonder if doing some kind of fadvise call might help see POSIX_FADV_SEQUENTIAL or POSIX_FADV_DONTNEED

#14 lore raw

RE: [PATCH v2] net/pcap: fix timeout of stopping device

Quoted 68 lines
Quoted 26 lines
Quoted 14 lines
Quoted 14 lines
Quoted 11 lines · On Tue, 6 Sep 2022 16:05:11 +0800 Yiding Zhou <mailto:[email protected]>…

The pcap file will be synchronized to the disk when stopping the device. It takes a long time if the file is large that would cause the 'detach sync request' timeout when the device is closed under multi-process scenario.

This commit fixes the issue by using alarm handler to release dumper.

Fixes 0ecfb6c04d54 ("net/pcap: move handler to process

private")
Cc: mailto:[email protected]

Quoted 7 lines
Quoted 5 lines
Quoted 1 line

I think you need to redesign the handshake if this the case. Forcing 30 second delay at the end of all uses of pcap is not acceptable.

@Zhang, Qi Z Do we need to redesign the handshake to fix this?

Hi, Ferruh
Sorry for the late reply.
I did not receive your email on Oct 6, I got your comments from patchwork.

Quoted 6 lines

"Can you please provide more details on multi-process communication and call trace, to help us think about a solution to address this issue in a more generic way (not just for pcap but for any case device close takes more than multi-process timeout)?"

I try to explain this issue with a sequence diagram, hope it can be displayed

correctly in the mail.

Quoted 11 lines

thread intr thread intr thread thread
of secondary of secondary of primary of primary
| | | |
| | | |
rte_eal_hotplug_remove
rte_dev_remove
eal_dev_hotplug_request_to_primary
rte_mp_request_sync ------------------------------------------------------->|

                                                                                                                    |

handle_secondary_request

Quoted 14 lines
                                                                                         |<-----------------|
                                                                                         |
                                                                   __handle_secondary_request
                                                          eal_dev_hotplug_request_to_secondary
           |<------------------------------------- rte_mp_request_sync
           |
handle_primary_request--------->|
                                                           |
                            __handle_primary_request
                               local_dev_remove(this will take long time)
                                            rte_mp_reply -------------------------------->|
                                                                                         |
                                                                             local_dev_remove
          |<-------------------------------------------------
Quoted 3 lines

rte_mp_reply

The marked 'local_dev_remove()' in the secondary process will perform a

pcap file synchronization operation.

Quoted 1 line

When the pcap file is too large, it will take a lot of time (according to my test

100G takes 20+ seconds).

Quoted 1 line

This caused the processing of hot_plug message to time out.

Quoted 6 lines

Part of the problem maybe a hidden file sync in some library. Normally, closing a file should be fast even with lots of outstanding data. The actual write done by OS will continue from file cache.

I wonder if doing some kind of fadvise call might help see POSIX_FADV_SEQUENTIAL or POSIX_FADV_DONTNEED

Thanks for your advice, I will try it and give you feedback.

#15 lore raw

Re: [PATCH v2] net/pcap: fix timeout of stopping device

In reply to Zhou, YidingX
Quoted 79 lines · YidingX
Quoted 35 lines

-----Original Message-----
From: Zhou, YidingX <[email protected]>
Sent: Wednesday, September 21, 2022 3:15 PM
To: Stephen Hemminger <[email protected]>; Zhang, Qi Z <[email protected]>
Cc: [email protected]; Burakov, Anatoly <[email protected]>; He, Xingguang <[email protected]>; [email protected]
Subject: RE: [PATCH v2] net/pcap: fix timeout of stopping device

Quoted 22 lines

-----Original Message-----
From: Stephen Hemminger <mailto:[email protected]>
Sent: Tuesday, September 6, 2022 10:58 PM
To: Zhou, YidingX <mailto:[email protected]>
Cc: mailto:[email protected]; Zhang, Qi Z <mailto:[email protected]>; Burakov, Anatoly <mailto:[email protected]>; He, Xingguang <mailto:[email protected]>; mailto:[email protected]
Subject: Re: [PATCH v2] net/pcap: fix timeout of stopping device

Quoted 10 lines · On Tue, 6 Sep 2022 16:05:11 +0800 Yiding Zhou <mailto:[email protected]>…

The pcap file will be synchronized to the disk when stopping the device. It takes a long time if the file is large that would cause the 'detach sync request' timeout when the device is closed under multi-process scenario.

This commit fixes the issue by using alarm handler to release dumper.

Fixes 0ecfb6c04d54 ("net/pcap: move handler to process private")
Quoted 7 lines
Quoted 5 lines
Quoted 1 line

I think you need to redesign the handshake if this the case. Forcing 30 second delay at the end of all uses of pcap is not acceptable.

@Zhang, Qi Z Do we need to redesign the handshake to fix this?

Hi, Ferruh
Sorry for the late reply.
I did not receive your email on Oct 6, I got your comments from patchwork.

"Can you please provide more details on multi-process communication and call trace, to help us think about a solution to address this issue in a more generic way (not just for pcap but for any case device close takes more than multi-process timeout)?"

I try to explain this issue with a sequence diagram, hope it can be displayed correctly in the mail.

thread intr thread intr thread thread
of secondary of secondary of primary of primary
| | | |
| | | |
rte_eal_hotplug_remove
rte_dev_remove
eal_dev_hotplug_request_to_primary
rte_mp_request_sync ------------------------------------------------------->|

                                                                                                                    |
                                                                                              handle_secondary_request
                                                                                         |<-----------------|
                                                                                         |
                                                                   __handle_secondary_request
                                                          eal_dev_hotplug_request_to_secondary
           |<------------------------------------- rte_mp_request_sync
           |
handle_primary_request--------->|
                                                           |
                            __handle_primary_request
                               local_dev_remove(this will take long time)
                                            rte_mp_reply -------------------------------->|                              
                                                                                         |
                                                                             local_dev_remove
          |<------------------------------------------------- rte_mp_reply
Quoted 3 lines

The marked 'local_dev_remove()' in the secondary process will perform a pcap file synchronization operation. When the pcap file is too large, it will take a lot of time (according to my test 100G takes 20+ seconds). This caused the processing of hot_plug message to time out.

Hi Yiding,

Thanks for the information,

Right now all MP operations timeout is hardcoded in the code and it is 5 seconds.
Do you think does it work to have an API to set custom timeout, something like rte_mp_timeout_set(), and call this from pdump?

This gives a generic solution for similar cases, not just for pcap. But my concern is if this is too much multi-process related internal detail to update, @Anatoly may comment on this.

#16 lore raw

RE: [PATCH v2] net/pcap: fix timeout of stopping device

Quoted 72 lines
Quoted 27 lines
Quoted 15 lines
Quoted 15 lines
Quoted 10 lines · On Tue, 6 Sep 2022 16:05:11 +0800 Yiding Zhou <mailto:[email protected]>…

The pcap file will be synchronized to the disk when stopping the device. It takes a long time if the file is large that would cause the 'detach sync request' timeout when the device is closed under multi-process scenario.

This commit fixes the issue by using alarm handler to release dumper.

Fixes 0ecfb6c04d54 ("net/pcap: move handler to process private")
Quoted 7 lines
Quoted 5 lines
Quoted 1 line

I think you need to redesign the handshake if this the case. Forcing 30 second delay at the end of all uses of pcap is not acceptable.

@Zhang, Qi Z Do we need to redesign the handshake to fix this?

Hi, Ferruh
Sorry for the late reply.
I did not receive your email on Oct 6, I got your comments from patchwork.

Quoted 6 lines

"Can you please provide more details on multi-process communication and call trace, to help us think about a solution to address this issue in a more generic way (not just for pcap but for any case device close takes more than multi-process timeout)?"

I try to explain this issue with a sequence diagram, hope it can be displayed

correctly in the mail.

Quoted 11 lines

thread intr thread intr thread thread
of secondary of secondary of primary of primary
| | | |
| | | |
rte_eal_hotplug_remove
rte_dev_remove
eal_dev_hotplug_request_to_primary
rte_mp_request_sync ------------------------------------------------------->|

                                                                                                                    |

handle_secondary_request

Quoted 14 lines
                                                                                         |<-----------------|
                                                                                         |
                                                                   __handle_secondary_request
                                                          eal_dev_hotplug_request_to_secondary
           |<------------------------------------- rte_mp_request_sync
           |
handle_primary_request--------->|
                                                           |
                            __handle_primary_request
                               local_dev_remove(this will take long time)
                                            rte_mp_reply -------------------------------->|
                                                                                         |
                                                                             local_dev_remove
          |<-------------------------------------------------
Quoted 3 lines

rte_mp_reply

The marked 'local_dev_remove()' in the secondary process will perform a

pcap file synchronization operation.

Quoted 1 line

When the pcap file is too large, it will take a lot of time (according to my test

100G takes 20+ seconds).

Quoted 1 line

This caused the processing of hot_plug message to time out.

Hi Yiding,

Thanks for the information,

Quoted 8 lines

Right now all MP operations timeout is hardcoded in the code and it is 5 seconds.
Do you think does it work to have an API to set custom timeout, something like rte_mp_timeout_set(), and call this from pdump?

This gives a generic solution for similar cases, not just for pcap. But my concern is if this is too much multi-process related internal detail to update, @Anatoly may comment on this.

Hi, Ferruh
For pdump case only, I think the timeout is affected by pcap's size and other system components, such as the type of FS, system memory size. It may be difficult to predict the specific time value for setting.

#17 lore raw

Re: [PATCH v2] net/pcap: fix timeout of stopping device

Quoted 77 lines · YidingX
Quoted 77 lines
Quoted 27 lines
Quoted 15 lines
Quoted 15 lines
Quoted 10 lines · On Tue, 6 Sep 2022 16:05:11 +0800 Yiding Zhou <mailto:[email protected]>…

The pcap file will be synchronized to the disk when stopping the device. It takes a long time if the file is large that would cause the 'detach sync request' timeout when the device is closed under multi-process scenario.

This commit fixes the issue by using alarm handler to release dumper.

Fixes 0ecfb6c04d54 ("net/pcap: move handler to process private")
Quoted 7 lines
Quoted 5 lines
Quoted 1 line

I think you need to redesign the handshake if this the case. Forcing 30 second delay at the end of all uses of pcap is not acceptable.

@Zhang, Qi Z Do we need to redesign the handshake to fix this?

Hi, Ferruh
Sorry for the late reply.
I did not receive your email on Oct 6, I got your comments from patchwork.

Quoted 6 lines

"Can you please provide more details on multi-process communication and call trace, to help us think about a solution to address this issue in a more generic way (not just for pcap but for any case device close takes more than multi-process timeout)?"

I try to explain this issue with a sequence diagram, hope it can be displayed

correctly in the mail.

Quoted 11 lines

thread intr thread intr thread thread
of secondary of secondary of primary of primary
| | | |
| | | |
rte_eal_hotplug_remove
rte_dev_remove
eal_dev_hotplug_request_to_primary
rte_mp_request_sync ------------------------------------------------------->|

                                                                                                                    |

handle_secondary_request

Quoted 14 lines
                                                                                         |<-----------------|
                                                                                         |
                                                                   __handle_secondary_request
                                                          eal_dev_hotplug_request_to_secondary
           |<------------------------------------- rte_mp_request_sync
           |
handle_primary_request--------->|
                                                           |
                            __handle_primary_request
                               local_dev_remove(this will take long time)
                                            rte_mp_reply -------------------------------->|
                                                                                         |
                                                                             local_dev_remove
          |<-------------------------------------------------
Quoted 3 lines

rte_mp_reply

The marked 'local_dev_remove()' in the secondary process will perform a

pcap file synchronization operation.

Quoted 1 line

When the pcap file is too large, it will take a lot of time (according to my test

100G takes 20+ seconds).

Quoted 1 line

This caused the processing of hot_plug message to time out.

Hi Yiding,

Thanks for the information,

Right now all MP operations timeout is hardcoded in the code and it is 5 seconds.
Do you think does it work to have an API to set custom timeout, something like rte_mp_timeout_set(), and call this from pdump?

Quoted 7 lines
Quoted 3 lines

This gives a generic solution for similar cases, not just for pcap. But my concern is if this is too much multi-process related internal detail to update, @Anatoly may comment on this.

Hi, Ferruh
For pdump case only, I think the timeout is affected by pcap's size and other system components, such as the type of FS, system memory size. It may be difficult to predict the specific time value for setting.

It doesn't have to be specific.

Point here is to have a multi process API to set timeout, instead of put a hardcoded timeout in pcap PMD.

#18 lore raw

RE: [PATCH v2] net/pcap: fix timeout of stopping device

Quoted 87 lines
Quoted 30 lines
Quoted 30 lines
Quoted 30 lines
Quoted 13 lines
Quoted 13 lines
Quoted 10 lines · On Tue, 6 Sep 2022 16:05:11 +0800 Yiding Zhou <mailto:[email protected]>…

The pcap file will be synchronized to the disk when stopping the device. It takes a long time if the file is large that would cause the 'detach sync request' timeout when the device is closed under multi-process scenario.

This commit fixes the issue by using alarm handler to release dumper.

Fixes 0ecfb6c04d54 ("net/pcap: move handler to process private")
Quoted 7 lines
Quoted 5 lines
Quoted 1 line

I think you need to redesign the handshake if this the case. Forcing 30 second delay at the end of all uses of pcap is not acceptable.

@Zhang, Qi Z Do we need to redesign the handshake to fix this?

Hi, Ferruh
Sorry for the late reply.
I did not receive your email on Oct 6, I got your comments from patchwork.

"Can you please provide more details on multi-process communication and call trace, to help us think about a solution to address this issue in a more generic way (not just for pcap but for any case device close takes more than multi-process timeout)?"

Quoted 6 lines
Quoted 6 lines
Quoted 2 lines

I try to explain this issue with a sequence diagram, hope it can be displayed

correctly in the mail.

Quoted 3 lines

thread intr thread intr thread thread
of secondary of secondary of primary of

primary

Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
             |                                              |                         |                          |
             |                                              |                         |                          |

rte_eal_hotplug_remove
rte_dev_remove
eal_dev_hotplug_request_to_primary
rte_mp_request_sync ------------------------------------------------------->|

Quoted 7 lines
Quoted 7 lines
Quoted 2 lines

|

handle_secondary_request

Quoted 4 lines
                                                                                         |<-----------------|
                                                                                         |

__handle_secondary_request

eal_dev_hotplug_request_to_secondary

Quoted 29 lines
Quoted 29 lines
Quoted 10 lines
           |<------------------------------------- rte_mp_request_sync
           |
handle_primary_request--------->|
                                                           |
                            __handle_primary_request
                               local_dev_remove(this will take long time)
                                            rte_mp_reply -------------------------------->|
                                                                                         |
                                                                             local_dev_remove
          |<-------------------------------------------------
Quoted 4 lines

rte_mp_reply

The marked 'local_dev_remove()' in the secondary process will perform a

pcap file synchronization operation.

Quoted 2 lines

When the pcap file is too large, it will take a lot of time (according to my test

100G takes 20+ seconds).

Quoted 1 line

This caused the processing of hot_plug message to time out.

Hi Yiding,

Thanks for the information,

Right now all MP operations timeout is hardcoded in the code and it is 5 seconds.
Do you think does it work to have an API to set custom timeout, something like rte_mp_timeout_set(), and call this from pdump?

Quoted 6 lines
Quoted 3 lines

This gives a generic solution for similar cases, not just for pcap. But my concern is if this is too much multi-process related internal detail to update, @Anatoly may comment on this.

Hi, Ferruh
For pdump case only, I think the timeout is affected by pcap's size and other

Quoted 7 lines

system components, such as the type of FS, system memory size.

Quoted 1 line

It may be difficult to predict the specific time value for setting.

It doesn't have to be specific.

Point here is to have a multi process API to set timeout, instead of put a hardcoded timeout in pcap PMD.

OK, I understood.